From 2902ca5dfc343e4c38662c5f9c540378190a24f6 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 14:54:11 +0100 Subject: [PATCH 01/65] Use a dataclass for CFConfig --- server/pyproject.toml | 1 + server/recceiver/cfstore.py | 132 ++++++++++++++++++++---------------- 2 files changed, 74 insertions(+), 59 deletions(-) diff --git a/server/pyproject.toml b/server/pyproject.toml index 2b00e593..c577c923 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -12,6 +12,7 @@ version="1.5" readme = "README.md" requires-python = ">=3.6" dependencies = [ + "dataclasses; python_version < '3.7'", "requests", "twisted", "channelfinder @ https://github.com/ChannelFinder/pyCFClient/archive/refs/tags/v3.0.0.zip" diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 10549001..5457e4dc 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -1,13 +1,11 @@ # -*- coding: utf-8 -*- import datetime -import json import logging -import os import socket import time from collections import defaultdict -from operator import itemgetter +from dataclasses import dataclass from typing import Dict, List, Set from channelfinder import ChannelFinderClient @@ -20,6 +18,7 @@ from twisted.internet.threads import deferToThread from . import interfaces +from .processors import ConfigAdapter _log = logging.getLogger(__name__) @@ -39,16 +38,50 @@ RECCEIVERID_DEFAULT = socket.gethostname() +@dataclass +class CFConfig: + alias_enabled: bool = False + record_type_enabled: bool = False + environment_variables: str = "" + info_tags: str = "" + ioc_connection_info: bool = True + record_description_enabled: bool = False + clean_on_start: bool = True + clean_on_stop: bool = True + username: str = "cfstore" + recceiver_id: str = RECCEIVERID_DEFAULT + timezone: str = "" + cf_query_limit: int = 10000 + + @staticmethod + def from_config_adapter(conf: ConfigAdapter) -> "CFConfig": + return CFConfig( + alias_enabled=conf.get("alias", False), + record_type_enabled=conf.get("recordType", False), + environment_variables=conf.get("environment_vars", ""), + info_tags=conf.get("infotags", ""), + ioc_connection_info=conf.get("iocConnectionInfo", True), + record_description_enabled=conf.get("recordDesc", False), + clean_on_start=conf.get("cleanOnStart", True), + clean_on_stop=conf.get("cleanOnStop", True), + username=conf.get("username", "cfstore"), + recceiver_id=conf.get("recceiverId", RECCEIVERID_DEFAULT), + timezone=conf.get("timezone", ""), + cf_query_limit=conf.get("findSizeLimit", 10000), + ) + + @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): _log.info("CF_INIT {name}".format(name=name)) - self.name, self.conf = name, conf + self.name = name self.channel_dict = defaultdict(list) self.iocs = dict() self.client = None self.currentTime = getCurrentTime self.lock = DeferredLock() + self.cf_config = CFConfig.from_config_adapter(conf) def startService(self): service.Service.startService(self) @@ -89,11 +122,11 @@ def _startServiceWithLock(self): RECCEIVERID_KEY, } - if self.conf.getboolean("alias"): + if self.cf_config.alias_enabled: required_properties.add("alias") - if self.conf.getboolean("recordType"): + if self.cf_config.record_type_enabled: required_properties.add("recordType") - env_vars_setting = self.conf.get("environment_vars") + env_vars_setting = self.cf_config.environment_variables self.env_vars = {} if env_vars_setting != "" and env_vars_setting is not None: env_vars_dict = dict(item.strip().split(":") for item in env_vars_setting.split(",")) @@ -103,17 +136,14 @@ def _startServiceWithLock(self): # Standard property names for CA/PVA name server connections. These are # environment variables from reccaster so take advantage of env_vars # iocConnectionInfo enabled by default - if self.conf.getboolean("iocConnectionInfo", True): + if self.cf_config.ioc_connection_info: self.env_vars["RSRV_SERVER_PORT"] = "caPort" self.env_vars["PVAS_SERVER_PORT"] = "pvaPort" required_properties.add("caPort") required_properties.add("pvaPort") - infotags_whitelist = self.conf.get("infotags", list()) - if infotags_whitelist: - record_property_names_list = [s.strip(", ") for s in infotags_whitelist.split()] - else: - record_property_names_list = [] - if self.conf.getboolean("recordDesc"): + + record_property_names_list = [s.strip(", ") for s in self.cf_config.info_tags.split()] + if self.cf_config.record_description_enabled: record_property_names_list.append("recordDesc") # Are any required properties not already present on CF? properties = required_properties - set(cf_properties) @@ -121,7 +151,7 @@ def _startServiceWithLock(self): # If so, add them too. properties.update(set(record_property_names_list) - set(cf_properties)) - owner = self.conf.get("username", "cfstore") + owner = self.cf_config.username for cf_property in properties: self.client.set(property={"name": cf_property, "owner": owner}) @@ -132,7 +162,7 @@ def _startServiceWithLock(self): _log.exception("Cannot connect to Channelfinder service") raise else: - if self.conf.getboolean("cleanOnStart", True): + if self.cf_config.clean_on_start: self.clean_service() def stopService(self): @@ -142,7 +172,7 @@ def stopService(self): def _stopServiceWithLock(self): # Set channels to inactive and close connection to client - if self.conf.getboolean("cleanOnStop", True): + if self.cf_config.clean_on_stop: self.clean_service() _log.info("CF_STOP with lock") @@ -210,9 +240,9 @@ def _commitWithThread(self, transaction): owner = ( transaction.client_infos.get("ENGINEER") or transaction.client_infos.get("CF_USERNAME") - or self.conf.get("username", "cfstore") + or self.cf_config.username ) - time = self.currentTime(timezone=self.conf.get("timezone")) + time = self.currentTime(timezone=self.cf_config.timezone) """The unique identifier for a particular IOC""" iocid = host + ":" + str(port) @@ -221,7 +251,7 @@ def _commitWithThread(self, transaction): recordInfo = {} for record_id, (record_name, record_type) in transaction.records_to_add.items(): recordInfo[record_id] = {"pvName": record_name} - if self.conf.getboolean("recordType"): + if self.cf_config.record_type_enabled: recordInfo[record_id]["recordType"] = record_type for record_id, (record_infos_to_add) in transaction.record_infos_to_add.items(): # find intersection of these sets @@ -295,7 +325,7 @@ def _commitWithThread(self, transaction): self.channel_dict[record_name].append(iocid) self.iocs[iocid]["channelcount"] += 1 """In case, alias exists""" - if self.conf.getboolean("alias"): + if self.cf_config.alias_enabled: if record_name in recordInfoByName and "aliases" in recordInfoByName[record_name]: for alias in recordInfoByName[record_name]["aliases"]: self.channel_dict[alias].append(iocid) # add iocname to pvName in dict @@ -304,7 +334,7 @@ def _commitWithThread(self, transaction): if iocid in self.channel_dict[record_name]: self.remove_channel(record_name, iocid) """In case, alias exists""" - if self.conf.getboolean("alias"): + if self.cf_config.alias_enabled: if record_name in recordInfoByName and "aliases" in recordInfoByName[record_name]: for alias in recordInfoByName[record_name]["aliases"]: self.remove_channel(alias, iocid) @@ -320,7 +350,6 @@ def _commitWithThread(self, transaction): owner, time, ) - dict_to_file(self.channel_dict, self.iocs, self.conf) def remove_channel(self, recordName, iocid): self.channel_dict[recordName].remove(iocid) @@ -339,8 +368,8 @@ def clean_service(self): """ sleep = 1 retry_limit = 5 - owner = self.conf.get("username", "cfstore") - recceiverid = self.conf.get(RECCEIVERID_KEY, RECCEIVERID_DEFAULT) + owner = self.cf_config.username + recceiverid = self.cf_config.recceiver_id while 1: try: _log.info("CF Clean Started") @@ -366,7 +395,7 @@ def clean_service(self): def get_active_channels(self, recceiverid): return self.client.findByArgs( - prepareFindArgs(self.conf, [("pvStatus", "Active"), (RECCEIVERID_KEY, recceiverid)]) + prepareFindArgs(self.cf_config, [("pvStatus", "Active"), (RECCEIVERID_KEY, recceiverid)]) ) def clean_channels(self, owner, channels): @@ -383,21 +412,6 @@ def clean_channels(self, owner, channels): ) -def dict_to_file(dict, iocs, conf): - filename = conf.get("debug_file_loc", None) - if filename: - if os.path.isfile(filename): - os.remove(filename) - list = [] - for key in dict: - list.append([key, iocs[dict[key][-1]]["hostname"], iocs[dict[key][-1]]["iocname"]]) - - list.sort(key=itemgetter(0)) - - with open(filename, "w+") as f: - json.dump(list, f) - - def create_channel(name: str, owner: str, properties: List[Dict[str, str]]): return { "name": name, @@ -439,7 +453,7 @@ def create_time_property(owner: str, time: str): def __updateCF__( - processor, + processor: CFProcessor, recordInfoByName, records_to_delete, hostName, @@ -459,8 +473,8 @@ def __updateCF__( client = processor.client channels_dict = processor.channel_dict iocs = processor.iocs - conf = processor.conf - recceiverid = conf.get(RECCEIVERID_KEY, RECCEIVERID_DEFAULT) + cf_config = processor.cf_config + recceiverid = processor.cf_config.recceiver_id new_channels = set(recordInfoByName.keys()) if iocid in iocs: @@ -481,7 +495,7 @@ def __updateCF__( channels = [] """A list of channels in channelfinder with the associated hostName and iocName""" _log.debug("Find existing channels by IOCID: {iocid}".format(iocid=iocid)) - old_channels = client.findByArgs(prepareFindArgs(conf, [("iocid", iocid)])) + old_channels = client.findByArgs(prepareFindArgs(cf_config, [("iocid", iocid)])) if processor.cancelled: raise defer.CancelledError() @@ -498,7 +512,7 @@ def __updateCF__( cf_channel, processor.managed_properties, ) - if conf.getboolean("recordType"): + if cf_config.record_type_enabled: cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( create_recordType_property( @@ -511,7 +525,7 @@ def __updateCF__( channels.append(cf_channel) _log.debug("Add existing channel to previous IOC: {s}".format(s=channels[-1])) """In case alias exist, also delete them""" - if conf.getboolean("alias"): + if cf_config.alias_enabled: if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: for alias in recordInfoByName[cf_channel["name"]]["aliases"]: if alias["name"] in channels_dict: @@ -528,7 +542,7 @@ def __updateCF__( alias, processor.managed_properties, ) - if conf.getboolean("recordType"): + if cf_config.record_type_enabled: cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( create_recordType_property( @@ -554,7 +568,7 @@ def __updateCF__( channels.append(cf_channel) _log.debug("Add orphaned channel with no IOC: {s}".format(s=channels[-1])) """Also orphan any alias""" - if conf.getboolean("alias"): + if cf_config.alias_enabled: if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: for alias in recordInfoByName[cf_channel["name"]]["aliases"]: alias["properties"] = __merge_property_lists( @@ -590,7 +604,7 @@ def __updateCF__( new_channels.remove(cf_channel["name"]) """In case, alias exist""" - if conf.getboolean("alias"): + if cf_config.alias_enabled: if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: for alias in recordInfoByName[cf_channel["name"]]["aliases"]: if alias in old_channels: @@ -651,14 +665,14 @@ def __updateCF__( for eachSearchString in searchStrings: _log.debug("Find existing channels by name: {search}".format(search=eachSearchString)) - for cf_channel in client.findByArgs(prepareFindArgs(conf, [("~name", eachSearchString)])): + for cf_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", eachSearchString)])): existingChannels[cf_channel["name"]] = cf_channel if processor.cancelled: raise defer.CancelledError() for channel_name in new_channels: newProps = create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid) - if conf.getboolean("recordType"): + if cf_config.record_type_enabled: newProps.append(create_recordType_property(owner, recordInfoByName[channel_name]["recordType"])) if channel_name in recordInfoByName and "infoProperties" in recordInfoByName[channel_name]: newProps = newProps + recordInfoByName[channel_name]["infoProperties"] @@ -677,7 +691,7 @@ def __updateCF__( channels.append(existingChannel) _log.debug("Add existing channel with different IOC: {s}".format(s=channels[-1])) """in case, alias exists, update their properties too""" - if conf.getboolean("alias"): + if cf_config.alias_enabled: if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: alProps = [create_alias_property(owner, channel_name)] for p in newProps: @@ -699,7 +713,7 @@ def __updateCF__( """New channel""" channels.append({"name": channel_name, "owner": owner, "properties": newProps}) _log.debug("Add new channel: {s}".format(s=channels[-1])) - if conf.getboolean("alias"): + if cf_config.alias_enabled: if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: alProps = [create_alias_property(owner, channel_name)] for p in newProps: @@ -709,10 +723,10 @@ def __updateCF__( _log.debug("Add new alias: {s}".format(s=channels[-1])) _log.info("Total channels to update: {nChannels} {iocName}".format(nChannels=len(channels), iocName=iocName)) if len(channels) != 0: - cf_set_chunked(client, channels, conf.get("findSizeLimit", 10000)) + cf_set_chunked(client, channels, cf_config.cf_query_limit) else: if old_channels and len(old_channels) != 0: - cf_set_chunked(client, channels, conf.get("findSizeLimit", 10000)) + cf_set_chunked(client, channels, cf_config.cf_query_limit) if processor.cancelled: raise defer.CancelledError() @@ -768,8 +782,8 @@ def getCurrentTime(timezone=False): return str(datetime.datetime.now()) -def prepareFindArgs(conf, args, size=0): - size_limit = int(conf.get("findSizeLimit", size)) +def prepareFindArgs(cf_config: CFConfig, args, size=0): + size_limit = int(cf_config.cf_query_limit) if size_limit > 0: args.append(("~size", size_limit)) return args @@ -777,7 +791,7 @@ def prepareFindArgs(conf, args, size=0): def poll( update_method, - processor, + processor: CFProcessor, recordInfoByName, records_to_delete, hostName, From 87c81960f1be7440a85383d3d5ab103705839857 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Thu, 6 Nov 2025 09:13:22 +0100 Subject: [PATCH 02/65] Use set for cf_properties --- server/recceiver/cfstore.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 5457e4dc..e9085e27 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -111,7 +111,7 @@ def _startServiceWithLock(self): """ self.client = ChannelFinderClient() try: - cf_properties = [cf_property["name"] for cf_property in self.client.getAllProperties()] + cf_properties = {cf_property["name"] for cf_property in self.client.getAllProperties()} required_properties = { "hostName", "iocName", @@ -146,14 +146,14 @@ def _startServiceWithLock(self): if self.cf_config.record_description_enabled: record_property_names_list.append("recordDesc") # Are any required properties not already present on CF? - properties = required_properties - set(cf_properties) + properties = required_properties - cf_properties # Are any whitelisted properties not already present on CF? # If so, add them too. - properties.update(set(record_property_names_list) - set(cf_properties)) + properties.update(set(record_property_names_list) - cf_properties) owner = self.cf_config.username - for cf_property in properties: - self.client.set(property={"name": cf_property, "owner": owner}) + for cf_property_name in properties: + self.client.set(property={"name": cf_property_name, "owner": owner}) self.record_property_names_list = set(record_property_names_list) self.managed_properties = required_properties.union(record_property_names_list) From 78a72ac2cec3aeee6273051342d6c186e79e69e0 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 16:02:00 +0100 Subject: [PATCH 03/65] Use dataclass for CFProperty --- server/recceiver/cfstore.py | 95 ++++++++++++++++++++++++------------- 1 file changed, 61 insertions(+), 34 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index e9085e27..a160be64 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -6,7 +6,7 @@ import time from collections import defaultdict from dataclasses import dataclass -from typing import Dict, List, Set +from typing import Dict, List, Optional, Set from channelfinder import ChannelFinderClient from requests import ConnectionError, RequestException @@ -71,6 +71,24 @@ def from_config_adapter(conf: ConfigAdapter) -> "CFConfig": ) +@dataclass +class CFProperty: + name: str + owner: str + value: Optional[str] = None + + def as_dict(self) -> Dict[str, str]: + return {"name": self.name, "owner": self.owner, "value": str(self.value)} + + @staticmethod + def from_channelfinder_dict(prop_dict: Dict[str, str]) -> "CFProperty": + return CFProperty( + name=prop_dict.get("name", ""), + owner=prop_dict.get("owner", ""), + value=prop_dict.get("value"), + ) + + @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): @@ -267,7 +285,7 @@ def _commitWithThread(self, transaction): recordInfo[record_id]["infoProperties"] = list() for infotag in recinfo_wl: recordInfo[record_id]["infoProperties"].append( - create_property(owner, infotag, record_infos_to_add[infotag]) + CFProperty(infotag, owner, record_infos_to_add[infotag]) ) for record_id, alias in transaction.aliases.items(): @@ -286,7 +304,7 @@ def _commitWithThread(self, transaction): if "infoProperties" not in recordInfo[record_id]: recordInfo[record_id]["infoProperties"] = list() recordInfo[record_id]["infoProperties"].append( - create_property(owner, cf_prop_name, transaction.client_infos.get(epics_env_var_name)) + CFProperty(cf_prop_name, owner, transaction.client_infos.get(epics_env_var_name)) ) else: _log.debug( @@ -407,12 +425,12 @@ def clean_channels(self, owner, channels): 'Update "pvStatus" property to "Inactive" for {n_channels} channels'.format(n_channels=len(new_channels)) ) self.client.update( - property=create_inactive_property(owner), + property=create_inactive_property(owner).as_dict(), channelNames=new_channels, ) -def create_channel(name: str, owner: str, properties: List[Dict[str, str]]): +def create_channel(name: str, owner: str, properties: List[CFProperty]): return { "name": name, "owner": owner, @@ -420,36 +438,28 @@ def create_channel(name: str, owner: str, properties: List[Dict[str, str]]): } -def create_property(owner: str, name: str, value: str) -> Dict[str, str]: - return { - "name": name, - "owner": owner, - "value": value, - } - - -def create_recordType_property(owner: str, recordType: str): - return create_property(owner, "recordType", recordType) +def create_recordType_property(owner: str, recordType: str) -> CFProperty: + return CFProperty("recordType", owner, recordType) -def create_alias_property(owner: str, alias: str): - return create_property(owner, "alias", alias) +def create_alias_property(owner: str, alias: str) -> CFProperty: + return CFProperty("alias", owner, alias) -def create_pvStatus_property(owner: str, pvStatus: str): - return create_property(owner, "pvStatus", pvStatus) +def create_pvStatus_property(owner: str, pvStatus: str) -> CFProperty: + return CFProperty("pvStatus", owner, pvStatus) -def create_active_property(owner: str): +def create_active_property(owner: str) -> CFProperty: return create_pvStatus_property(owner, "Active") -def create_inactive_property(owner: str): +def create_inactive_property(owner: str) -> CFProperty: return create_pvStatus_property(owner, "Inactive") -def create_time_property(owner: str, time: str): - return create_property(owner, "time", time) +def create_time_property(owner: str, time: str) -> CFProperty: + return CFProperty("time", owner, time) def __updateCF__( @@ -496,6 +506,15 @@ def __updateCF__( """A list of channels in channelfinder with the associated hostName and iocName""" _log.debug("Find existing channels by IOCID: {iocid}".format(iocid=iocid)) old_channels = client.findByArgs(prepareFindArgs(cf_config, [("iocid", iocid)])) + old_channels = [ + { + "name": ch["name"], + "owner": ch["owner"], + "properties": [CFProperty.from_channelfinder_dict(prop) for prop in ch["properties"]], + } + for ch in old_channels + ] + if processor.cancelled: raise defer.CancelledError() @@ -666,7 +685,11 @@ def __updateCF__( for eachSearchString in searchStrings: _log.debug("Find existing channels by name: {search}".format(search=eachSearchString)) for cf_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", eachSearchString)])): - existingChannels[cf_channel["name"]] = cf_channel + existingChannels[cf_channel["name"]] = { + "name": cf_channel["name"], + "owner": cf_channel["owner"], + "properties": [CFProperty.from_channelfinder_dict(prop) for prop in cf_channel["properties"]], + } if processor.cancelled: raise defer.CancelledError() @@ -733,19 +756,23 @@ def __updateCF__( def cf_set_chunked(client, channels, chunk_size=10000): for i in range(0, len(channels), chunk_size): - chunk = channels[i : i + chunk_size] + chunk = [ + {"name": ch["name"], "owner": ch["owner"], "properties": [prop.as_dict() for prop in ch["properties"]]} + for ch in channels[i : i + chunk_size] + ] + _log.debug("Updating chunk %s", chunk) client.set(channels=chunk) def create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid): return [ - create_property(owner, "hostName", hostName), - create_property(owner, "iocName", iocName), - create_property(owner, "iocid", iocid), - create_property(owner, "iocIP", iocIP), + CFProperty("hostName", owner, hostName), + CFProperty("iocName", owner, iocName), + CFProperty("iocid", owner, iocid), + CFProperty("iocIP", owner, iocIP), create_active_property(owner), create_time_property(owner, iocTime), - create_property(owner, RECCEIVERID_KEY, recceiverid), + CFProperty(RECCEIVERID_KEY, owner, recceiverid), ] @@ -762,16 +789,16 @@ def create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, def __merge_property_lists( - newProperties: List[Dict[str, str]], channel: Dict[str, List[Dict[str, str]]], managed_properties: Set[str] = set() -) -> List[Dict[str, str]]: + newProperties: List[CFProperty], channel: Dict[str, List[CFProperty]], managed_properties: Set[str] = set() +) -> List[CFProperty]: """ Merges two lists of properties ensuring that there are no 2 properties with the same name In case of overlap between the new and old property lists the new property list wins out """ - newPropNames = [p["name"] for p in newProperties] + newPropNames = [p.name for p in newProperties] for oldProperty in channel["properties"]: - if oldProperty["name"] not in newPropNames and (oldProperty["name"] not in managed_properties): + if oldProperty.name not in newPropNames and (oldProperty.name not in managed_properties): newProperties = newProperties + [oldProperty] return newProperties From 7ebeaff50bf6f8af412dc45df17ed10ecd6f3fe2 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 16:23:25 +0100 Subject: [PATCH 04/65] Add committransaction dataclass --- server/recceiver/cfstore.py | 16 +++------------- server/recceiver/interfaces.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index a160be64..67d3556b 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -10,28 +10,18 @@ from channelfinder import ChannelFinderClient from requests import ConnectionError, RequestException -from zope.interface import implementer - from twisted.application import service from twisted.internet import defer from twisted.internet.defer import DeferredLock from twisted.internet.threads import deferToThread +from zope.interface import implementer from . import interfaces +from .interfaces import CommitTransaction from .processors import ConfigAdapter _log = logging.getLogger(__name__) -# ITRANSACTION FORMAT: -# -# source_address = source address -# records_to_add = records ein added ( recname, rectype, {key:val}) -# records_to_delete = a set() of records which are being removed -# client_infos = dictionary of client client_infos -# record_infos_to_add = additional client_infos being added to existing records -# "recid: {key:value}" -# - __all__ = ["CFProcessor"] RECCEIVERID_KEY = "recceiverID" @@ -234,7 +224,7 @@ def chainResult(_ignored): t.addCallbacks(chainResult, chainError) return d - def _commitWithThread(self, transaction): + def _commitWithThread(self, transaction: CommitTransaction): if not self.running: raise defer.CancelledError( "CF Processor is not running (transaction: {host}:{port})", diff --git a/server/recceiver/interfaces.py b/server/recceiver/interfaces.py index 6a139109..1ac6e8f2 100644 --- a/server/recceiver/interfaces.py +++ b/server/recceiver/interfaces.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- +from dataclasses import dataclass +from typing import Dict, List, Set, Tuple + from zope.interface import Attribute, Interface from twisted.application import service @@ -21,6 +24,24 @@ class ITransaction(Interface): """) +@dataclass +class SourceAddress: + host: str + port: int + + +@dataclass +class CommitTransaction: + source_address: SourceAddress + client_infos: Dict[str, str] + records_to_add: Dict[str, Tuple[str, str]] + records_to_delete: Set[str] + record_infos_to_add: Dict[str, Dict[str, str]] + aliases: Dict[str, List[str]] + initial: bool + connected: bool + + class IProcessor(service.IService): def commit(transaction): """Consume and process the provided ITransaction. From 981fdabedbe741d4d152fcf34b682b91849de146 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 16:41:29 +0100 Subject: [PATCH 05/65] Add RecordInfo dataclass --- server/recceiver/cfstore.py | 119 ++++++++++++++++++++---------------- 1 file changed, 65 insertions(+), 54 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 67d3556b..2acc5280 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -5,7 +5,7 @@ import socket import time from collections import defaultdict -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Dict, List, Optional, Set from channelfinder import ChannelFinderClient @@ -79,6 +79,14 @@ def from_channelfinder_dict(prop_dict: Dict[str, str]) -> "CFProperty": ) +@dataclass +class RecordInfo: + pvName: str + recordType: Optional[str] = None + infoProperties: List[CFProperty] = field(default_factory=list) + aliases: List[str] = field(default_factory=list) + + @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): @@ -256,11 +264,11 @@ def _commitWithThread(self, transaction: CommitTransaction): iocid = host + ":" + str(port) _log.debug("transaction: {s}".format(s=repr(transaction))) - recordInfo = {} + recordInfo: Dict[str, RecordInfo] = {} for record_id, (record_name, record_type) in transaction.records_to_add.items(): - recordInfo[record_id] = {"pvName": record_name} + recordInfo[record_id] = RecordInfo(pvName=record_name, recordType=None, infoProperties=[], aliases=[]) if self.cf_config.record_type_enabled: - recordInfo[record_id]["recordType"] = record_type + recordInfo[record_id].recordType = record_type for record_id, (record_infos_to_add) in transaction.record_infos_to_add.items(): # find intersection of these sets if record_id not in recordInfo: @@ -272,9 +280,8 @@ def _commitWithThread(self, transaction: CommitTransaction): continue recinfo_wl = [p for p in self.record_property_names_list if p in record_infos_to_add.keys()] if recinfo_wl: - recordInfo[record_id]["infoProperties"] = list() for infotag in recinfo_wl: - recordInfo[record_id]["infoProperties"].append( + recordInfo[record_id].infoProperties.append( CFProperty(infotag, owner, record_infos_to_add[infotag]) ) @@ -286,14 +293,12 @@ def _commitWithThread(self, transaction: CommitTransaction): ) ) continue - recordInfo[record_id]["aliases"] = alias + recordInfo[record_id].aliases = alias for record_id in recordInfo: for epics_env_var_name, cf_prop_name in self.env_vars.items(): if transaction.client_infos.get(epics_env_var_name) is not None: - if "infoProperties" not in recordInfo[record_id]: - recordInfo[record_id]["infoProperties"] = list() - recordInfo[record_id]["infoProperties"].append( + recordInfo[record_id].infoProperties.append( CFProperty(cf_prop_name, owner, transaction.client_infos.get(epics_env_var_name)) ) else: @@ -308,14 +313,12 @@ def _commitWithThread(self, transaction: CommitTransaction): recordInfoByName = {} for record_id, (info) in recordInfo.items(): - if info["pvName"] in recordInfoByName: + if info.pvName in recordInfoByName: _log.warning( - "Commit contains multiple records with PV name: {pv} ({iocid})".format( - pv=info["pvName"], iocid=iocid - ) + "Commit contains multiple records with PV name: {pv} ({iocid})".format(pv=info.pvName, iocid=iocid) ) continue - recordInfoByName[info["pvName"]] = info + recordInfoByName[info.pvName] = info if transaction.initial: """Add IOC to source list """ @@ -334,8 +337,8 @@ def _commitWithThread(self, transaction: CommitTransaction): self.iocs[iocid]["channelcount"] += 1 """In case, alias exists""" if self.cf_config.alias_enabled: - if record_name in recordInfoByName and "aliases" in recordInfoByName[record_name]: - for alias in recordInfoByName[record_name]["aliases"]: + if record_name in recordInfoByName: + for alias in recordInfoByName[record_name].aliases: self.channel_dict[alias].append(iocid) # add iocname to pvName in dict self.iocs[iocid]["channelcount"] += 1 for record_name in records_to_delete: @@ -343,8 +346,8 @@ def _commitWithThread(self, transaction: CommitTransaction): self.remove_channel(record_name, iocid) """In case, alias exists""" if self.cf_config.alias_enabled: - if record_name in recordInfoByName and "aliases" in recordInfoByName[record_name]: - for alias in recordInfoByName[record_name]["aliases"]: + if record_name in recordInfoByName: + for alias in recordInfoByName[record_name].aliases: self.remove_channel(alias, iocid) poll( __updateCF__, @@ -454,7 +457,7 @@ def create_time_property(owner: str, time: str) -> CFProperty: def __updateCF__( processor: CFProcessor, - recordInfoByName, + recordInfoByName: Dict[str, RecordInfo], records_to_delete, hostName, iocName, @@ -535,11 +538,13 @@ def __updateCF__( _log.debug("Add existing channel to previous IOC: {s}".format(s=channels[-1])) """In case alias exist, also delete them""" if cf_config.alias_enabled: - if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: - for alias in recordInfoByName[cf_channel["name"]]["aliases"]: - if alias["name"] in channels_dict: - alias["owner"] = iocs[channels_dict[alias["name"]][-1]]["owner"] - alias["properties"] = __merge_property_lists( + if cf_channel["name"] in recordInfoByName: + for alias_name in recordInfoByName[cf_channel["name"]].aliases: + # TODO Remove? This code couldn't have been working.... + alias_channel = {"name": alias_name, "properties": [], "owner": ""} + if alias_name in channels_dict: + alias_channel["owner"] = iocs[channels_dict[alias_name][-1]]["owner"] + alias_channel["properties"] = __merge_property_lists( create_default_properties( owner, iocTime, @@ -548,7 +553,7 @@ def __updateCF__( iocs, cf_channel, ), - alias, + alias_channel, processor.managed_properties, ) if cf_config.record_type_enabled: @@ -556,13 +561,13 @@ def __updateCF__( cf_channel["properties"].append( create_recordType_property( owner, - iocs[channels_dict[alias["name"]][-1]]["recordType"], + iocs[channels_dict[alias_name][-1]]["recordType"], ) ), cf_channel, processor.managed_properties, ) - channels.append(alias) + channels.append(alias_channel) _log.debug("Add existing alias to previous IOC: {s}".format(s=channels[-1])) else: @@ -578,16 +583,17 @@ def __updateCF__( _log.debug("Add orphaned channel with no IOC: {s}".format(s=channels[-1])) """Also orphan any alias""" if cf_config.alias_enabled: - if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: - for alias in recordInfoByName[cf_channel["name"]]["aliases"]: - alias["properties"] = __merge_property_lists( + if cf_channel["name"] in recordInfoByName: + for alias_name in recordInfoByName[cf_channel["name"]].aliases: + alias_channel = {"name": alias_name, "properties": [], "owner": ""} + alias_channel["properties"] = __merge_property_lists( [ create_inactive_property(owner), create_time_property(owner, iocTime), ], - alias, + alias_channel, ) - channels.append(alias) + channels.append(alias_channel) _log.debug("Add orphaned alias with no IOC: {s}".format(s=channels[-1])) else: if cf_channel["name"] in new_channels: # case: channel in old and new @@ -614,20 +620,21 @@ def __updateCF__( """In case, alias exist""" if cf_config.alias_enabled: - if cf_channel["name"] in recordInfoByName and "aliases" in recordInfoByName[cf_channel["name"]]: - for alias in recordInfoByName[cf_channel["name"]]["aliases"]: - if alias in old_channels: + if cf_channel["name"] in recordInfoByName: + for alias_name in recordInfoByName[cf_channel["name"]].aliases: + if alias_name in old_channels: """alias exists in old list""" - alias["properties"] = __merge_property_lists( + alias_channel = {"name": alias_name, "properties": [], "owner": ""} + alias_channel["properties"] = __merge_property_lists( [ create_active_property(owner), create_time_property(owner, iocTime), ], - alias, + alias_channel, processor.managed_properties, ) - channels.append(alias) - new_channels.remove(alias["name"]) + channels.append(alias_channel) + new_channels.remove(alias_name) else: """alias exists but not part of old list""" aprops = __merge_property_lists( @@ -644,12 +651,12 @@ def __updateCF__( ) channels.append( create_channel( - alias["name"], + alias_name, owner, aprops, ) ) - new_channels.remove(alias["name"]) + new_channels.remove(alias_name) _log.debug("Add existing alias with same IOC: {s}".format(s=channels[-1])) # now pvNames contains a list of pv's new on this host/ioc """A dictionary representing the current channelfinder information associated with the pvNames""" @@ -685,10 +692,14 @@ def __updateCF__( for channel_name in new_channels: newProps = create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid) - if cf_config.record_type_enabled: - newProps.append(create_recordType_property(owner, recordInfoByName[channel_name]["recordType"])) - if channel_name in recordInfoByName and "infoProperties" in recordInfoByName[channel_name]: - newProps = newProps + recordInfoByName[channel_name]["infoProperties"] + if ( + cf_config.record_type_enabled + and channel_name in recordInfoByName + and recordInfoByName[channel_name].recordType + ): + newProps.append(create_recordType_property(owner, recordInfoByName[channel_name].recordType)) + if channel_name in recordInfoByName: + newProps = newProps + recordInfoByName[channel_name].infoProperties if channel_name in existingChannels: _log.debug( @@ -705,13 +716,13 @@ def __updateCF__( _log.debug("Add existing channel with different IOC: {s}".format(s=channels[-1])) """in case, alias exists, update their properties too""" if cf_config.alias_enabled: - if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: + if channel_name in recordInfoByName: alProps = [create_alias_property(owner, channel_name)] for p in newProps: alProps.append(p) - for alias in recordInfoByName[channel_name]["aliases"]: - if alias in existingChannels: - ach = existingChannels[alias] + for alias_name in recordInfoByName[channel_name].aliases: + if alias_name in existingChannels: + ach = existingChannels[alias_name] ach["properties"] = __merge_property_lists( alProps, ach, @@ -719,7 +730,7 @@ def __updateCF__( ) channels.append(ach) else: - channels.append(create_channel(alias, owner, alProps)) + channels.append(create_channel(alias_name, owner, alProps)) _log.debug("Add existing alias with different IOC: {s}".format(s=channels[-1])) else: @@ -727,11 +738,11 @@ def __updateCF__( channels.append({"name": channel_name, "owner": owner, "properties": newProps}) _log.debug("Add new channel: {s}".format(s=channels[-1])) if cf_config.alias_enabled: - if channel_name in recordInfoByName and "aliases" in recordInfoByName[channel_name]: + if channel_name in recordInfoByName: alProps = [create_alias_property(owner, channel_name)] for p in newProps: alProps.append(p) - for alias in recordInfoByName[channel_name]["aliases"]: + for alias in recordInfoByName[channel_name].aliases: channels.append({"name": alias, "owner": owner, "properties": alProps}) _log.debug("Add new alias: {s}".format(s=channels[-1])) _log.info("Total channels to update: {nChannels} {iocName}".format(nChannels=len(channels), iocName=iocName)) @@ -809,7 +820,7 @@ def prepareFindArgs(cf_config: CFConfig, args, size=0): def poll( update_method, processor: CFProcessor, - recordInfoByName, + recordInfoByName: Dict[str, RecordInfo], records_to_delete, hostName, iocName, From 56bacf7ee3deeb8d465d80d2c4722f9c1512a092 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 16:49:22 +0100 Subject: [PATCH 06/65] Add CFPropertyName enum --- server/recceiver/cfstore.py | 63 +++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 2acc5280..aa798894 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import datetime +import enum import logging import socket import time @@ -24,7 +25,6 @@ __all__ = ["CFProcessor"] -RECCEIVERID_KEY = "recceiverID" RECCEIVERID_DEFAULT = socket.gethostname() @@ -87,6 +87,21 @@ class RecordInfo: aliases: List[str] = field(default_factory=list) +class CFPropertyName(enum.Enum): + hostName = enum.auto() + iocName = enum.auto() + iocid = enum.auto() + iocIP = enum.auto() + pvStatus = enum.auto() + time = enum.auto() + recceiverID = enum.auto() + alias = enum.auto() + recordType = enum.auto() + recordDesc = enum.auto() + caPort = enum.auto() + pvaPort = enum.auto() + + @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): @@ -129,19 +144,19 @@ def _startServiceWithLock(self): try: cf_properties = {cf_property["name"] for cf_property in self.client.getAllProperties()} required_properties = { - "hostName", - "iocName", - "pvStatus", - "time", - "iocid", - "iocIP", - RECCEIVERID_KEY, + CFPropertyName.hostName.name, + CFPropertyName.iocName.name, + CFPropertyName.iocid.name, + CFPropertyName.iocIP.name, + CFPropertyName.pvStatus.name, + CFPropertyName.time.name, + CFPropertyName.recceiverID.name, } if self.cf_config.alias_enabled: - required_properties.add("alias") + required_properties.add(CFPropertyName.alias.name) if self.cf_config.record_type_enabled: - required_properties.add("recordType") + required_properties.add(CFPropertyName.recordType.name) env_vars_setting = self.cf_config.environment_variables self.env_vars = {} if env_vars_setting != "" and env_vars_setting is not None: @@ -155,12 +170,12 @@ def _startServiceWithLock(self): if self.cf_config.ioc_connection_info: self.env_vars["RSRV_SERVER_PORT"] = "caPort" self.env_vars["PVAS_SERVER_PORT"] = "pvaPort" - required_properties.add("caPort") - required_properties.add("pvaPort") + required_properties.add(CFPropertyName.caPort.name) + required_properties.add(CFPropertyName.pvaPort.name) record_property_names_list = [s.strip(", ") for s in self.cf_config.info_tags.split()] if self.cf_config.record_description_enabled: - record_property_names_list.append("recordDesc") + record_property_names_list.append(CFPropertyName.recordDesc.name) # Are any required properties not already present on CF? properties = required_properties - cf_properties # Are any whitelisted properties not already present on CF? @@ -406,7 +421,9 @@ def clean_service(self): def get_active_channels(self, recceiverid): return self.client.findByArgs( - prepareFindArgs(self.cf_config, [("pvStatus", "Active"), (RECCEIVERID_KEY, recceiverid)]) + prepareFindArgs( + self.cf_config, [(CFPropertyName.pvStatus.name, "Active"), (CFPropertyName.recceiverID.name, recceiverid)] + ) ) def clean_channels(self, owner, channels): @@ -432,15 +449,15 @@ def create_channel(name: str, owner: str, properties: List[CFProperty]): def create_recordType_property(owner: str, recordType: str) -> CFProperty: - return CFProperty("recordType", owner, recordType) + return CFProperty(CFPropertyName.recordType.name, owner, recordType) def create_alias_property(owner: str, alias: str) -> CFProperty: - return CFProperty("alias", owner, alias) + return CFProperty(CFPropertyName.alias.name, owner, alias) def create_pvStatus_property(owner: str, pvStatus: str) -> CFProperty: - return CFProperty("pvStatus", owner, pvStatus) + return CFProperty(CFPropertyName.pvStatus.name, owner, pvStatus) def create_active_property(owner: str) -> CFProperty: @@ -452,7 +469,7 @@ def create_inactive_property(owner: str) -> CFProperty: def create_time_property(owner: str, time: str) -> CFProperty: - return CFProperty("time", owner, time) + return CFProperty(CFPropertyName.time.name, owner, time) def __updateCF__( @@ -767,13 +784,13 @@ def cf_set_chunked(client, channels, chunk_size=10000): def create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid): return [ - CFProperty("hostName", owner, hostName), - CFProperty("iocName", owner, iocName), - CFProperty("iocid", owner, iocid), - CFProperty("iocIP", owner, iocIP), + CFProperty(CFPropertyName.hostName.name, owner, hostName), + CFProperty(CFPropertyName.iocName.name, owner, iocName), + CFProperty(CFPropertyName.iocid.name, owner, iocid), + CFProperty(CFPropertyName.iocIP.name, owner, iocIP), create_active_property(owner), create_time_property(owner, iocTime), - CFProperty(RECCEIVERID_KEY, owner, recceiverid), + CFProperty(CFPropertyName.recceiverID.name, owner, recceiverid), ] From d9b1c0c490d6ce5c84dcfce1930692d6eca116ec Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 16:51:23 +0100 Subject: [PATCH 07/65] Add PVStatus enum --- server/recceiver/cfstore.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index aa798894..9baeaef5 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -102,6 +102,11 @@ class CFPropertyName(enum.Enum): pvaPort = enum.auto() +class PVStatus(enum.Enum): + Active = enum.auto() + Inactive = enum.auto() + + @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): @@ -422,7 +427,8 @@ def clean_service(self): def get_active_channels(self, recceiverid): return self.client.findByArgs( prepareFindArgs( - self.cf_config, [(CFPropertyName.pvStatus.name, "Active"), (CFPropertyName.recceiverID.name, recceiverid)] + self.cf_config, + [(CFPropertyName.pvStatus.name, PVStatus.Active.name), (CFPropertyName.recceiverID.name, recceiverid)], ) ) @@ -461,11 +467,11 @@ def create_pvStatus_property(owner: str, pvStatus: str) -> CFProperty: def create_active_property(owner: str) -> CFProperty: - return create_pvStatus_property(owner, "Active") + return create_pvStatus_property(owner, PVStatus.Active.name) def create_inactive_property(owner: str) -> CFProperty: - return create_pvStatus_property(owner, "Inactive") + return create_pvStatus_property(owner, PVStatus.Inactive.name) def create_time_property(owner: str, time: str) -> CFProperty: From 30f7c07bdfcd759ae080c4833a156d8a805d6216 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 17:19:04 +0100 Subject: [PATCH 08/65] Create IocInfo Dataclass --- server/recceiver/cfstore.py | 241 +++++++++++++++++------------------- 1 file changed, 113 insertions(+), 128 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 9baeaef5..d048d1e7 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -107,13 +107,29 @@ class PVStatus(enum.Enum): Inactive = enum.auto() +@dataclass +class IocInfo: + host: str + hostname: str + ioc_name: str + ioc_IP: str + owner: str + time: str + channelcount: int + port: int + + @property + def ioc_id(self): + return self.host + ":" + str(self.port) + + @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): _log.info("CF_INIT {name}".format(name=name)) self.name = name - self.channel_dict = defaultdict(list) - self.iocs = dict() + self.channel_ioc_ids = defaultdict(list) + self.iocs: Dict[str, IocInfo] = dict() self.client = None self.currentTime = getCurrentTime self.lock = DeferredLock() @@ -268,20 +284,23 @@ def _commitWithThread(self, transaction: CommitTransaction): {record_id: { "pvName":"recordName", "infoProperties":{propName:value, ...}}} """ - - host = transaction.source_address.host - port = transaction.source_address.port - iocName = transaction.client_infos.get("IOCNAME") or transaction.source_address.port - hostName = transaction.client_infos.get("HOSTNAME") or transaction.source_address.host - owner = ( - transaction.client_infos.get("ENGINEER") - or transaction.client_infos.get("CF_USERNAME") - or self.cf_config.username + ioc_info = IocInfo( + host=transaction.source_address.host, + hostname=transaction.client_infos.get("HOSTNAME") or transaction.source_address.host, + ioc_name=transaction.client_infos.get("IOCNAME") or str(transaction.source_address.port), + ioc_IP=transaction.source_address.host, + owner=( + transaction.client_infos.get("ENGINEER") + or transaction.client_infos.get("CF_USERNAME") + or self.cf_config.username + ), + time=self.currentTime(timezone=self.cf_config.timezone), + port=transaction.source_address.port, + channelcount=0, ) - time = self.currentTime(timezone=self.cf_config.timezone) """The unique identifier for a particular IOC""" - iocid = host + ":" + str(port) + iocid = ioc_info.ioc_id _log.debug("transaction: {s}".format(s=repr(transaction))) recordInfo: Dict[str, RecordInfo] = {} @@ -302,7 +321,7 @@ def _commitWithThread(self, transaction: CommitTransaction): if recinfo_wl: for infotag in recinfo_wl: recordInfo[record_id].infoProperties.append( - CFProperty(infotag, owner, record_infos_to_add[infotag]) + CFProperty(infotag, ioc_info.owner, record_infos_to_add[infotag]) ) for record_id, alias in transaction.aliases.items(): @@ -319,13 +338,13 @@ def _commitWithThread(self, transaction: CommitTransaction): for epics_env_var_name, cf_prop_name in self.env_vars.items(): if transaction.client_infos.get(epics_env_var_name) is not None: recordInfo[record_id].infoProperties.append( - CFProperty(cf_prop_name, owner, transaction.client_infos.get(epics_env_var_name)) + CFProperty(cf_prop_name, ioc_info.owner, transaction.client_infos.get(epics_env_var_name)) ) else: _log.debug( "EPICS environment var %s listed in environment_vars setting list not found in this IOC: %s", epics_env_var_name, - iocName, + ioc_info.ioc_name, ) records_to_delete = list(transaction.records_to_delete) @@ -342,56 +361,38 @@ def _commitWithThread(self, transaction: CommitTransaction): if transaction.initial: """Add IOC to source list """ - self.iocs[iocid] = { - "iocname": iocName, - "hostname": hostName, - "iocIP": host, - "owner": owner, - "time": time, - "channelcount": 0, - } + self.iocs[iocid] = ioc_info if not transaction.connected: - records_to_delete.extend(self.channel_dict.keys()) + records_to_delete.extend(self.channel_ioc_ids.keys()) for record_name in recordInfoByName.keys(): - self.channel_dict[record_name].append(iocid) - self.iocs[iocid]["channelcount"] += 1 + self.channel_ioc_ids[record_name].append(iocid) + self.iocs[iocid].channelcount += 1 """In case, alias exists""" if self.cf_config.alias_enabled: if record_name in recordInfoByName: for alias in recordInfoByName[record_name].aliases: - self.channel_dict[alias].append(iocid) # add iocname to pvName in dict - self.iocs[iocid]["channelcount"] += 1 + self.channel_ioc_ids[alias].append(iocid) # add iocname to pvName in dict + self.iocs[iocid].channelcount += 1 for record_name in records_to_delete: - if iocid in self.channel_dict[record_name]: + if iocid in self.channel_ioc_ids[record_name]: self.remove_channel(record_name, iocid) """In case, alias exists""" if self.cf_config.alias_enabled: if record_name in recordInfoByName: for alias in recordInfoByName[record_name].aliases: self.remove_channel(alias, iocid) - poll( - __updateCF__, - self, - recordInfoByName, - records_to_delete, - hostName, - iocName, - host, - iocid, - owner, - time, - ) + poll(__updateCF__, self, recordInfoByName, records_to_delete, ioc_info) def remove_channel(self, recordName, iocid): - self.channel_dict[recordName].remove(iocid) + self.channel_ioc_ids[recordName].remove(iocid) if iocid in self.iocs: - self.iocs[iocid]["channelcount"] -= 1 - if self.iocs[iocid]["channelcount"] == 0: + self.iocs[iocid].channelcount -= 1 + if self.iocs[iocid].channelcount == 0: self.iocs.pop(iocid, None) - elif self.iocs[iocid]["channelcount"] < 0: + elif self.iocs[iocid].channelcount < 0: _log.error("Channel count negative: {s}", s=iocid) - if len(self.channel_dict[recordName]) <= 0: # case: channel has no more iocs - del self.channel_dict[recordName] + if len(self.channel_ioc_ids[recordName]) <= 0: # case: channel has no more iocs + del self.channel_ioc_ids[recordName] def clean_service(self): """ @@ -478,41 +479,26 @@ def create_time_property(owner: str, time: str) -> CFProperty: return CFProperty(CFPropertyName.time.name, owner, time) -def __updateCF__( - processor: CFProcessor, - recordInfoByName: Dict[str, RecordInfo], - records_to_delete, - hostName, - iocName, - iocIP, - iocid, - owner, - iocTime, -): - _log.info("CF Update IOC: {iocid}".format(iocid=iocid)) +def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): + _log.info("CF Update IOC: {iocid}".format(iocid=ioc_info.ioc_id)) _log.debug( "CF Update IOC: {iocid} recordInfoByName {recordInfoByName}".format( - iocid=iocid, recordInfoByName=recordInfoByName + iocid=ioc_info.ioc_id, recordInfoByName=recordInfoByName ) ) # Consider making this function a class methed then 'processor' simply becomes 'self' client = processor.client - channels_dict = processor.channel_dict + channel_ioc_ids = processor.channel_ioc_ids iocs = processor.iocs cf_config = processor.cf_config recceiverid = processor.cf_config.recceiver_id new_channels = set(recordInfoByName.keys()) + iocid = ioc_info.ioc_id - if iocid in iocs: - hostName = iocs[iocid]["hostname"] - iocName = iocs[iocid]["iocname"] - owner = iocs[iocid]["owner"] - iocTime = iocs[iocid]["time"] - iocIP = iocs[iocid]["iocIP"] - else: + if iocid not in iocs: _log.warning("IOC Env Info not found: {iocid}".format(iocid=iocid)) - if hostName is None or iocName is None: + if ioc_info.hostname is None or ioc_info.ioc_name is None: raise Exception("missing hostName or iocName") if processor.cancelled: @@ -540,10 +526,10 @@ def __updateCF__( len(new_channels) == 0 or cf_channel["name"] in records_to_delete ): # case: empty commit/del, remove all reference to ioc _log.debug("Channel {s} exists in Channelfinder not in new_channels".format(s=cf_channel["name"])) - if cf_channel["name"] in channels_dict: - cf_channel["owner"] = iocs[channels_dict[cf_channel["name"]][-1]]["owner"] + if cf_channel["name"] in channel_ioc_ids: + cf_channel["owner"] = iocs[channel_ioc_ids[cf_channel["name"]][-1]].owner cf_channel["properties"] = __merge_property_lists( - create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel), + create_default_properties(ioc_info, recceiverid, channel_ioc_ids, iocs, cf_channel), cf_channel, processor.managed_properties, ) @@ -551,7 +537,7 @@ def __updateCF__( cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( create_recordType_property( - owner, iocs[channels_dict[cf_channel["name"]][-1]]["recordType"] + ioc_info.owner, iocs[channel_ioc_ids[cf_channel["name"]][-1]]["recordType"] ) ), cf_channel, @@ -565,14 +551,13 @@ def __updateCF__( for alias_name in recordInfoByName[cf_channel["name"]].aliases: # TODO Remove? This code couldn't have been working.... alias_channel = {"name": alias_name, "properties": [], "owner": ""} - if alias_name in channels_dict: - alias_channel["owner"] = iocs[channels_dict[alias_name][-1]]["owner"] + if alias_name in channel_ioc_ids: + alias_channel["owner"] = iocs[channel_ioc_ids[alias_name][-1]].owner alias_channel["properties"] = __merge_property_lists( create_default_properties( - owner, - iocTime, + ioc_info, recceiverid, - channels_dict, + channel_ioc_ids, iocs, cf_channel, ), @@ -583,8 +568,8 @@ def __updateCF__( cf_channel["properties"] = __merge_property_lists( cf_channel["properties"].append( create_recordType_property( - owner, - iocs[channels_dict[alias_name][-1]]["recordType"], + ioc_info.owner, + iocs[channel_ioc_ids[alias_name][-1]]["recordType"], ) ), cf_channel, @@ -597,8 +582,8 @@ def __updateCF__( """Orphan the channel : mark as inactive, keep the old hostName and iocName""" cf_channel["properties"] = __merge_property_lists( [ - create_inactive_property(owner), - create_time_property(owner, iocTime), + create_inactive_property(ioc_info.owner), + create_time_property(ioc_info.owner, ioc_info.time), ], cf_channel, ) @@ -611,8 +596,8 @@ def __updateCF__( alias_channel = {"name": alias_name, "properties": [], "owner": ""} alias_channel["properties"] = __merge_property_lists( [ - create_inactive_property(owner), - create_time_property(owner, iocTime), + create_inactive_property(ioc_info.owner), + create_time_property(ioc_info.owner, ioc_info.time), ], alias_channel, ) @@ -631,8 +616,8 @@ def __updateCF__( ) cf_channel["properties"] = __merge_property_lists( [ - create_active_property(owner), - create_time_property(owner, iocTime), + create_active_property(ioc_info.owner), + create_time_property(ioc_info.owner, ioc_info.time), ], cf_channel, processor.managed_properties, @@ -650,8 +635,8 @@ def __updateCF__( alias_channel = {"name": alias_name, "properties": [], "owner": ""} alias_channel["properties"] = __merge_property_lists( [ - create_active_property(owner), - create_time_property(owner, iocTime), + create_active_property(ioc_info.owner), + create_time_property(ioc_info.owner, ioc_info.time), ], alias_channel, processor.managed_properties, @@ -662,10 +647,10 @@ def __updateCF__( """alias exists but not part of old list""" aprops = __merge_property_lists( [ - create_active_property(owner), - create_time_property(owner, iocTime), + create_active_property(ioc_info.owner), + create_time_property(ioc_info.owner, ioc_info.time), create_alias_property( - owner, + ioc_info.owner, cf_channel["name"], ), ], @@ -675,7 +660,7 @@ def __updateCF__( channels.append( create_channel( alias_name, - owner, + ioc_info.owner, aprops, ) ) @@ -714,13 +699,21 @@ def __updateCF__( raise defer.CancelledError() for channel_name in new_channels: - newProps = create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid) + newProps = create_properties( + ioc_info.owner, + ioc_info.time, + recceiverid, + ioc_info.hostname, + ioc_info.ioc_name, + ioc_info.ioc_IP, + ioc_info.ioc_id, + ) if ( cf_config.record_type_enabled and channel_name in recordInfoByName and recordInfoByName[channel_name].recordType ): - newProps.append(create_recordType_property(owner, recordInfoByName[channel_name].recordType)) + newProps.append(create_recordType_property(ioc_info.owner, recordInfoByName[channel_name].recordType)) if channel_name in recordInfoByName: newProps = newProps + recordInfoByName[channel_name].infoProperties @@ -740,7 +733,7 @@ def __updateCF__( """in case, alias exists, update their properties too""" if cf_config.alias_enabled: if channel_name in recordInfoByName: - alProps = [create_alias_property(owner, channel_name)] + alProps = [create_alias_property(ioc_info.owner, channel_name)] for p in newProps: alProps.append(p) for alias_name in recordInfoByName[channel_name].aliases: @@ -753,22 +746,24 @@ def __updateCF__( ) channels.append(ach) else: - channels.append(create_channel(alias_name, owner, alProps)) + channels.append(create_channel(alias_name, ioc_info.owner, alProps)) _log.debug("Add existing alias with different IOC: {s}".format(s=channels[-1])) else: """New channel""" - channels.append({"name": channel_name, "owner": owner, "properties": newProps}) + channels.append({"name": channel_name, "owner": ioc_info.owner, "properties": newProps}) _log.debug("Add new channel: {s}".format(s=channels[-1])) if cf_config.alias_enabled: if channel_name in recordInfoByName: - alProps = [create_alias_property(owner, channel_name)] + alProps = [create_alias_property(ioc_info.owner, channel_name)] for p in newProps: alProps.append(p) for alias in recordInfoByName[channel_name].aliases: - channels.append({"name": alias, "owner": owner, "properties": alProps}) + channels.append({"name": alias, "owner": ioc_info.owner, "properties": alProps}) _log.debug("Add new alias: {s}".format(s=channels[-1])) - _log.info("Total channels to update: {nChannels} {iocName}".format(nChannels=len(channels), iocName=iocName)) + _log.info( + "Total channels to update: {nChannels} {iocName}".format(nChannels=len(channels), iocName=ioc_info.ioc_name) + ) if len(channels) != 0: cf_set_chunked(client, channels, cf_config.cf_query_limit) else: @@ -788,7 +783,7 @@ def cf_set_chunked(client, channels, chunk_size=10000): client.set(channels=chunk) -def create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, iocid): +def create_properties(owner: str, iocTime: str, recceiverid: str, hostName: str, iocName: str, iocIP: str, iocid: str): return [ CFProperty(CFPropertyName.hostName.name, owner, hostName), CFProperty(CFPropertyName.iocName.name, owner, iocName), @@ -800,15 +795,19 @@ def create_properties(owner, iocTime, recceiverid, hostName, iocName, iocIP, ioc ] -def create_default_properties(owner, iocTime, recceiverid, channels_dict, iocs, cf_channel): +def create_default_properties( + ioc_info: IocInfo, recceiverid: str, channels_iocs: Dict[str, List[str]], iocs: Dict[str, IocInfo], cf_channel +): + channel_name = cf_channel["name"] + last_ioc_info = iocs[channels_iocs[channel_name][-1]] return create_properties( - owner, - iocTime, + ioc_info.owner, + ioc_info.time, recceiverid, - iocs[channels_dict[cf_channel["name"]][-1]]["hostname"], - iocs[channels_dict[cf_channel["name"]][-1]]["iocname"], - iocs[channels_dict[cf_channel["name"]][-1]]["iocIP"], - channels_dict[cf_channel["name"]][-1], + last_ioc_info.hostname, + last_ioc_info.ioc_name, + last_ioc_info.ioc_IP, + last_ioc_info.ioc_id, ) @@ -845,29 +844,14 @@ def poll( processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, - hostName, - iocName, - iocIP, - iocid, - owner, - iocTime, + ioc_info: IocInfo, ): - _log.info("Polling {iocName} begins...".format(iocName=iocName)) + _log.info("Polling {iocName} begins...".format(iocName=ioc_info.ioc_name)) sleep = 1 success = False while not success: try: - update_method( - processor, - recordInfoByName, - records_to_delete, - hostName, - iocName, - iocIP, - iocid, - owner, - iocTime, - ) + update_method(processor, recordInfoByName, records_to_delete, ioc_info) success = True return success except RequestException as e: @@ -876,4 +860,5 @@ def poll( _log.info("ChannelFinder update retry in {retry_seconds} seconds".format(retry_seconds=retry_seconds)) time.sleep(retry_seconds) sleep *= 1.5 - _log.info("Polling {iocName} complete".format(iocName=iocName)) + _log.info("Polling {iocName} complete".format(iocName=ioc_info.ioc_name)) + From 4965b730d868c0cec6f3c6c8c80df61ea890089b Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 17:39:14 +0100 Subject: [PATCH 09/65] Add CFChannel dataclass --- server/recceiver/cfstore.py | 172 ++++++++++++++++++------------------ 1 file changed, 88 insertions(+), 84 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index d048d1e7..4f876b20 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -7,15 +7,16 @@ import time from collections import defaultdict from dataclasses import dataclass, field -from typing import Dict, List, Optional, Set +from typing import Any, Dict, List, Optional, Set from channelfinder import ChannelFinderClient from requests import ConnectionError, RequestException +from zope.interface import implementer + from twisted.application import service from twisted.internet import defer from twisted.internet.defer import DeferredLock from twisted.internet.threads import deferToThread -from zope.interface import implementer from . import interfaces from .interfaces import CommitTransaction @@ -123,6 +124,27 @@ def ioc_id(self): return self.host + ":" + str(self.port) +@dataclass +class CFChannel: + name: str + owner: str + properties: List[CFProperty] + + def as_dict(self) -> Dict[str, Any]: + return { + "name": self.name, + "owner": self.owner, + "properties": [p.as_dict() for p in self.properties], + } + + def from_channelfinder_dict(channel_dict: Dict[str, Any]) -> "CFChannel": + return CFChannel( + name=channel_dict.get("name", ""), + owner=channel_dict.get("owner", ""), + properties=[CFProperty.from_channelfinder_dict(p) for p in channel_dict.get("properties", [])], + ) + + @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): @@ -425,18 +447,24 @@ def clean_service(self): _log.info("Abandoning clean after {retry_limit} seconds".format(retry_limit=retry_limit)) return - def get_active_channels(self, recceiverid): - return self.client.findByArgs( - prepareFindArgs( - self.cf_config, - [(CFPropertyName.pvStatus.name, PVStatus.Active.name), (CFPropertyName.recceiverID.name, recceiverid)], + def get_active_channels(self, recceiverid) -> List[CFChannel]: + return [ + CFChannel.from_channelfinder_dict(ch) + for ch in self.client.findByArgs( + prepareFindArgs( + self.cf_config, + [ + (CFPropertyName.pvStatus.name, PVStatus.Active.name), + (CFPropertyName.recceiverID.name, recceiverid), + ], + ) ) - ) + ] - def clean_channels(self, owner, channels): + def clean_channels(self, owner: str, channels: List[CFChannel]): new_channels = [] for cf_channel in channels or []: - new_channels.append(cf_channel["name"]) + new_channels.append(cf_channel.name) _log.info("Total channels to update: {nChannels}".format(nChannels=len(new_channels))) _log.debug( 'Update "pvStatus" property to "Inactive" for {n_channels} channels'.format(n_channels=len(new_channels)) @@ -447,14 +475,6 @@ def clean_channels(self, owner, channels): ) -def create_channel(name: str, owner: str, properties: List[CFProperty]): - return { - "name": name, - "owner": owner, - "properties": properties, - } - - def create_recordType_property(owner: str, recordType: str) -> CFProperty: return CFProperty(CFPropertyName.recordType.name, owner, recordType) @@ -504,41 +524,34 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] if processor.cancelled: raise defer.CancelledError() - channels = [] + channels: List[CFChannel] = [] """A list of channels in channelfinder with the associated hostName and iocName""" _log.debug("Find existing channels by IOCID: {iocid}".format(iocid=iocid)) - old_channels = client.findByArgs(prepareFindArgs(cf_config, [("iocid", iocid)])) - old_channels = [ - { - "name": ch["name"], - "owner": ch["owner"], - "properties": [CFProperty.from_channelfinder_dict(prop) for prop in ch["properties"]], - } - for ch in old_channels + old_channels: List[CFChannel] = [ + CFChannel.from_channelfinder_dict(ch) + for ch in client.findByArgs(prepareFindArgs(cf_config, [("iocid", iocid)])) ] - if processor.cancelled: raise defer.CancelledError() if old_channels is not None: for cf_channel in old_channels: if ( - len(new_channels) == 0 or cf_channel["name"] in records_to_delete + len(new_channels) == 0 or cf_channel.name in records_to_delete ): # case: empty commit/del, remove all reference to ioc - _log.debug("Channel {s} exists in Channelfinder not in new_channels".format(s=cf_channel["name"])) - if cf_channel["name"] in channel_ioc_ids: - cf_channel["owner"] = iocs[channel_ioc_ids[cf_channel["name"]][-1]].owner - cf_channel["properties"] = __merge_property_lists( + _log.debug("Channel {s} exists in Channelfinder not in new_channels".format(s=cf_channel.name)) + if cf_channel.name in channel_ioc_ids: + last_ioc_id = channel_ioc_ids[cf_channel.name][-1] + cf_channel.owner = iocs[last_ioc_id].owner + cf_channel.properties = __merge_property_lists( create_default_properties(ioc_info, recceiverid, channel_ioc_ids, iocs, cf_channel), cf_channel, processor.managed_properties, ) if cf_config.record_type_enabled: - cf_channel["properties"] = __merge_property_lists( - cf_channel["properties"].append( - create_recordType_property( - ioc_info.owner, iocs[channel_ioc_ids[cf_channel["name"]][-1]]["recordType"] - ) + cf_channel.properties = __merge_property_lists( + cf_channel.properties.append( + create_recordType_property(ioc_info.owner, iocs[last_ioc_id]["recordType"]) ), cf_channel, processor.managed_properties, @@ -547,13 +560,14 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] _log.debug("Add existing channel to previous IOC: {s}".format(s=channels[-1])) """In case alias exist, also delete them""" if cf_config.alias_enabled: - if cf_channel["name"] in recordInfoByName: - for alias_name in recordInfoByName[cf_channel["name"]].aliases: + if cf_channel.name in recordInfoByName: + for alias_name in recordInfoByName[cf_channel.name].aliases: # TODO Remove? This code couldn't have been working.... - alias_channel = {"name": alias_name, "properties": [], "owner": ""} + alias_channel = CFChannel(alias_name, "", []) if alias_name in channel_ioc_ids: - alias_channel["owner"] = iocs[channel_ioc_ids[alias_name][-1]].owner - alias_channel["properties"] = __merge_property_lists( + last_alias_ioc_id = channel_ioc_ids[alias_name][-1] + alias_channel.owner = iocs[last_alias_ioc_id].owner + alias_channel.properties = __merge_property_lists( create_default_properties( ioc_info, recceiverid, @@ -565,11 +579,11 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] processor.managed_properties, ) if cf_config.record_type_enabled: - cf_channel["properties"] = __merge_property_lists( - cf_channel["properties"].append( + cf_channel.properties = __merge_property_lists( + cf_channel.properties.append( create_recordType_property( ioc_info.owner, - iocs[channel_ioc_ids[alias_name][-1]]["recordType"], + iocs[last_alias_ioc_id]["recordType"], ) ), cf_channel, @@ -580,7 +594,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] else: """Orphan the channel : mark as inactive, keep the old hostName and iocName""" - cf_channel["properties"] = __merge_property_lists( + cf_channel.properties = __merge_property_lists( [ create_inactive_property(ioc_info.owner), create_time_property(ioc_info.owner, ioc_info.time), @@ -591,10 +605,10 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] _log.debug("Add orphaned channel with no IOC: {s}".format(s=channels[-1])) """Also orphan any alias""" if cf_config.alias_enabled: - if cf_channel["name"] in recordInfoByName: - for alias_name in recordInfoByName[cf_channel["name"]].aliases: - alias_channel = {"name": alias_name, "properties": [], "owner": ""} - alias_channel["properties"] = __merge_property_lists( + if cf_channel.name in recordInfoByName: + for alias_name in recordInfoByName[cf_channel.name].aliases: + alias_channel = CFChannel(alias_name, "", []) + alias_channel.properties = __merge_property_lists( [ create_inactive_property(ioc_info.owner), create_time_property(ioc_info.owner, ioc_info.time), @@ -604,17 +618,15 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] channels.append(alias_channel) _log.debug("Add orphaned alias with no IOC: {s}".format(s=channels[-1])) else: - if cf_channel["name"] in new_channels: # case: channel in old and new + if cf_channel.name in new_channels: # case: channel in old and new """ Channel exists in Channelfinder with same hostname and iocname. Update the status to ensure it is marked active and update the time. """ _log.debug( - "Channel {s} exists in Channelfinder with same hostname and iocname".format( - s=cf_channel["name"] - ) + "Channel {s} exists in Channelfinder with same hostname and iocname".format(s=cf_channel.name) ) - cf_channel["properties"] = __merge_property_lists( + cf_channel.properties = __merge_property_lists( [ create_active_property(ioc_info.owner), create_time_property(ioc_info.owner, ioc_info.time), @@ -624,16 +636,16 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] ) channels.append(cf_channel) _log.debug("Add existing channel with same IOC: {s}".format(s=channels[-1])) - new_channels.remove(cf_channel["name"]) + new_channels.remove(cf_channel.name) """In case, alias exist""" if cf_config.alias_enabled: - if cf_channel["name"] in recordInfoByName: - for alias_name in recordInfoByName[cf_channel["name"]].aliases: + if cf_channel.name in recordInfoByName: + for alias_name in recordInfoByName[cf_channel.name].aliases: if alias_name in old_channels: """alias exists in old list""" - alias_channel = {"name": alias_name, "properties": [], "owner": ""} - alias_channel["properties"] = __merge_property_lists( + alias_channel = CFChannel(alias_name, "", []) + alias_channel.properties = __merge_property_lists( [ create_active_property(ioc_info.owner), create_time_property(ioc_info.owner, ioc_info.time), @@ -651,14 +663,14 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] create_time_property(ioc_info.owner, ioc_info.time), create_alias_property( ioc_info.owner, - cf_channel["name"], + cf_channel.name, ), ], cf_channel, processor.managed_properties, ) channels.append( - create_channel( + CFChannel( alias_name, ioc_info.owner, aprops, @@ -668,7 +680,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] _log.debug("Add existing alias with same IOC: {s}".format(s=channels[-1])) # now pvNames contains a list of pv's new on this host/ioc """A dictionary representing the current channelfinder information associated with the pvNames""" - existingChannels = {} + existingChannels: Dict[str, CFChannel] = {} """ The list of pv's is searched keeping in mind the limitations on the URL length @@ -689,12 +701,8 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] for eachSearchString in searchStrings: _log.debug("Find existing channels by name: {search}".format(search=eachSearchString)) - for cf_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", eachSearchString)])): - existingChannels[cf_channel["name"]] = { - "name": cf_channel["name"], - "owner": cf_channel["owner"], - "properties": [CFProperty.from_channelfinder_dict(prop) for prop in cf_channel["properties"]], - } + for found_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", eachSearchString)])): + existingChannels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) if processor.cancelled: raise defer.CancelledError() @@ -723,7 +731,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] ) existingChannel = existingChannels[channel_name] - existingChannel["properties"] = __merge_property_lists( + existingChannel.properties = __merge_property_lists( newProps, existingChannel, processor.managed_properties, @@ -739,19 +747,19 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] for alias_name in recordInfoByName[channel_name].aliases: if alias_name in existingChannels: ach = existingChannels[alias_name] - ach["properties"] = __merge_property_lists( + ach.properties = __merge_property_lists( alProps, ach, processor.managed_properties, ) channels.append(ach) else: - channels.append(create_channel(alias_name, ioc_info.owner, alProps)) + channels.append(CFChannel(alias_name, ioc_info.owner, alProps)) _log.debug("Add existing alias with different IOC: {s}".format(s=channels[-1])) else: """New channel""" - channels.append({"name": channel_name, "owner": ioc_info.owner, "properties": newProps}) + channels.append(CFChannel(channel_name, ioc_info.owner, newProps)) _log.debug("Add new channel: {s}".format(s=channels[-1])) if cf_config.alias_enabled: if channel_name in recordInfoByName: @@ -759,7 +767,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] for p in newProps: alProps.append(p) for alias in recordInfoByName[channel_name].aliases: - channels.append({"name": alias, "owner": ioc_info.owner, "properties": alProps}) + channels.append(CFChannel(alias, ioc_info.owner, alProps)) _log.debug("Add new alias: {s}".format(s=channels[-1])) _log.info( "Total channels to update: {nChannels} {iocName}".format(nChannels=len(channels), iocName=ioc_info.ioc_name) @@ -773,13 +781,9 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] raise defer.CancelledError() -def cf_set_chunked(client, channels, chunk_size=10000): +def cf_set_chunked(client, channels: List[CFChannel], chunk_size=10000): for i in range(0, len(channels), chunk_size): - chunk = [ - {"name": ch["name"], "owner": ch["owner"], "properties": [prop.as_dict() for prop in ch["properties"]]} - for ch in channels[i : i + chunk_size] - ] - _log.debug("Updating chunk %s", chunk) + chunk = [ch.as_dict() for ch in channels[i : i + chunk_size]] client.set(channels=chunk) @@ -798,7 +802,7 @@ def create_properties(owner: str, iocTime: str, recceiverid: str, hostName: str, def create_default_properties( ioc_info: IocInfo, recceiverid: str, channels_iocs: Dict[str, List[str]], iocs: Dict[str, IocInfo], cf_channel ): - channel_name = cf_channel["name"] + channel_name = cf_channel.name last_ioc_info = iocs[channels_iocs[channel_name][-1]] return create_properties( ioc_info.owner, @@ -812,7 +816,7 @@ def create_default_properties( def __merge_property_lists( - newProperties: List[CFProperty], channel: Dict[str, List[CFProperty]], managed_properties: Set[str] = set() + newProperties: List[CFProperty], channel: CFChannel, managed_properties: Set[str] = set() ) -> List[CFProperty]: """ Merges two lists of properties ensuring that there are no 2 properties with @@ -820,7 +824,7 @@ def __merge_property_lists( new property list wins out """ newPropNames = [p.name for p in newProperties] - for oldProperty in channel["properties"]: + for oldProperty in channel.properties: if oldProperty.name not in newPropNames and (oldProperty.name not in managed_properties): newProperties = newProperties + [oldProperty] return newProperties From c03b990f2aa9798dc2a12263e50e5af137682885 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 1 Dec 2025 14:57:36 +0100 Subject: [PATCH 10/65] Use static CFProperty methods In favour of create_property_blah --- server/recceiver/cfstore.py | 120 +++++++++++++++++++++++------------- 1 file changed, 77 insertions(+), 43 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 4f876b20..62fd4d58 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -79,6 +79,64 @@ def from_channelfinder_dict(prop_dict: Dict[str, str]) -> "CFProperty": value=prop_dict.get("value"), ) + @staticmethod + def recordType(owner: str, recordType: str) -> "CFProperty": + """Create a Channelfinder recordType property. + + Args: + owner: The owner of the property. + recordType: The recordType of the property. + """ + return CFProperty(CFPropertyName.recordType.name, owner, recordType) + + @staticmethod + def alias(owner: str, alias: str) -> "CFProperty": + """Create a Channelfinder alias property. + + Args: + owner: The owner of the property. + alias: The alias of the property. + """ + return CFProperty(CFPropertyName.alias.name, owner, alias) + + @staticmethod + def pvStatus(owner: str, pvStatus: str) -> "CFProperty": + """Create a Channelfinder pvStatus property. + + Args: + owner: The owner of the property. + pvStatus: The pvStatus of the property. + """ + return CFProperty(CFPropertyName.pvStatus.name, owner, pvStatus) + + @staticmethod + def active(owner: str) -> "CFProperty": + """Create a Channelfinder active property. + + Args: + owner: The owner of the property. + """ + return CFProperty.pvStatus(owner, PVStatus.Active.name) + + @staticmethod + def inactive(owner: str) -> "CFProperty": + """Create a Channelfinder inactive property. + + Args: + owner: The owner of the property. + """ + return CFProperty.pvStatus(owner, PVStatus.Inactive.name) + + @staticmethod + def time(owner: str, time: str) -> "CFProperty": + """Create a Channelfinder time property. + + Args: + owner: The owner of the property. + time: The time of the property. + """ + return CFProperty(CFPropertyName.time.name, owner, time) + @dataclass class RecordInfo: @@ -470,35 +528,11 @@ def clean_channels(self, owner: str, channels: List[CFChannel]): 'Update "pvStatus" property to "Inactive" for {n_channels} channels'.format(n_channels=len(new_channels)) ) self.client.update( - property=create_inactive_property(owner).as_dict(), + property=CFProperty.inactive(owner).as_dict(), channelNames=new_channels, ) -def create_recordType_property(owner: str, recordType: str) -> CFProperty: - return CFProperty(CFPropertyName.recordType.name, owner, recordType) - - -def create_alias_property(owner: str, alias: str) -> CFProperty: - return CFProperty(CFPropertyName.alias.name, owner, alias) - - -def create_pvStatus_property(owner: str, pvStatus: str) -> CFProperty: - return CFProperty(CFPropertyName.pvStatus.name, owner, pvStatus) - - -def create_active_property(owner: str) -> CFProperty: - return create_pvStatus_property(owner, PVStatus.Active.name) - - -def create_inactive_property(owner: str) -> CFProperty: - return create_pvStatus_property(owner, PVStatus.Inactive.name) - - -def create_time_property(owner: str, time: str) -> CFProperty: - return CFProperty(CFPropertyName.time.name, owner, time) - - def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): _log.info("CF Update IOC: {iocid}".format(iocid=ioc_info.ioc_id)) _log.debug( @@ -551,7 +585,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] if cf_config.record_type_enabled: cf_channel.properties = __merge_property_lists( cf_channel.properties.append( - create_recordType_property(ioc_info.owner, iocs[last_ioc_id]["recordType"]) + CFProperty.recordType(ioc_info.owner, iocs[last_ioc_id]["recordType"]) ), cf_channel, processor.managed_properties, @@ -581,7 +615,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] if cf_config.record_type_enabled: cf_channel.properties = __merge_property_lists( cf_channel.properties.append( - create_recordType_property( + CFProperty.recordType( ioc_info.owner, iocs[last_alias_ioc_id]["recordType"], ) @@ -596,8 +630,8 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] """Orphan the channel : mark as inactive, keep the old hostName and iocName""" cf_channel.properties = __merge_property_lists( [ - create_inactive_property(ioc_info.owner), - create_time_property(ioc_info.owner, ioc_info.time), + CFProperty.inactive(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), ], cf_channel, ) @@ -610,8 +644,8 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] alias_channel = CFChannel(alias_name, "", []) alias_channel.properties = __merge_property_lists( [ - create_inactive_property(ioc_info.owner), - create_time_property(ioc_info.owner, ioc_info.time), + CFProperty.inactive(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), ], alias_channel, ) @@ -628,8 +662,8 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] ) cf_channel.properties = __merge_property_lists( [ - create_active_property(ioc_info.owner), - create_time_property(ioc_info.owner, ioc_info.time), + CFProperty.active(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), ], cf_channel, processor.managed_properties, @@ -647,8 +681,8 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] alias_channel = CFChannel(alias_name, "", []) alias_channel.properties = __merge_property_lists( [ - create_active_property(ioc_info.owner), - create_time_property(ioc_info.owner, ioc_info.time), + CFProperty.active(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), ], alias_channel, processor.managed_properties, @@ -659,9 +693,9 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] """alias exists but not part of old list""" aprops = __merge_property_lists( [ - create_active_property(ioc_info.owner), - create_time_property(ioc_info.owner, ioc_info.time), - create_alias_property( + CFProperty.active(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), + CFProperty.alias( ioc_info.owner, cf_channel.name, ), @@ -721,7 +755,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] and channel_name in recordInfoByName and recordInfoByName[channel_name].recordType ): - newProps.append(create_recordType_property(ioc_info.owner, recordInfoByName[channel_name].recordType)) + newProps.append(CFProperty.recordType(ioc_info.owner, recordInfoByName[channel_name].recordType)) if channel_name in recordInfoByName: newProps = newProps + recordInfoByName[channel_name].infoProperties @@ -741,7 +775,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] """in case, alias exists, update their properties too""" if cf_config.alias_enabled: if channel_name in recordInfoByName: - alProps = [create_alias_property(ioc_info.owner, channel_name)] + alProps = [CFProperty.alias(ioc_info.owner, channel_name)] for p in newProps: alProps.append(p) for alias_name in recordInfoByName[channel_name].aliases: @@ -763,7 +797,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] _log.debug("Add new channel: {s}".format(s=channels[-1])) if cf_config.alias_enabled: if channel_name in recordInfoByName: - alProps = [create_alias_property(ioc_info.owner, channel_name)] + alProps = [CFProperty.alias(ioc_info.owner, channel_name)] for p in newProps: alProps.append(p) for alias in recordInfoByName[channel_name].aliases: @@ -793,8 +827,8 @@ def create_properties(owner: str, iocTime: str, recceiverid: str, hostName: str, CFProperty(CFPropertyName.iocName.name, owner, iocName), CFProperty(CFPropertyName.iocid.name, owner, iocid), CFProperty(CFPropertyName.iocIP.name, owner, iocIP), - create_active_property(owner), - create_time_property(owner, iocTime), + CFProperty.active(owner), + CFProperty.time(owner, iocTime), CFProperty(CFPropertyName.recceiverID.name, owner, recceiverid), ] From 18d533c56eb817620d9cc09b03466c6a613f4b1a Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 17:39:51 +0100 Subject: [PATCH 11/65] sleep is a float --- server/recceiver/cfstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 62fd4d58..66144b85 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -885,7 +885,7 @@ def poll( ioc_info: IocInfo, ): _log.info("Polling {iocName} begins...".format(iocName=ioc_info.ioc_name)) - sleep = 1 + sleep = 1.0 success = False while not success: try: From fd6210a57bb294784adbf09937fa151eb2cfcf76 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 3 Nov 2025 17:41:56 +0100 Subject: [PATCH 12/65] CancelledError has no host or port properties --- server/recceiver/cfstore.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 66144b85..52b11a35 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -350,11 +350,9 @@ def chainResult(_ignored): def _commitWithThread(self, transaction: CommitTransaction): if not self.running: - raise defer.CancelledError( - "CF Processor is not running (transaction: {host}:{port})", - host=transaction.source_address.host, - port=transaction.source_address.port, - ) + host = transaction.source_address.host + port = transaction.source_address.port + raise defer.CancelledError(f"CF Processor is not running (transaction: {host}:{port})") _log.info("CF_COMMIT: {transaction}".format(transaction=transaction)) _log.debug("CF_COMMIT: transaction: {s}".format(s=repr(transaction))) From 6c97c05fc728b898931470ada7ab84604dcaf5e3 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 4 Nov 2025 09:56:50 +0100 Subject: [PATCH 13/65] Split up commitWithThread --- server/recceiver/cfstore.py | 117 ++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 52b11a35..9fd67992 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -348,52 +348,17 @@ def chainResult(_ignored): t.addCallbacks(chainResult, chainError) return d - def _commitWithThread(self, transaction: CommitTransaction): - if not self.running: - host = transaction.source_address.host - port = transaction.source_address.port - raise defer.CancelledError(f"CF Processor is not running (transaction: {host}:{port})") - - _log.info("CF_COMMIT: {transaction}".format(transaction=transaction)) - _log.debug("CF_COMMIT: transaction: {s}".format(s=repr(transaction))) - """ - a dictionary with a list of records with their associated property info - pvInfo - {record_id: { "pvName":"recordName", - "infoProperties":{propName:value, ...}}} - """ - ioc_info = IocInfo( - host=transaction.source_address.host, - hostname=transaction.client_infos.get("HOSTNAME") or transaction.source_address.host, - ioc_name=transaction.client_infos.get("IOCNAME") or str(transaction.source_address.port), - ioc_IP=transaction.source_address.host, - owner=( - transaction.client_infos.get("ENGINEER") - or transaction.client_infos.get("CF_USERNAME") - or self.cf_config.username - ), - time=self.currentTime(timezone=self.cf_config.timezone), - port=transaction.source_address.port, - channelcount=0, - ) - - """The unique identifier for a particular IOC""" - iocid = ioc_info.ioc_id - _log.debug("transaction: {s}".format(s=repr(transaction))) - + def transaction_to_recordInfo(self, ioc_info: IocInfo, transaction: CommitTransaction) -> Dict[str, RecordInfo]: recordInfo: Dict[str, RecordInfo] = {} for record_id, (record_name, record_type) in transaction.records_to_add.items(): recordInfo[record_id] = RecordInfo(pvName=record_name, recordType=None, infoProperties=[], aliases=[]) if self.cf_config.record_type_enabled: recordInfo[record_id].recordType = record_type + for record_id, (record_infos_to_add) in transaction.record_infos_to_add.items(): # find intersection of these sets if record_id not in recordInfo: - _log.warning( - "IOC: {iocid}: PV not found for recinfo with RID: {record_id}".format( - iocid=iocid, record_id=record_id - ) - ) + _log.warning("IOC: %s: PV not found for recinfo with RID: {record_id}", ioc_info, record_id) continue recinfo_wl = [p for p in self.record_property_names_list if p in record_infos_to_add.keys()] if recinfo_wl: @@ -402,15 +367,11 @@ def _commitWithThread(self, transaction: CommitTransaction): CFProperty(infotag, ioc_info.owner, record_infos_to_add[infotag]) ) - for record_id, alias in transaction.aliases.items(): + for record_id, record_aliases in transaction.aliases.items(): if record_id not in recordInfo: - _log.warning( - "IOC: {iocid}: PV not found for alias with RID: {record_id}".format( - iocid=iocid, record_id=record_id - ) - ) + _log.warning("IOC: %s: PV not found for alias with RID: %s", ioc_info, record_id) continue - recordInfo[record_id].aliases = alias + recordInfo[record_id].aliases = record_aliases for record_id in recordInfo: for epics_env_var_name, cf_prop_name in self.env_vars.items(): @@ -422,21 +383,27 @@ def _commitWithThread(self, transaction: CommitTransaction): _log.debug( "EPICS environment var %s listed in environment_vars setting list not found in this IOC: %s", epics_env_var_name, - ioc_info.ioc_name, + ioc_info, ) + return recordInfo - records_to_delete = list(transaction.records_to_delete) - _log.debug("Delete records: {s}".format(s=records_to_delete)) - + def record_info_by_name(self, recordInfo, ioc_info) -> Dict[str, RecordInfo]: recordInfoByName = {} for record_id, (info) in recordInfo.items(): if info.pvName in recordInfoByName: - _log.warning( - "Commit contains multiple records with PV name: {pv} ({iocid})".format(pv=info.pvName, iocid=iocid) - ) + _log.warning("Commit contains multiple records with PV name: %s (%s)", info.pvName, ioc_info) continue recordInfoByName[info.pvName] = info - + return recordInfoByName + + def update_ioc_infos( + self, + transaction: CommitTransaction, + ioc_info: IocInfo, + records_to_delete: List[str], + recordInfoByName: Dict[str, RecordInfo], + ): + iocid = ioc_info.ioc_id if transaction.initial: """Add IOC to source list """ self.iocs[iocid] = ioc_info @@ -448,8 +415,8 @@ def _commitWithThread(self, transaction: CommitTransaction): """In case, alias exists""" if self.cf_config.alias_enabled: if record_name in recordInfoByName: - for alias in recordInfoByName[record_name].aliases: - self.channel_ioc_ids[alias].append(iocid) # add iocname to pvName in dict + for record_aliases in recordInfoByName[record_name].aliases: + self.channel_ioc_ids[record_aliases].append(iocid) # add iocname to pvName in dict self.iocs[iocid].channelcount += 1 for record_name in records_to_delete: if iocid in self.channel_ioc_ids[record_name]: @@ -457,18 +424,50 @@ def _commitWithThread(self, transaction: CommitTransaction): """In case, alias exists""" if self.cf_config.alias_enabled: if record_name in recordInfoByName: - for alias in recordInfoByName[record_name].aliases: - self.remove_channel(alias, iocid) + for record_aliases in recordInfoByName[record_name].aliases: + self.remove_channel(record_aliases, iocid) + + def _commitWithThread(self, transaction: CommitTransaction): + if not self.running: + host = transaction.source_address.host + port = transaction.source_address.port + raise defer.CancelledError(f"CF Processor is not running (transaction: {host}:{port})") + + _log.info("CF_COMMIT: %s", transaction) + _log.debug("CF_COMMIT: transaction: %s", repr(transaction)) + + ioc_info = IocInfo( + host=transaction.source_address.host, + hostname=transaction.client_infos.get("HOSTNAME") or transaction.source_address.host, + ioc_name=transaction.client_infos.get("IOCNAME") or str(transaction.source_address.port), + ioc_IP=transaction.source_address.host, + owner=( + transaction.client_infos.get("ENGINEER") + or transaction.client_infos.get("CF_USERNAME") + or self.cf_config.username + ), + time=self.currentTime(timezone=self.cf_config.timezone), + port=transaction.source_address.port, + channelcount=0, + ) + + recordInfo = self.transaction_to_recordInfo(ioc_info, transaction) + + records_to_delete = list(transaction.records_to_delete) + _log.debug("Delete records: {s}".format(s=records_to_delete)) + + recordInfoByName = self.record_info_by_name(recordInfo, ioc_info) + self.update_ioc_infos(transaction, ioc_info, records_to_delete, recordInfoByName) poll(__updateCF__, self, recordInfoByName, records_to_delete, ioc_info) - def remove_channel(self, recordName, iocid): + def remove_channel(self, recordName: str, iocid: str): self.channel_ioc_ids[recordName].remove(iocid) if iocid in self.iocs: self.iocs[iocid].channelcount -= 1 if self.iocs[iocid].channelcount == 0: self.iocs.pop(iocid, None) elif self.iocs[iocid].channelcount < 0: - _log.error("Channel count negative: {s}", s=iocid) + _log.error("Channel count negative: %s", iocid) if len(self.channel_ioc_ids[recordName]) <= 0: # case: channel has no more iocs del self.channel_ioc_ids[recordName] From 3cdf7dfeb11217361143222034dd212342100c60 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 4 Nov 2025 10:12:03 +0100 Subject: [PATCH 14/65] Improve logging log the whole object when available use inbuilt log formatting rather than f strings or .format (should improve performance) --- server/recceiver/cfstore.py | 86 +++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 9fd67992..4c3fa247 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -206,14 +206,14 @@ def from_channelfinder_dict(channel_dict: Dict[str, Any]) -> "CFChannel": @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): - _log.info("CF_INIT {name}".format(name=name)) + self.cf_config = CFConfig.from_config_adapter(conf) + _log.info("CF_INIT %s", self.cf_config) self.name = name self.channel_ioc_ids = defaultdict(list) self.iocs: Dict[str, IocInfo] = dict() self.client = None self.currentTime = getCurrentTime self.lock = DeferredLock() - self.cf_config = CFConfig.from_config_adapter(conf) def startService(self): service.Service.startService(self) @@ -289,7 +289,7 @@ def _startServiceWithLock(self): self.record_property_names_list = set(record_property_names_list) self.managed_properties = required_properties.union(record_property_names_list) - _log.debug("record_property_names_list = {}".format(self.record_property_names_list)) + _log.debug("record_property_names_list = %s", self.record_property_names_list) except ConnectionError: _log.exception("Cannot connect to Channelfinder service") raise @@ -339,9 +339,9 @@ def chainError(err): else: d.callback(None) - def chainResult(_ignored): + def chainResult(result): if self.cancelled: - raise defer.CancelledError() + raise defer.CancelledError(f"CF Processor is cancelled, due to {result}") else: d.callback(None) @@ -454,7 +454,7 @@ def _commitWithThread(self, transaction: CommitTransaction): recordInfo = self.transaction_to_recordInfo(ioc_info, transaction) records_to_delete = list(transaction.records_to_delete) - _log.debug("Delete records: {s}".format(s=records_to_delete)) + _log.debug("Delete records: %s", records_to_delete) recordInfoByName = self.record_info_by_name(recordInfo, ioc_info) self.update_ioc_infos(transaction, ioc_info, records_to_delete, recordInfoByName) @@ -495,11 +495,11 @@ def clean_service(self): except RequestException as e: _log.error("Clean service failed: {s}".format(s=e)) retry_seconds = min(60, sleep) - _log.info("Clean service retry in {retry_seconds} seconds".format(retry_seconds=retry_seconds)) + _log.info("Clean service retry in %s seconds", retry_seconds) time.sleep(retry_seconds) sleep *= 1.5 if self.running == 0 and sleep >= retry_limit: - _log.info("Abandoning clean after {retry_limit} seconds".format(retry_limit=retry_limit)) + _log.info("Abandoning clean after %s seconds", retry_limit) return def get_active_channels(self, recceiverid) -> List[CFChannel]: @@ -520,9 +520,9 @@ def clean_channels(self, owner: str, channels: List[CFChannel]): new_channels = [] for cf_channel in channels or []: new_channels.append(cf_channel.name) - _log.info("Total channels to update: {nChannels}".format(nChannels=len(new_channels))) + _log.info("Cleaning %s channels.", len(new_channels)) _log.debug( - 'Update "pvStatus" property to "Inactive" for {n_channels} channels'.format(n_channels=len(new_channels)) + 'Update "pvStatus" property to "Inactive" for %s channels', len(new_channels) ) self.client.update( property=CFProperty.inactive(owner).as_dict(), @@ -531,12 +531,8 @@ def clean_channels(self, owner: str, channels: List[CFChannel]): def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): - _log.info("CF Update IOC: {iocid}".format(iocid=ioc_info.ioc_id)) - _log.debug( - "CF Update IOC: {iocid} recordInfoByName {recordInfoByName}".format( - iocid=ioc_info.ioc_id, recordInfoByName=recordInfoByName - ) - ) + _log.info("CF Update IOC: %s", ioc_info) + _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) # Consider making this function a class methed then 'processor' simply becomes 'self' client = processor.client channel_ioc_ids = processor.channel_ioc_ids @@ -547,30 +543,28 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] iocid = ioc_info.ioc_id if iocid not in iocs: - _log.warning("IOC Env Info not found: {iocid}".format(iocid=iocid)) + _log.warning("IOC Env Info %s not found in ioc list: %s", ioc_info, iocs) if ioc_info.hostname is None or ioc_info.ioc_name is None: - raise Exception("missing hostName or iocName") + raise Exception(f"Missing hostName {ioc_info.hostname} or iocName {ioc_info.ioc_name}") if processor.cancelled: - raise defer.CancelledError() + raise defer.CancelledError("Processor cancelled in __updateCF__") channels: List[CFChannel] = [] """A list of channels in channelfinder with the associated hostName and iocName""" - _log.debug("Find existing channels by IOCID: {iocid}".format(iocid=iocid)) + _log.debug("Find existing channels by IOCID: %s", ioc_info) old_channels: List[CFChannel] = [ CFChannel.from_channelfinder_dict(ch) for ch in client.findByArgs(prepareFindArgs(cf_config, [("iocid", iocid)])) ] - if processor.cancelled: - raise defer.CancelledError() if old_channels is not None: for cf_channel in old_channels: if ( len(new_channels) == 0 or cf_channel.name in records_to_delete ): # case: empty commit/del, remove all reference to ioc - _log.debug("Channel {s} exists in Channelfinder not in new_channels".format(s=cf_channel.name)) + _log.debug("Channel %s exists in Channelfinder not in new_channels", cf_channel) if cf_channel.name in channel_ioc_ids: last_ioc_id = channel_ioc_ids[cf_channel.name][-1] cf_channel.owner = iocs[last_ioc_id].owner @@ -588,7 +582,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] processor.managed_properties, ) channels.append(cf_channel) - _log.debug("Add existing channel to previous IOC: {s}".format(s=channels[-1])) + _log.debug("Add existing channel %s to previous IOC %s", cf_channel, last_ioc_id) """In case alias exist, also delete them""" if cf_config.alias_enabled: if cf_channel.name in recordInfoByName: @@ -621,7 +615,9 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] processor.managed_properties, ) channels.append(alias_channel) - _log.debug("Add existing alias to previous IOC: {s}".format(s=channels[-1])) + _log.debug( + "Add existing alias %s to previous IOC: %s", alias_channel, last_alias_ioc_id + ) else: """Orphan the channel : mark as inactive, keep the old hostName and iocName""" @@ -633,7 +629,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] cf_channel, ) channels.append(cf_channel) - _log.debug("Add orphaned channel with no IOC: {s}".format(s=channels[-1])) + _log.debug("Add orphaned channel %s with no IOC: %s", cf_channel, ioc_info) """Also orphan any alias""" if cf_config.alias_enabled: if cf_channel.name in recordInfoByName: @@ -647,16 +643,14 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] alias_channel, ) channels.append(alias_channel) - _log.debug("Add orphaned alias with no IOC: {s}".format(s=channels[-1])) + _log.debug("Add orphaned alias %s with no IOC: %s", alias_channel, ioc_info) else: if cf_channel.name in new_channels: # case: channel in old and new """ - Channel exists in Channelfinder with same hostname and iocname. + Channel exists in Channelfinder with same iocid. Update the status to ensure it is marked active and update the time. """ - _log.debug( - "Channel {s} exists in Channelfinder with same hostname and iocname".format(s=cf_channel.name) - ) + _log.debug("Channel %s exists in Channelfinder with same iocid %s", cf_channel.name, iocid) cf_channel.properties = __merge_property_lists( [ CFProperty.active(ioc_info.owner), @@ -666,7 +660,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] processor.managed_properties, ) channels.append(cf_channel) - _log.debug("Add existing channel with same IOC: {s}".format(s=channels[-1])) + _log.debug("Add existing channel with same IOC: %s", cf_channel) new_channels.remove(cf_channel.name) """In case, alias exist""" @@ -708,7 +702,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] ) ) new_channels.remove(alias_name) - _log.debug("Add existing alias with same IOC: {s}".format(s=channels[-1])) + _log.debug("Add existing alias with same IOC: %s", cf_channel) # now pvNames contains a list of pv's new on this host/ioc """A dictionary representing the current channelfinder information associated with the pvNames""" existingChannels: Dict[str, CFChannel] = {} @@ -731,7 +725,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] searchStrings.append(searchString) for eachSearchString in searchStrings: - _log.debug("Find existing channels by name: {search}".format(search=eachSearchString)) + _log.debug("Find existing channels by name: %s", eachSearchString) for found_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", eachSearchString)])): existingChannels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) if processor.cancelled: @@ -757,9 +751,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] newProps = newProps + recordInfoByName[channel_name].infoProperties if channel_name in existingChannels: - _log.debug( - f"""update existing channel{channel_name}: exists but with a different hostName and/or iocName""" - ) + _log.debug("update existing channel %s: exists but with a different iocid from %s", channel_name, iocid) existingChannel = existingChannels[channel_name] existingChannel.properties = __merge_property_lists( @@ -768,7 +760,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] processor.managed_properties, ) channels.append(existingChannel) - _log.debug("Add existing channel with different IOC: {s}".format(s=channels[-1])) + _log.debug("Add existing channel with different IOC: %s", existingChannel) """in case, alias exists, update their properties too""" if cf_config.alias_enabled: if channel_name in recordInfoByName: @@ -786,12 +778,14 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] channels.append(ach) else: channels.append(CFChannel(alias_name, ioc_info.owner, alProps)) - _log.debug("Add existing alias with different IOC: {s}".format(s=channels[-1])) + _log.debug( + "Add existing alias %s of %s with different IOC from %s", alias_name, channel_name, iocid + ) else: """New channel""" channels.append(CFChannel(channel_name, ioc_info.owner, newProps)) - _log.debug("Add new channel: {s}".format(s=channels[-1])) + _log.debug("Add new channel: %s", channel_name) if cf_config.alias_enabled: if channel_name in recordInfoByName: alProps = [CFProperty.alias(ioc_info.owner, channel_name)] @@ -799,10 +793,9 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] alProps.append(p) for alias in recordInfoByName[channel_name].aliases: channels.append(CFChannel(alias, ioc_info.owner, alProps)) - _log.debug("Add new alias: {s}".format(s=channels[-1])) - _log.info( - "Total channels to update: {nChannels} {iocName}".format(nChannels=len(channels), iocName=ioc_info.ioc_name) - ) + _log.debug("Add new alias: %s from %s", alias, channel_name) + _log.info("Total channels to update: %s for ioc: %s", len(channels), ioc_info) + if len(channels) != 0: cf_set_chunked(client, channels, cf_config.cf_query_limit) else: @@ -881,7 +874,7 @@ def poll( records_to_delete, ioc_info: IocInfo, ): - _log.info("Polling {iocName} begins...".format(iocName=ioc_info.ioc_name)) + _log.info("Polling for %s begins...", ioc_info) sleep = 1.0 success = False while not success: @@ -895,5 +888,4 @@ def poll( _log.info("ChannelFinder update retry in {retry_seconds} seconds".format(retry_seconds=retry_seconds)) time.sleep(retry_seconds) sleep *= 1.5 - _log.info("Polling {iocName} complete".format(iocName=ioc_info.ioc_name)) - + _log.info("Polling %s complete", ioc_info) From 17f1cb6f6cbe0c5230e3774606fd386abfc7ef28 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 13:43:51 +0100 Subject: [PATCH 15/65] Pull out channel_is_old --- server/recceiver/cfstore.py | 125 ++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 54 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 4c3fa247..b2a13bf8 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -521,15 +521,74 @@ def clean_channels(self, owner: str, channels: List[CFChannel]): for cf_channel in channels or []: new_channels.append(cf_channel.name) _log.info("Cleaning %s channels.", len(new_channels)) - _log.debug( - 'Update "pvStatus" property to "Inactive" for %s channels', len(new_channels) - ) + _log.debug('Update "pvStatus" property to "Inactive" for %s channels', len(new_channels)) self.client.update( property=CFProperty.inactive(owner).as_dict(), channelNames=new_channels, ) +def handle_channel_is_old( + channel_ioc_ids: Dict[str, List[str]], + cf_channel: CFChannel, + iocs: Dict[str, IocInfo], + ioc_info: IocInfo, + recceiverid: str, + processor: CFProcessor, + cf_config: CFConfig, + channels: List[CFChannel], + recordInfoByName: Dict[str, RecordInfo], +): + last_ioc_id = channel_ioc_ids[cf_channel.name][-1] + cf_channel.owner = iocs[last_ioc_id].owner + cf_channel.properties = __merge_property_lists( + create_default_properties(ioc_info, recceiverid, channel_ioc_ids, iocs, cf_channel), + cf_channel, + processor.managed_properties, + ) + if cf_config.record_type_enabled: + cf_channel.properties = __merge_property_lists( + cf_channel.properties.append(CFProperty.recordType(ioc_info.owner, iocs[last_ioc_id]["recordType"])), + cf_channel, + processor.managed_properties, + ) + channels.append(cf_channel) + _log.debug("Add existing channel %s to previous IOC %s", cf_channel, last_ioc_id) + """In case alias exist, also delete them""" + if cf_config.alias_enabled: + if cf_channel.name in recordInfoByName: + for alias_name in recordInfoByName[cf_channel.name].aliases: + # TODO Remove? This code couldn't have been working.... + alias_channel = CFChannel(alias_name, "", []) + if alias_name in channel_ioc_ids: + last_alias_ioc_id = channel_ioc_ids[alias_name][-1] + alias_channel.owner = iocs[last_alias_ioc_id].owner + alias_channel.properties = __merge_property_lists( + create_default_properties( + ioc_info, + recceiverid, + channel_ioc_ids, + iocs, + cf_channel, + ), + alias_channel, + processor.managed_properties, + ) + if cf_config.record_type_enabled: + cf_channel.properties = __merge_property_lists( + cf_channel.properties.append( + CFProperty.recordType( + ioc_info.owner, + iocs[last_alias_ioc_id]["recordType"], + ) + ), + cf_channel, + processor.managed_properties, + ) + channels.append(alias_channel) + _log.debug("Add existing alias %s to previous IOC: %s", alias_channel, last_alias_ioc_id) + + def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): _log.info("CF Update IOC: %s", ioc_info) _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) @@ -566,59 +625,17 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] ): # case: empty commit/del, remove all reference to ioc _log.debug("Channel %s exists in Channelfinder not in new_channels", cf_channel) if cf_channel.name in channel_ioc_ids: - last_ioc_id = channel_ioc_ids[cf_channel.name][-1] - cf_channel.owner = iocs[last_ioc_id].owner - cf_channel.properties = __merge_property_lists( - create_default_properties(ioc_info, recceiverid, channel_ioc_ids, iocs, cf_channel), + handle_channel_is_old( + channel_ioc_ids, cf_channel, - processor.managed_properties, + iocs, + ioc_info, + recceiverid, + processor, + cf_config, + channels, + recordInfoByName, ) - if cf_config.record_type_enabled: - cf_channel.properties = __merge_property_lists( - cf_channel.properties.append( - CFProperty.recordType(ioc_info.owner, iocs[last_ioc_id]["recordType"]) - ), - cf_channel, - processor.managed_properties, - ) - channels.append(cf_channel) - _log.debug("Add existing channel %s to previous IOC %s", cf_channel, last_ioc_id) - """In case alias exist, also delete them""" - if cf_config.alias_enabled: - if cf_channel.name in recordInfoByName: - for alias_name in recordInfoByName[cf_channel.name].aliases: - # TODO Remove? This code couldn't have been working.... - alias_channel = CFChannel(alias_name, "", []) - if alias_name in channel_ioc_ids: - last_alias_ioc_id = channel_ioc_ids[alias_name][-1] - alias_channel.owner = iocs[last_alias_ioc_id].owner - alias_channel.properties = __merge_property_lists( - create_default_properties( - ioc_info, - recceiverid, - channel_ioc_ids, - iocs, - cf_channel, - ), - alias_channel, - processor.managed_properties, - ) - if cf_config.record_type_enabled: - cf_channel.properties = __merge_property_lists( - cf_channel.properties.append( - CFProperty.recordType( - ioc_info.owner, - iocs[last_alias_ioc_id]["recordType"], - ) - ), - cf_channel, - processor.managed_properties, - ) - channels.append(alias_channel) - _log.debug( - "Add existing alias %s to previous IOC: %s", alias_channel, last_alias_ioc_id - ) - else: """Orphan the channel : mark as inactive, keep the old hostName and iocName""" cf_channel.properties = __merge_property_lists( From c292d1bba9ab5bdd36bc63cd00f79ad17686cdb7 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 13:47:01 +0100 Subject: [PATCH 16/65] Pull out orphan channel --- server/recceiver/cfstore.py | 56 ++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index b2a13bf8..3ee06aa4 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -589,6 +589,38 @@ def handle_channel_is_old( _log.debug("Add existing alias %s to previous IOC: %s", alias_channel, last_alias_ioc_id) +def orphan_channel( + cf_channel: CFChannel, + ioc_info: IocInfo, + channels: List[CFChannel], + cf_config: CFConfig, + recordInfoByName: Dict[str, RecordInfo], +): + cf_channel.properties = __merge_property_lists( + [ + CFProperty.inactive(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), + ], + cf_channel, + ) + channels.append(cf_channel) + _log.debug("Add orphaned channel %s with no IOC: %s", cf_channel, ioc_info) + """Also orphan any alias""" + if cf_config.alias_enabled: + if cf_channel.name in recordInfoByName: + for alias_name in recordInfoByName[cf_channel.name].aliases: + alias_channel = CFChannel(alias_name, "", []) + alias_channel.properties = __merge_property_lists( + [ + CFProperty.inactive(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), + ], + alias_channel, + ) + channels.append(alias_channel) + _log.debug("Add orphaned alias %s with no IOC: %s", alias_channel, ioc_info) + + def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): _log.info("CF Update IOC: %s", ioc_info) _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) @@ -638,29 +670,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] ) else: """Orphan the channel : mark as inactive, keep the old hostName and iocName""" - cf_channel.properties = __merge_property_lists( - [ - CFProperty.inactive(ioc_info.owner), - CFProperty.time(ioc_info.owner, ioc_info.time), - ], - cf_channel, - ) - channels.append(cf_channel) - _log.debug("Add orphaned channel %s with no IOC: %s", cf_channel, ioc_info) - """Also orphan any alias""" - if cf_config.alias_enabled: - if cf_channel.name in recordInfoByName: - for alias_name in recordInfoByName[cf_channel.name].aliases: - alias_channel = CFChannel(alias_name, "", []) - alias_channel.properties = __merge_property_lists( - [ - CFProperty.inactive(ioc_info.owner), - CFProperty.time(ioc_info.owner, ioc_info.time), - ], - alias_channel, - ) - channels.append(alias_channel) - _log.debug("Add orphaned alias %s with no IOC: %s", alias_channel, ioc_info) + orphan_channel(cf_channel, ioc_info, channels, cf_config, recordInfoByName) else: if cf_channel.name in new_channels: # case: channel in old and new """ From 46aec00589a8e12e44666abae2bb945ba7cc5e60 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 13:48:53 +0100 Subject: [PATCH 17/65] pull out handle channel old and new --- server/recceiver/cfstore.py | 134 +++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 55 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 3ee06aa4..8fd1b258 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -621,6 +621,76 @@ def orphan_channel( _log.debug("Add orphaned alias %s with no IOC: %s", alias_channel, ioc_info) +def handle_channel_old_and_new( + cf_channel: CFChannel, + iocid: str, + ioc_info: IocInfo, + processor: CFProcessor, + channels: List[CFChannel], + new_channels: Set[str], + cf_config: CFConfig, + recordInfoByName: Dict[str, RecordInfo], + old_channels: List[CFChannel], +): + """ + Channel exists in Channelfinder with same iocid. + Update the status to ensure it is marked active and update the time. + """ + _log.debug("Channel %s exists in Channelfinder with same iocid %s", cf_channel.name, iocid) + cf_channel.properties = __merge_property_lists( + [ + CFProperty.active(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), + ], + cf_channel, + processor.managed_properties, + ) + channels.append(cf_channel) + _log.debug("Add existing channel with same IOC: %s", cf_channel) + new_channels.remove(cf_channel.name) + + """In case, alias exist""" + if cf_config.alias_enabled: + if cf_channel.name in recordInfoByName: + for alias_name in recordInfoByName[cf_channel.name].aliases: + if alias_name in old_channels: + """alias exists in old list""" + alias_channel = CFChannel(alias_name, "", []) + alias_channel.properties = __merge_property_lists( + [ + CFProperty.active(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), + ], + alias_channel, + processor.managed_properties, + ) + channels.append(alias_channel) + new_channels.remove(alias_name) + else: + """alias exists but not part of old list""" + aprops = __merge_property_lists( + [ + CFProperty.active(ioc_info.owner), + CFProperty.time(ioc_info.owner, ioc_info.time), + CFProperty.alias( + ioc_info.owner, + cf_channel.name, + ), + ], + cf_channel, + processor.managed_properties, + ) + channels.append( + CFChannel( + alias_name, + ioc_info.owner, + aprops, + ) + ) + new_channels.remove(alias_name) + _log.debug("Add existing alias with same IOC: %s", cf_channel) + + def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): _log.info("CF Update IOC: %s", ioc_info) _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) @@ -673,63 +743,17 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] orphan_channel(cf_channel, ioc_info, channels, cf_config, recordInfoByName) else: if cf_channel.name in new_channels: # case: channel in old and new - """ - Channel exists in Channelfinder with same iocid. - Update the status to ensure it is marked active and update the time. - """ - _log.debug("Channel %s exists in Channelfinder with same iocid %s", cf_channel.name, iocid) - cf_channel.properties = __merge_property_lists( - [ - CFProperty.active(ioc_info.owner), - CFProperty.time(ioc_info.owner, ioc_info.time), - ], + handle_channel_old_and_new( cf_channel, - processor.managed_properties, + iocid, + ioc_info, + processor, + channels, + new_channels, + cf_config, + recordInfoByName, + old_channels, ) - channels.append(cf_channel) - _log.debug("Add existing channel with same IOC: %s", cf_channel) - new_channels.remove(cf_channel.name) - - """In case, alias exist""" - if cf_config.alias_enabled: - if cf_channel.name in recordInfoByName: - for alias_name in recordInfoByName[cf_channel.name].aliases: - if alias_name in old_channels: - """alias exists in old list""" - alias_channel = CFChannel(alias_name, "", []) - alias_channel.properties = __merge_property_lists( - [ - CFProperty.active(ioc_info.owner), - CFProperty.time(ioc_info.owner, ioc_info.time), - ], - alias_channel, - processor.managed_properties, - ) - channels.append(alias_channel) - new_channels.remove(alias_name) - else: - """alias exists but not part of old list""" - aprops = __merge_property_lists( - [ - CFProperty.active(ioc_info.owner), - CFProperty.time(ioc_info.owner, ioc_info.time), - CFProperty.alias( - ioc_info.owner, - cf_channel.name, - ), - ], - cf_channel, - processor.managed_properties, - ) - channels.append( - CFChannel( - alias_name, - ioc_info.owner, - aprops, - ) - ) - new_channels.remove(alias_name) - _log.debug("Add existing alias with same IOC: %s", cf_channel) # now pvNames contains a list of pv's new on this host/ioc """A dictionary representing the current channelfinder information associated with the pvNames""" existingChannels: Dict[str, CFChannel] = {} From 93d861e896a4fb3d6360af10e007ecd7a9eb5327 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 13:50:20 +0100 Subject: [PATCH 18/65] pull out get_existing channels --- server/recceiver/cfstore.py | 59 +++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 8fd1b258..09abfa15 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -691,6 +691,38 @@ def handle_channel_old_and_new( _log.debug("Add existing alias with same IOC: %s", cf_channel) +def get_existing_channels( + new_channels: Set[str], client: ChannelFinderClient, cf_config: CFConfig, processor: CFProcessor +) -> Dict[str, CFChannel]: + """A dictionary representing the current channelfinder information associated with the pvNames""" + existingChannels: Dict[str, CFChannel] = {} + + """ + The list of pv's is searched keeping in mind the limitations on the URL length + The search is split into groups to ensure that the size does not exceed 600 characters + """ + searchStrings = [] + searchString = "" + for channel_name in new_channels: + if not searchString: + searchString = channel_name + elif len(searchString) + len(channel_name) < 600: + searchString = searchString + "|" + channel_name + else: + searchStrings.append(searchString) + searchString = channel_name + if searchString: + searchStrings.append(searchString) + + for eachSearchString in searchStrings: + _log.debug("Find existing channels by name: %s", eachSearchString) + for found_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", eachSearchString)])): + existingChannels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) + if processor.cancelled: + raise defer.CancelledError() + return existingChannels + + def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): _log.info("CF Update IOC: %s", ioc_info) _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) @@ -755,32 +787,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] old_channels, ) # now pvNames contains a list of pv's new on this host/ioc - """A dictionary representing the current channelfinder information associated with the pvNames""" - existingChannels: Dict[str, CFChannel] = {} - - """ - The list of pv's is searched keeping in mind the limitations on the URL length - The search is split into groups to ensure that the size does not exceed 600 characters - """ - searchStrings = [] - searchString = "" - for channel_name in new_channels: - if not searchString: - searchString = channel_name - elif len(searchString) + len(channel_name) < 600: - searchString = searchString + "|" + channel_name - else: - searchStrings.append(searchString) - searchString = channel_name - if searchString: - searchStrings.append(searchString) - - for eachSearchString in searchStrings: - _log.debug("Find existing channels by name: %s", eachSearchString) - for found_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", eachSearchString)])): - existingChannels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) - if processor.cancelled: - raise defer.CancelledError() + existingChannels = get_existing_channels(new_channels, client, cf_config, processor) for channel_name in new_channels: newProps = create_properties( From ec65e3fb0220b7fa8830481d676731adc2739007 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 13:53:35 +0100 Subject: [PATCH 19/65] pull out handle old channels --- server/recceiver/cfstore.py | 97 ++++++++++++++++++++++++------------- 1 file changed, 64 insertions(+), 33 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 09abfa15..2c0929d7 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -723,6 +723,56 @@ def get_existing_channels( return existingChannels +def handle_old_channels( + old_channels: List[CFChannel], + new_channels: Set[str], + records_to_delete: List[str], + channel_ioc_ids: Dict[str, List[str]], + iocs: Dict[str, IocInfo], + ioc_info: IocInfo, + recceiverid: str, + processor: CFProcessor, + cf_config: CFConfig, + channels: List[CFChannel], + recordInfoByName: Dict[str, RecordInfo], + iocid: str, +): + for cf_channel in old_channels: + if ( + len(new_channels) == 0 or cf_channel.name in records_to_delete + ): # case: empty commit/del, remove all reference to ioc + _log.debug("Channel %s exists in Channelfinder not in new_channels", cf_channel) + if cf_channel.name in channel_ioc_ids: + handle_channel_is_old( + channel_ioc_ids, + cf_channel, + iocs, + ioc_info, + recceiverid, + processor, + cf_config, + channels, + recordInfoByName, + ) + else: + """Orphan the channel : mark as inactive, keep the old hostName and iocName""" + orphan_channel(cf_channel, ioc_info, channels, cf_config, recordInfoByName) + else: + if cf_channel.name in new_channels: # case: channel in old and new + handle_channel_old_and_new( + cf_channel, + iocid, + ioc_info, + processor, + channels, + new_channels, + cf_config, + recordInfoByName, + old_channels, + ) + return cf_channel + + def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): _log.info("CF Update IOC: %s", ioc_info) _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) @@ -753,39 +803,20 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] ] if old_channels is not None: - for cf_channel in old_channels: - if ( - len(new_channels) == 0 or cf_channel.name in records_to_delete - ): # case: empty commit/del, remove all reference to ioc - _log.debug("Channel %s exists in Channelfinder not in new_channels", cf_channel) - if cf_channel.name in channel_ioc_ids: - handle_channel_is_old( - channel_ioc_ids, - cf_channel, - iocs, - ioc_info, - recceiverid, - processor, - cf_config, - channels, - recordInfoByName, - ) - else: - """Orphan the channel : mark as inactive, keep the old hostName and iocName""" - orphan_channel(cf_channel, ioc_info, channels, cf_config, recordInfoByName) - else: - if cf_channel.name in new_channels: # case: channel in old and new - handle_channel_old_and_new( - cf_channel, - iocid, - ioc_info, - processor, - channels, - new_channels, - cf_config, - recordInfoByName, - old_channels, - ) + handle_old_channels( + old_channels, + new_channels, + records_to_delete, + channel_ioc_ids, + iocs, + ioc_info, + recceiverid, + processor, + cf_config, + channels, + recordInfoByName, + iocid, + ) # now pvNames contains a list of pv's new on this host/ioc existingChannels = get_existing_channels(new_channels, client, cf_config, processor) From 6efecaa58f01827b6b38353d9c08e37def0f243e Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 13:55:26 +0100 Subject: [PATCH 20/65] pull out update_existing_channel_diff_iocid --- server/recceiver/cfstore.py | 76 +++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 2c0929d7..7413a8ae 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -773,6 +773,45 @@ def handle_old_channels( return cf_channel +def update_existing_channel_diff_iocid( + existingChannels: Dict[str, CFChannel], + channel_name: str, + newProps: List[CFProperty], + processor: CFProcessor, + channels: List[CFChannel], + cf_config: CFConfig, + recordInfoByName: Dict[str, RecordInfo], + ioc_info: IocInfo, + iocid: str, +): + existingChannel = existingChannels[channel_name] + existingChannel.properties = __merge_property_lists( + newProps, + existingChannel, + processor.managed_properties, + ) + channels.append(existingChannel) + _log.debug("Add existing channel with different IOC: %s", existingChannel) + """in case, alias exists, update their properties too""" + if cf_config.alias_enabled: + if channel_name in recordInfoByName: + alProps = [CFProperty.alias(ioc_info.owner, channel_name)] + for p in newProps: + alProps.append(p) + for alias_name in recordInfoByName[channel_name].aliases: + if alias_name in existingChannels: + ach = existingChannels[alias_name] + ach.properties = __merge_property_lists( + alProps, + ach, + processor.managed_properties, + ) + channels.append(ach) + else: + channels.append(CFChannel(alias_name, ioc_info.owner, alProps)) + _log.debug("Add existing alias %s of %s with different IOC from %s", alias_name, channel_name, iocid) + + def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): _log.info("CF Update IOC: %s", ioc_info) _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) @@ -841,36 +880,17 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] if channel_name in existingChannels: _log.debug("update existing channel %s: exists but with a different iocid from %s", channel_name, iocid) - - existingChannel = existingChannels[channel_name] - existingChannel.properties = __merge_property_lists( + update_existing_channel_diff_iocid( + existingChannels, + channel_name, newProps, - existingChannel, - processor.managed_properties, + processor, + channels, + cf_config, + recordInfoByName, + ioc_info, + iocid, ) - channels.append(existingChannel) - _log.debug("Add existing channel with different IOC: %s", existingChannel) - """in case, alias exists, update their properties too""" - if cf_config.alias_enabled: - if channel_name in recordInfoByName: - alProps = [CFProperty.alias(ioc_info.owner, channel_name)] - for p in newProps: - alProps.append(p) - for alias_name in recordInfoByName[channel_name].aliases: - if alias_name in existingChannels: - ach = existingChannels[alias_name] - ach.properties = __merge_property_lists( - alProps, - ach, - processor.managed_properties, - ) - channels.append(ach) - else: - channels.append(CFChannel(alias_name, ioc_info.owner, alProps)) - _log.debug( - "Add existing alias %s of %s with different IOC from %s", alias_name, channel_name, iocid - ) - else: """New channel""" channels.append(CFChannel(channel_name, ioc_info.owner, newProps)) From d6244dd25b6ff280cc52a5d43399e42d7291f364 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 13:56:49 +0100 Subject: [PATCH 21/65] pull out create_new_channel --- server/recceiver/cfstore.py | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 7413a8ae..bc72b315 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -812,6 +812,26 @@ def update_existing_channel_diff_iocid( _log.debug("Add existing alias %s of %s with different IOC from %s", alias_name, channel_name, iocid) +def create_new_channel( + channels: List[CFChannel], + channel_name: str, + ioc_info: IocInfo, + newProps: List[CFProperty], + cf_config: CFConfig, + recordInfoByName: Dict[str, RecordInfo], +): + channels.append(CFChannel(channel_name, ioc_info.owner, newProps)) + _log.debug("Add new channel: %s", channel_name) + if cf_config.alias_enabled: + if channel_name in recordInfoByName: + alProps = [CFProperty.alias(ioc_info.owner, channel_name)] + for p in newProps: + alProps.append(p) + for alias in recordInfoByName[channel_name].aliases: + channels.append(CFChannel(alias, ioc_info.owner, alProps)) + _log.debug("Add new alias: %s from %s", alias, channel_name) + + def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): _log.info("CF Update IOC: %s", ioc_info) _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) @@ -893,16 +913,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] ) else: """New channel""" - channels.append(CFChannel(channel_name, ioc_info.owner, newProps)) - _log.debug("Add new channel: %s", channel_name) - if cf_config.alias_enabled: - if channel_name in recordInfoByName: - alProps = [CFProperty.alias(ioc_info.owner, channel_name)] - for p in newProps: - alProps.append(p) - for alias in recordInfoByName[channel_name].aliases: - channels.append(CFChannel(alias, ioc_info.owner, alProps)) - _log.debug("Add new alias: %s from %s", alias, channel_name) + create_new_channel(channels, channel_name, ioc_info, newProps, cf_config, recordInfoByName) _log.info("Total channels to update: %s for ioc: %s", len(channels), ioc_info) if len(channels) != 0: From 94f63177cfbb35d2c27d382610c6ef543a287079 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 4 Nov 2025 12:24:59 +0100 Subject: [PATCH 22/65] Rename create_properties to create_ioc_properties --- server/recceiver/cfstore.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index bc72b315..e703874c 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -880,7 +880,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] existingChannels = get_existing_channels(new_channels, client, cf_config, processor) for channel_name in new_channels: - newProps = create_properties( + newProps = create_ioc_properties( ioc_info.owner, ioc_info.time, recceiverid, @@ -931,7 +931,9 @@ def cf_set_chunked(client, channels: List[CFChannel], chunk_size=10000): client.set(channels=chunk) -def create_properties(owner: str, iocTime: str, recceiverid: str, hostName: str, iocName: str, iocIP: str, iocid: str): +def create_ioc_properties( + owner: str, iocTime: str, recceiverid: str, hostName: str, iocName: str, iocIP: str, iocid: str +): return [ CFProperty(CFPropertyName.hostName.name, owner, hostName), CFProperty(CFPropertyName.iocName.name, owner, iocName), @@ -948,7 +950,7 @@ def create_default_properties( ): channel_name = cf_channel.name last_ioc_info = iocs[channels_iocs[channel_name][-1]] - return create_properties( + return create_ioc_properties( ioc_info.owner, ioc_info.time, recceiverid, From dc4798ef1b7eec2b61824dc3d6c761755f859028 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 1 Dec 2025 13:39:36 +0100 Subject: [PATCH 23/65] rename to loads and classmethod Config.loads --- server/recceiver/cfstore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index e703874c..caa7d09b 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -44,8 +44,8 @@ class CFConfig: timezone: str = "" cf_query_limit: int = 10000 - @staticmethod - def from_config_adapter(conf: ConfigAdapter) -> "CFConfig": + @classmethod + def loads(cls, conf: ConfigAdapter) -> "CFConfig": return CFConfig( alias_enabled=conf.get("alias", False), record_type_enabled=conf.get("recordType", False), @@ -206,7 +206,7 @@ def from_channelfinder_dict(channel_dict: Dict[str, Any]) -> "CFChannel": @implementer(interfaces.IProcessor) class CFProcessor(service.Service): def __init__(self, name, conf): - self.cf_config = CFConfig.from_config_adapter(conf) + self.cf_config = CFConfig.loads(conf) _log.info("CF_INIT %s", self.cf_config) self.name = name self.channel_ioc_ids = defaultdict(list) From 51cecc8113d0b156ee3fb9da0c9c4ab1311de1bf Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 1 Dec 2025 14:47:24 +0100 Subject: [PATCH 24/65] Update pydoc, types and comments Set """ comments as comments Add pydoc to every method Add missing types for methods --- server/recceiver/cfstore.py | 420 ++++++++++++++++++++++++++++++------ 1 file changed, 355 insertions(+), 65 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index caa7d09b..18ef6eec 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -7,7 +7,7 @@ import time from collections import defaultdict from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Set +from typing import Any, Callable, Dict, List, Optional, Set, Tuple from channelfinder import ChannelFinderClient from requests import ConnectionError, RequestException @@ -31,6 +31,8 @@ @dataclass class CFConfig: + """Configuration options for the CF Processor""" + alias_enabled: bool = False record_type_enabled: bool = False environment_variables: str = "" @@ -41,11 +43,16 @@ class CFConfig: clean_on_stop: bool = True username: str = "cfstore" recceiver_id: str = RECCEIVERID_DEFAULT - timezone: str = "" + timezone: Optional[str] = None cf_query_limit: int = 10000 @classmethod def loads(cls, conf: ConfigAdapter) -> "CFConfig": + """Load configuration from a ConfigAdapter instance. + + Args: + conf: ConfigAdapter instance containing configuration data. + """ return CFConfig( alias_enabled=conf.get("alias", False), record_type_enabled=conf.get("recordType", False), @@ -69,10 +76,16 @@ class CFProperty: value: Optional[str] = None def as_dict(self) -> Dict[str, str]: + """Convert to dictionary for Channelfinder API.""" return {"name": self.name, "owner": self.owner, "value": str(self.value)} @staticmethod def from_channelfinder_dict(prop_dict: Dict[str, str]) -> "CFProperty": + """Create CFProperty from Channelfinder json output. + + Args: + prop_dict: Dictionary representing a property from Channelfinder. + """ return CFProperty( name=prop_dict.get("name", ""), owner=prop_dict.get("owner", ""), @@ -140,6 +153,8 @@ def time(owner: str, time: str) -> "CFProperty": @dataclass class RecordInfo: + """Information about a record to be stored in Channelfinder.""" + pvName: str recordType: Optional[str] = None infoProperties: List[CFProperty] = field(default_factory=list) @@ -147,6 +162,8 @@ class RecordInfo: class CFPropertyName(enum.Enum): + """Standard property names used in Channelfinder.""" + hostName = enum.auto() iocName = enum.auto() iocid = enum.auto() @@ -162,12 +179,16 @@ class CFPropertyName(enum.Enum): class PVStatus(enum.Enum): + """PV Status values.""" + Active = enum.auto() Inactive = enum.auto() @dataclass class IocInfo: + """Information about an IOC instance.""" + host: str hostname: str ioc_name: str @@ -179,16 +200,20 @@ class IocInfo: @property def ioc_id(self): + """Generate a unique IOC ID based on hostname and port.""" return self.host + ":" + str(self.port) @dataclass class CFChannel: + """Representation of a Channelfinder channel.""" + name: str owner: str properties: List[CFProperty] def as_dict(self) -> Dict[str, Any]: + """Convert to dictionary for conversion to json in Channelfinder API.""" return { "name": self.name, "owner": self.owner, @@ -196,6 +221,11 @@ def as_dict(self) -> Dict[str, Any]: } def from_channelfinder_dict(channel_dict: Dict[str, Any]) -> "CFChannel": + """Create CFChannel from Channelfinder json output. + + Args: + channel_dict: Dictionary representing a channel from Channelfinder. + """ return CFChannel( name=channel_dict.get("name", ""), owner=channel_dict.get("owner", ""), @@ -205,17 +235,26 @@ def from_channelfinder_dict(channel_dict: Dict[str, Any]) -> "CFChannel": @implementer(interfaces.IProcessor) class CFProcessor(service.Service): - def __init__(self, name, conf): + """Processor for committing IOC and Record information to Channelfinder.""" + + def __init__(self, name: Optional[str], conf: ConfigAdapter): + """Initialize the CFProcessor with configuration. + + Args: + name: The name of the processor. + conf: The configuration for the processor. + """ self.cf_config = CFConfig.loads(conf) _log.info("CF_INIT %s", self.cf_config) self.name = name - self.channel_ioc_ids = defaultdict(list) + self.channel_ioc_ids: Dict[str, List[str]] = defaultdict(list) self.iocs: Dict[str, IocInfo] = dict() - self.client = None - self.currentTime = getCurrentTime - self.lock = DeferredLock() + self.client: Optional[ChannelFinderClient] = None + self.currentTime: Callable[[Optional[str]], str] = getCurrentTime + self.lock: DeferredLock = DeferredLock() def startService(self): + """Start the CFProcessor service.""" service.Service.startService(self) # Returning a Deferred is not supported by startService(), # so instead attempt to acquire the lock synchonously! @@ -234,13 +273,14 @@ def startService(self): self.lock.release() def _startServiceWithLock(self): + """Start the CFProcessor service with lock held. + + Using the default python cf-client. The url, username, and + password are provided by the channelfinder._conf module. + """ _log.info("CF_START") if self.client is None: # For setting up mock test client - """ - Using the default python cf-client. The url, username, and - password are provided by the channelfinder._conf module. - """ self.client = ChannelFinderClient() try: cf_properties = {cf_property["name"] for cf_property in self.client.getAllProperties()} @@ -298,38 +338,58 @@ def _startServiceWithLock(self): self.clean_service() def stopService(self): + """Stop the CFProcessor service.""" _log.info("CF_STOP") service.Service.stopService(self) return self.lock.run(self._stopServiceWithLock) def _stopServiceWithLock(self): - # Set channels to inactive and close connection to client + """Stop the CFProcessor service with lock held. + + If clean_on_stop is enabled, mark all channels as inactive. + """ if self.cf_config.clean_on_stop: self.clean_service() _log.info("CF_STOP with lock") # @defer.inlineCallbacks # Twisted v16 does not support cancellation! - def commit(self, transaction_record): + def commit(self, transaction_record: interfaces.ITransaction) -> defer.Deferred: + """Commit a transaction to Channelfinder. + + Args: + transaction_record: The transaction to commit. + """ return self.lock.run(self._commitWithLock, transaction_record) - def _commitWithLock(self, transaction): + def _commitWithLock(self, transaction: interfaces.ITransaction) -> defer.Deferred: + """Commit a transaction to Channelfinder with lock held. + + Args: + transaction: The transaction to commit. + """ self.cancelled = False t = deferToThread(self._commitWithThread, transaction) - def cancelCommit(d): + def cancelCommit(d: defer.Deferred): + """Cancel the commit operation.""" self.cancelled = True d.callback(None) - d = defer.Deferred(cancelCommit) + d: defer.Deferred = defer.Deferred(cancelCommit) def waitForThread(_ignored): + """Wait for the commit thread to finish.""" if self.cancelled: return t d.addCallback(waitForThread) def chainError(err): + """Handle errors from the commit thread. + + Note this is not foolproof as the thread may still be running. + """ if not err.check(defer.CancelledError): _log.error("CF_COMMIT FAILURE: {s}".format(s=err)) if self.cancelled: @@ -340,6 +400,10 @@ def chainError(err): d.callback(None) def chainResult(result): + """Handle successful completion of the commit thread. + + If the commit was cancelled, raise CancelledError. + """ if self.cancelled: raise defer.CancelledError(f"CF Processor is cancelled, due to {result}") else: @@ -348,7 +412,17 @@ def chainResult(result): t.addCallbacks(chainResult, chainError) return d - def transaction_to_recordInfo(self, ioc_info: IocInfo, transaction: CommitTransaction) -> Dict[str, RecordInfo]: + def transaction_to_recordInfosById( + self, ioc_info: IocInfo, transaction: CommitTransaction + ) -> Dict[str, RecordInfo]: + """Convert a CommitTransaction and IocInfo to a dictionary of RecordInfo objects. + + Combines record additions, info tags, aliases, and environment variables. + + Args: + ioc_info: Information from the IOC + transaction: transaction from reccaster + """ recordInfo: Dict[str, RecordInfo] = {} for record_id, (record_name, record_type) in transaction.records_to_add.items(): recordInfo[record_id] = RecordInfo(pvName=record_name, recordType=None, infoProperties=[], aliases=[]) @@ -387,9 +461,17 @@ def transaction_to_recordInfo(self, ioc_info: IocInfo, transaction: CommitTransa ) return recordInfo - def record_info_by_name(self, recordInfo, ioc_info) -> Dict[str, RecordInfo]: + def record_info_by_name( + self, recordInfosByRecordID: Dict[str, RecordInfo], ioc_info: IocInfo + ) -> Dict[str, RecordInfo]: + """Create a dictionary of RecordInfo objects keyed by pvName. + + Args: + recordInfosByRecordID: Dictionary of RecordInfo objects keyed by record_id. + ioc_info: Information from the IOC. + """ recordInfoByName = {} - for record_id, (info) in recordInfo.items(): + for record_id, (info) in recordInfosByRecordID.items(): if info.pvName in recordInfoByName: _log.warning("Commit contains multiple records with PV name: %s (%s)", info.pvName, ioc_info) continue @@ -402,17 +484,27 @@ def update_ioc_infos( ioc_info: IocInfo, records_to_delete: List[str], recordInfoByName: Dict[str, RecordInfo], - ): + ) -> None: + """Update the internal IOC information based on the transaction. + + Makes changed to self.iocs and self.channel_ioc_ids and records_to_delete. + + Args: + transaction: The CommitTransaction being processed. + ioc_info: The IocInfo for the IOC in the transaction. + records_to_delete: List of record names to delete. + recordInfoByName: Dictionary of RecordInfo objects keyed by pvName. + """ iocid = ioc_info.ioc_id if transaction.initial: - """Add IOC to source list """ + # Add IOC to source list self.iocs[iocid] = ioc_info if not transaction.connected: records_to_delete.extend(self.channel_ioc_ids.keys()) for record_name in recordInfoByName.keys(): self.channel_ioc_ids[record_name].append(iocid) self.iocs[iocid].channelcount += 1 - """In case, alias exists""" + # In case, alias exists if self.cf_config.alias_enabled: if record_name in recordInfoByName: for record_aliases in recordInfoByName[record_name].aliases: @@ -421,13 +513,25 @@ def update_ioc_infos( for record_name in records_to_delete: if iocid in self.channel_ioc_ids[record_name]: self.remove_channel(record_name, iocid) - """In case, alias exists""" + # In case, alias exists if self.cf_config.alias_enabled: if record_name in recordInfoByName: for record_aliases in recordInfoByName[record_name].aliases: self.remove_channel(record_aliases, iocid) def _commitWithThread(self, transaction: CommitTransaction): + """Commit the transaction to Channelfinder. + + Collects the ioc info from the transaction. + Collects the record infos from the transaction. + Collects the records to delete from the transaction. + Calculates the records by names. + Updates the local IOC information. + Polls Channelfinder with the required updates until it passes. + + Args: + transaction: The transaction to commit. + """ if not self.running: host = transaction.source_address.host port = transaction.source_address.port @@ -446,21 +550,27 @@ def _commitWithThread(self, transaction: CommitTransaction): or transaction.client_infos.get("CF_USERNAME") or self.cf_config.username ), - time=self.currentTime(timezone=self.cf_config.timezone), + time=self.currentTime(self.cf_config.timezone), port=transaction.source_address.port, channelcount=0, ) - recordInfo = self.transaction_to_recordInfo(ioc_info, transaction) + recordInfoById = self.transaction_to_recordInfosById(ioc_info, transaction) records_to_delete = list(transaction.records_to_delete) _log.debug("Delete records: %s", records_to_delete) - recordInfoByName = self.record_info_by_name(recordInfo, ioc_info) + recordInfoByName = self.record_info_by_name(recordInfoById, ioc_info) self.update_ioc_infos(transaction, ioc_info, records_to_delete, recordInfoByName) poll(__updateCF__, self, recordInfoByName, records_to_delete, ioc_info) - def remove_channel(self, recordName: str, iocid: str): + def remove_channel(self, recordName: str, iocid: str) -> None: + """Remove channel from self.iocs and self.channel_ioc_ids. + + Args: + recordName: The name of the record to remove. + iocid: The IOC ID of the record to remove from. + """ self.channel_ioc_ids[recordName].remove(iocid) if iocid in self.iocs: self.iocs[iocid].channelcount -= 1 @@ -471,10 +581,8 @@ def remove_channel(self, recordName: str, iocid: str): if len(self.channel_ioc_ids[recordName]) <= 0: # case: channel has no more iocs del self.channel_ioc_ids[recordName] - def clean_service(self): - """ - Marks all channels as "Inactive" until the recsync server is back up - """ + def clean_service(self) -> None: + """Marks all channels belonging to this recceiver (as found by the recceiver id) as 'Inactive'.""" sleep = 1 retry_limit = 5 owner = self.cf_config.username @@ -502,7 +610,12 @@ def clean_service(self): _log.info("Abandoning clean after %s seconds", retry_limit) return - def get_active_channels(self, recceiverid) -> List[CFChannel]: + def get_active_channels(self, recceiverid: str) -> List[CFChannel]: + """Gets all the channels which are active for the given recceiver id. + + Args: + recceiverid: The current recceiver id. + """ return [ CFChannel.from_channelfinder_dict(ch) for ch in self.client.findByArgs( @@ -516,7 +629,13 @@ def get_active_channels(self, recceiverid) -> List[CFChannel]: ) ] - def clean_channels(self, owner: str, channels: List[CFChannel]): + def clean_channels(self, owner: str, channels: List[CFChannel]) -> None: + """Set the pvStatus property to 'Inactive' for the given channels. + + Args: + owner: The owner of the channels. + channels: The channels to set to 'Inactive'. + """ new_channels = [] for cf_channel in channels or []: new_channels.append(cf_channel.name) @@ -538,7 +657,23 @@ def handle_channel_is_old( cf_config: CFConfig, channels: List[CFChannel], recordInfoByName: Dict[str, RecordInfo], -): +) -> None: + """Handle the case when the channel exists in channelfinder but not in the recceiver. + + Modifies: + channels + + Args: + channel_ioc_ids: mapping of channels to ioc ids + cf_channel: The channel that is old + iocs: List of all known iocs + ioc_info: Current ioc + recceiverid: id of current recceiver + processor: Processor going through transaction + cf_config: Configuration used for processor + channels: list of the current channel changes + recordInfoByName: Input information from the transaction + """ last_ioc_id = channel_ioc_ids[cf_channel.name][-1] cf_channel.owner = iocs[last_ioc_id].owner cf_channel.properties = __merge_property_lists( @@ -554,7 +689,7 @@ def handle_channel_is_old( ) channels.append(cf_channel) _log.debug("Add existing channel %s to previous IOC %s", cf_channel, last_ioc_id) - """In case alias exist, also delete them""" + # In case alias exist, also delete them if cf_config.alias_enabled: if cf_channel.name in recordInfoByName: for alias_name in recordInfoByName[cf_channel.name].aliases: @@ -595,7 +730,19 @@ def orphan_channel( channels: List[CFChannel], cf_config: CFConfig, recordInfoByName: Dict[str, RecordInfo], -): +) -> None: + """Handle a channel that exists in channelfinder but not on this recceiver. + + Modifies: + channels + + Args: + cf_channel: The channel to orphan + ioc_info: Info of the current ioc + channels: The current list of channel changes + cf_config: Configuration of the proccessor + recordInfoByName: information from the transaction + """ cf_channel.properties = __merge_property_lists( [ CFProperty.inactive(ioc_info.owner), @@ -605,7 +752,7 @@ def orphan_channel( ) channels.append(cf_channel) _log.debug("Add orphaned channel %s with no IOC: %s", cf_channel, ioc_info) - """Also orphan any alias""" + # Also orphan any alias if cf_config.alias_enabled: if cf_channel.name in recordInfoByName: for alias_name in recordInfoByName[cf_channel.name].aliases: @@ -631,10 +778,25 @@ def handle_channel_old_and_new( cf_config: CFConfig, recordInfoByName: Dict[str, RecordInfo], old_channels: List[CFChannel], -): +) -> None: """ Channel exists in Channelfinder with same iocid. Update the status to ensure it is marked active and update the time. + + Modifies: + channels + new_channels + + Args: + cf_channel: The channel to update + iocid: The IOC ID of the channel + ioc_info: Info of the current ioc + processor: Processor going through transaction + channels: The current list of channel changes + new_channels: The list of new channels + cf_config: Configuration of the processor + recordInfoByName: information from the transaction + old_channels: The list of old channels """ _log.debug("Channel %s exists in Channelfinder with same iocid %s", cf_channel.name, iocid) cf_channel.properties = __merge_property_lists( @@ -649,12 +811,12 @@ def handle_channel_old_and_new( _log.debug("Add existing channel with same IOC: %s", cf_channel) new_channels.remove(cf_channel.name) - """In case, alias exist""" + # In case, alias exist if cf_config.alias_enabled: if cf_channel.name in recordInfoByName: for alias_name in recordInfoByName[cf_channel.name].aliases: if alias_name in old_channels: - """alias exists in old list""" + # alias exists in old list alias_channel = CFChannel(alias_name, "", []) alias_channel.properties = __merge_property_lists( [ @@ -667,7 +829,7 @@ def handle_channel_old_and_new( channels.append(alias_channel) new_channels.remove(alias_name) else: - """alias exists but not part of old list""" + # alias exists but not part of old list aprops = __merge_property_lists( [ CFProperty.active(ioc_info.owner), @@ -694,13 +856,17 @@ def handle_channel_old_and_new( def get_existing_channels( new_channels: Set[str], client: ChannelFinderClient, cf_config: CFConfig, processor: CFProcessor ) -> Dict[str, CFChannel]: - """A dictionary representing the current channelfinder information associated with the pvNames""" - existingChannels: Dict[str, CFChannel] = {} + """Get the channels existing in channelfinder from the list of new channels. + Args: + new_channels: The list of new channels. + client: The client to contact channelfinder + cf_config: The configuration for the processor. + processor: The processor. """ - The list of pv's is searched keeping in mind the limitations on the URL length - The search is split into groups to ensure that the size does not exceed 600 characters - """ + existingChannels: Dict[str, CFChannel] = {} + + # The list of pv's is searched keeping in mind the limitations on the URL length searchStrings = [] searchString = "" for channel_name in new_channels: @@ -736,7 +902,37 @@ def handle_old_channels( channels: List[CFChannel], recordInfoByName: Dict[str, RecordInfo], iocid: str, -): +) -> None: + """Handle channels already present in Channelfinder for this IOC. + + Loops through all the old_channels, + if it is on another ioc clean up reference to old ioc + if it is not on another ioc set as Inactive + if it is on current ioc update the properties + + Modifies: + channels: The list of channels. + iocs: The dictionary of IOCs. + channel_ioc_ids: The dictionary of channel names to IOC IDs. + new_channels: The list of new channels. + + Args: + old_channels: The list of old channels. + new_channels: The list of new channels. + channel_ioc_ids: The dictionary of channel names to IOC IDs. + recceiver_id: The recceiver ID. + iocs: The dictionary of IOCs. + records_to_delete: The list of records to delete. + ioc_info: The IOC information. + managed_properties: The properites managed by this recceiver. + channels: The list of channels. + alias_enabled: Whether aliases are enabled. + record_type_enabled: Whether record types are enabled. + recordInfoByName: The dictionary of record names to information. + iocid: The IOC ID. + cf_config: The configuration for the processor. + processor: The processor. + """ for cf_channel in old_channels: if ( len(new_channels) == 0 or cf_channel.name in records_to_delete @@ -755,7 +951,6 @@ def handle_old_channels( recordInfoByName, ) else: - """Orphan the channel : mark as inactive, keep the old hostName and iocName""" orphan_channel(cf_channel, ioc_info, channels, cf_config, recordInfoByName) else: if cf_channel.name in new_channels: # case: channel in old and new @@ -770,7 +965,6 @@ def handle_old_channels( recordInfoByName, old_channels, ) - return cf_channel def update_existing_channel_diff_iocid( @@ -783,7 +977,23 @@ def update_existing_channel_diff_iocid( recordInfoByName: Dict[str, RecordInfo], ioc_info: IocInfo, iocid: str, -): +) -> None: + """Update existing channel with the changed properties. + + Modifies: + channels + + Args: + existingChannels: The dictionary of existing channels. + channel_name: The name of the channel. + newProps: The new properties. + processor: The processor. + channels: The list of channels. + cf_config: configuration of processor + recordInfoByName: The dictionary of record names to information. + ioc_info: The IOC information. + iocid: The IOC ID. + """ existingChannel = existingChannels[channel_name] existingChannel.properties = __merge_property_lists( newProps, @@ -792,7 +1002,7 @@ def update_existing_channel_diff_iocid( ) channels.append(existingChannel) _log.debug("Add existing channel with different IOC: %s", existingChannel) - """in case, alias exists, update their properties too""" + # in case, alias exists, update their properties too if cf_config.alias_enabled: if channel_name in recordInfoByName: alProps = [CFProperty.alias(ioc_info.owner, channel_name)] @@ -819,7 +1029,21 @@ def create_new_channel( newProps: List[CFProperty], cf_config: CFConfig, recordInfoByName: Dict[str, RecordInfo], -): +) -> None: + """Create a new channel. + + Modifies: + channels + + Args: + channels: The list of channels. + channel_name: The name of the channel. + ioc_info: The IOC information. + newProps: The new properties. + cf_config: configuration of processor + recordInfoByName: The dictionary of record names to information. + """ + channels.append(CFChannel(channel_name, ioc_info.owner, newProps)) _log.debug("Add new channel: %s", channel_name) if cf_config.alias_enabled: @@ -832,7 +1056,19 @@ def create_new_channel( _log.debug("Add new alias: %s from %s", alias, channel_name) -def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo): +def __updateCF__( + processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo +) -> None: + """Update Channelfinder with the provided IOC and Record information. + + Calculates the changes required to the channels list and pushes the update the channelfinder. + + Args: + processor: The processor. + recordInfoByName: The dictionary of record names to information. + records_to_delete: The list of records to delete. + ioc_info: The IOC information. + """ _log.info("CF Update IOC: %s", ioc_info) _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) # Consider making this function a class methed then 'processor' simply becomes 'self' @@ -854,7 +1090,7 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] raise defer.CancelledError("Processor cancelled in __updateCF__") channels: List[CFChannel] = [] - """A list of channels in channelfinder with the associated hostName and iocName""" + # A list of channels in channelfinder with the associated hostName and iocName _log.debug("Find existing channels by IOCID: %s", ioc_info) old_channels: List[CFChannel] = [ CFChannel.from_channelfinder_dict(ch) @@ -912,7 +1148,6 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] iocid, ) else: - """New channel""" create_new_channel(channels, channel_name, ioc_info, newProps, cf_config, recordInfoByName) _log.info("Total channels to update: %s for ioc: %s", len(channels), ioc_info) @@ -925,7 +1160,14 @@ def __updateCF__(processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo] raise defer.CancelledError() -def cf_set_chunked(client, channels: List[CFChannel], chunk_size=10000): +def cf_set_chunked(client: ChannelFinderClient, channels: List[CFChannel], chunk_size=10000) -> None: + """Submit a list of channels to channelfinder in a chunked way. + + Args: + client: The channelfinder client. + channels: The list of channels. + chunk_size: The chunk size. + """ for i in range(0, len(channels), chunk_size): chunk = [ch.as_dict() for ch in channels[i : i + chunk_size]] client.set(channels=chunk) @@ -933,7 +1175,18 @@ def cf_set_chunked(client, channels: List[CFChannel], chunk_size=10000): def create_ioc_properties( owner: str, iocTime: str, recceiverid: str, hostName: str, iocName: str, iocIP: str, iocid: str -): +) -> List[CFProperty]: + """Create the properties from an IOC. + + Args: + owner: The owner of the properties. + iocTime: The time of the properties. + recceiverid: The recceiver ID of the properties. + hostName: The host name of the properties. + iocName: The IOC name of the properties. + iocIP: The IOC IP of the properties. + iocid: The IOC ID of the properties. + """ return [ CFProperty(CFPropertyName.hostName.name, owner, hostName), CFProperty(CFPropertyName.iocName.name, owner, iocName), @@ -947,7 +1200,16 @@ def create_ioc_properties( def create_default_properties( ioc_info: IocInfo, recceiverid: str, channels_iocs: Dict[str, List[str]], iocs: Dict[str, IocInfo], cf_channel -): +) -> List[CFProperty]: + """Create the default properties for an IOC. + + Args: + ioc_info: The IOC information. + recceiverid: The recceiver ID of the properties. + channels_iocs: The dictionary of channel names to IOC IDs. + iocs: The dictionary of IOCs. + cf_channel: The Channelfinder channel. + """ channel_name = cf_channel.name last_ioc_info = iocs[channels_iocs[channel_name][-1]] return create_ioc_properties( @@ -964,10 +1226,16 @@ def create_default_properties( def __merge_property_lists( newProperties: List[CFProperty], channel: CFChannel, managed_properties: Set[str] = set() ) -> List[CFProperty]: - """ - Merges two lists of properties ensuring that there are no 2 properties with + """Merges two lists of properties. + + Ensures that there are no 2 properties with the same name In case of overlap between the new and old property lists the - new property list wins out + new property list wins out. + + Args: + newProperties: The new properties. + channel: The channel. + managed_properties: The managed properties """ newPropNames = [p.name for p in newProperties] for oldProperty in channel.properties: @@ -976,13 +1244,25 @@ def __merge_property_lists( return newProperties -def getCurrentTime(timezone=False): +def getCurrentTime(timezone: Optional[str] = None) -> str: + """Get the current time. + + Args: + timezone: The timezone. + """ if timezone: return str(datetime.datetime.now().astimezone()) return str(datetime.datetime.now()) -def prepareFindArgs(cf_config: CFConfig, args, size=0): +def prepareFindArgs(cf_config: CFConfig, args, size=0) -> List[Tuple[str, str]]: + """Prepare the find arguments. + + Args: + cf_config: The configuration. + args: The arguments. + size: The size. + """ size_limit = int(cf_config.cf_query_limit) if size_limit > 0: args.append(("~size", size_limit)) @@ -990,12 +1270,21 @@ def prepareFindArgs(cf_config: CFConfig, args, size=0): def poll( - update_method, + update_method: Callable[[CFProcessor, Dict[str, RecordInfo], List[str], IocInfo], None], processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo, -): +) -> bool: + """Poll channelfinder with updates until it passes. + + Args: + update_method: The update method. + processor: The processor. + recordInfoByName: The record information by name. + records_to_delete: The records to delete. + ioc_info: The IOC information. + """ _log.info("Polling for %s begins...", ioc_info) sleep = 1.0 success = False @@ -1011,3 +1300,4 @@ def poll( time.sleep(retry_seconds) sleep *= 1.5 _log.info("Polling %s complete", ioc_info) + return success From f34d060a6bda6c8a54dbaa7de2837b373d342448 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Mon, 1 Dec 2025 15:04:46 +0100 Subject: [PATCH 25/65] Add constant for search strings --- server/recceiver/cfstore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 18ef6eec..750ee7b9 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -27,6 +27,7 @@ __all__ = ["CFProcessor"] RECCEIVERID_DEFAULT = socket.gethostname() +DEFAULT_MAX_CHANNEL_NAME_QUERY_LENGTH = 600 @dataclass From 6d51eb071314c3804591b1f19b7842b3bbb3e160 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 14:21:41 +0100 Subject: [PATCH 26/65] Remove extraneous recordType check --- server/recceiver/cfstore.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 750ee7b9..77cc8558 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -682,12 +682,6 @@ def handle_channel_is_old( cf_channel, processor.managed_properties, ) - if cf_config.record_type_enabled: - cf_channel.properties = __merge_property_lists( - cf_channel.properties.append(CFProperty.recordType(ioc_info.owner, iocs[last_ioc_id]["recordType"])), - cf_channel, - processor.managed_properties, - ) channels.append(cf_channel) _log.debug("Add existing channel %s to previous IOC %s", cf_channel, last_ioc_id) # In case alias exist, also delete them @@ -710,17 +704,6 @@ def handle_channel_is_old( alias_channel, processor.managed_properties, ) - if cf_config.record_type_enabled: - cf_channel.properties = __merge_property_lists( - cf_channel.properties.append( - CFProperty.recordType( - ioc_info.owner, - iocs[last_alias_ioc_id]["recordType"], - ) - ), - cf_channel, - processor.managed_properties, - ) channels.append(alias_channel) _log.debug("Add existing alias %s to previous IOC: %s", alias_channel, last_alias_ioc_id) From 001ee3ba98ee47376114985ba655ba92bf6fd3eb Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:14:42 +0100 Subject: [PATCH 27/65] snake case record type property --- server/recceiver/cfstore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 77cc8558..bdfa05ef 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -94,14 +94,14 @@ def from_channelfinder_dict(prop_dict: Dict[str, str]) -> "CFProperty": ) @staticmethod - def recordType(owner: str, recordType: str) -> "CFProperty": + def record_type(owner: str, record_type: str) -> "CFProperty": """Create a Channelfinder recordType property. Args: owner: The owner of the property. recordType: The recordType of the property. """ - return CFProperty(CFPropertyName.recordType.name, owner, recordType) + return CFProperty(CFPropertyName.recordType.name, owner, record_type) @staticmethod def alias(owner: str, alias: str) -> "CFProperty": @@ -1114,7 +1114,7 @@ def __updateCF__( and channel_name in recordInfoByName and recordInfoByName[channel_name].recordType ): - newProps.append(CFProperty.recordType(ioc_info.owner, recordInfoByName[channel_name].recordType)) + newProps.append(CFProperty.record_type(ioc_info.owner, recordInfoByName[channel_name].recordType)) if channel_name in recordInfoByName: newProps = newProps + recordInfoByName[channel_name].infoProperties From 8ce4bd3ae9635c6a051057f32f42572499bd68c0 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:17:39 +0100 Subject: [PATCH 28/65] Use pvstatus enum snake case pv_status --- server/recceiver/cfstore.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index bdfa05ef..c4b4df4d 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -30,6 +30,13 @@ DEFAULT_MAX_CHANNEL_NAME_QUERY_LENGTH = 600 +class PVStatus(enum.Enum): + """PV Status values.""" + + Active = enum.auto() + Inactive = enum.auto() + + @dataclass class CFConfig: """Configuration options for the CF Processor""" @@ -114,14 +121,14 @@ def alias(owner: str, alias: str) -> "CFProperty": return CFProperty(CFPropertyName.alias.name, owner, alias) @staticmethod - def pvStatus(owner: str, pvStatus: str) -> "CFProperty": + def pv_status(owner: str, pv_status: PVStatus) -> "CFProperty": """Create a Channelfinder pvStatus property. Args: owner: The owner of the property. pvStatus: The pvStatus of the property. """ - return CFProperty(CFPropertyName.pvStatus.name, owner, pvStatus) + return CFProperty(CFPropertyName.pvStatus.name, owner, pv_status.name) @staticmethod def active(owner: str) -> "CFProperty": @@ -130,7 +137,7 @@ def active(owner: str) -> "CFProperty": Args: owner: The owner of the property. """ - return CFProperty.pvStatus(owner, PVStatus.Active.name) + return CFProperty.pv_status(owner, PVStatus.Active) @staticmethod def inactive(owner: str) -> "CFProperty": @@ -139,7 +146,7 @@ def inactive(owner: str) -> "CFProperty": Args: owner: The owner of the property. """ - return CFProperty.pvStatus(owner, PVStatus.Inactive.name) + return CFProperty.pv_status(owner, PVStatus.Inactive) @staticmethod def time(owner: str, time: str) -> "CFProperty": @@ -179,13 +186,6 @@ class CFPropertyName(enum.Enum): pvaPort = enum.auto() -class PVStatus(enum.Enum): - """PV Status values.""" - - Active = enum.auto() - Inactive = enum.auto() - - @dataclass class IocInfo: """Information about an IOC instance.""" From c110fe124485f7c1fa200e7460968aa78ac9c3ae Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:20:48 +0100 Subject: [PATCH 29/65] snake case recordinfo --- server/recceiver/cfstore.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index c4b4df4d..ead82a0a 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -163,9 +163,9 @@ def time(owner: str, time: str) -> "CFProperty": class RecordInfo: """Information about a record to be stored in Channelfinder.""" - pvName: str - recordType: Optional[str] = None - infoProperties: List[CFProperty] = field(default_factory=list) + pv_name: str + record_type: Optional[str] = None + info_properties: List[CFProperty] = field(default_factory=list) aliases: List[str] = field(default_factory=list) @@ -426,9 +426,9 @@ def transaction_to_recordInfosById( """ recordInfo: Dict[str, RecordInfo] = {} for record_id, (record_name, record_type) in transaction.records_to_add.items(): - recordInfo[record_id] = RecordInfo(pvName=record_name, recordType=None, infoProperties=[], aliases=[]) + recordInfo[record_id] = RecordInfo(pv_name=record_name, record_type=None, info_properties=[], aliases=[]) if self.cf_config.record_type_enabled: - recordInfo[record_id].recordType = record_type + recordInfo[record_id].record_type = record_type for record_id, (record_infos_to_add) in transaction.record_infos_to_add.items(): # find intersection of these sets @@ -438,7 +438,7 @@ def transaction_to_recordInfosById( recinfo_wl = [p for p in self.record_property_names_list if p in record_infos_to_add.keys()] if recinfo_wl: for infotag in recinfo_wl: - recordInfo[record_id].infoProperties.append( + recordInfo[record_id].info_properties.append( CFProperty(infotag, ioc_info.owner, record_infos_to_add[infotag]) ) @@ -451,7 +451,7 @@ def transaction_to_recordInfosById( for record_id in recordInfo: for epics_env_var_name, cf_prop_name in self.env_vars.items(): if transaction.client_infos.get(epics_env_var_name) is not None: - recordInfo[record_id].infoProperties.append( + recordInfo[record_id].info_properties.append( CFProperty(cf_prop_name, ioc_info.owner, transaction.client_infos.get(epics_env_var_name)) ) else: @@ -473,10 +473,10 @@ def record_info_by_name( """ recordInfoByName = {} for record_id, (info) in recordInfosByRecordID.items(): - if info.pvName in recordInfoByName: - _log.warning("Commit contains multiple records with PV name: %s (%s)", info.pvName, ioc_info) + if info.pv_name in recordInfoByName: + _log.warning("Commit contains multiple records with PV name: %s (%s)", info.pv_name, ioc_info) continue - recordInfoByName[info.pvName] = info + recordInfoByName[info.pv_name] = info return recordInfoByName def update_ioc_infos( @@ -1112,11 +1112,11 @@ def __updateCF__( if ( cf_config.record_type_enabled and channel_name in recordInfoByName - and recordInfoByName[channel_name].recordType + and recordInfoByName[channel_name].record_type ): - newProps.append(CFProperty.record_type(ioc_info.owner, recordInfoByName[channel_name].recordType)) + newProps.append(CFProperty.record_type(ioc_info.owner, recordInfoByName[channel_name].record_type)) if channel_name in recordInfoByName: - newProps = newProps + recordInfoByName[channel_name].infoProperties + newProps = newProps + recordInfoByName[channel_name].info_properties if channel_name in existingChannels: _log.debug("update existing channel %s: exists but with a different iocid from %s", channel_name, iocid) From 7237ef6cb7d054c8ef606d673bed87c02e6305c1 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:21:12 +0100 Subject: [PATCH 30/65] static method from channelfinder_dict --- server/recceiver/cfstore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index ead82a0a..a852dc8c 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -221,6 +221,7 @@ def as_dict(self) -> Dict[str, Any]: "properties": [p.as_dict() for p in self.properties], } + @staticmethod def from_channelfinder_dict(channel_dict: Dict[str, Any]) -> "CFChannel": """Create CFChannel from Channelfinder json output. From 7c9d04be5fa0307c7f114fb0899f148a18a83b89 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:21:39 +0100 Subject: [PATCH 31/65] snake case current time --- server/recceiver/cfstore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index a852dc8c..42ed2fa6 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -252,7 +252,7 @@ def __init__(self, name: Optional[str], conf: ConfigAdapter): self.channel_ioc_ids: Dict[str, List[str]] = defaultdict(list) self.iocs: Dict[str, IocInfo] = dict() self.client: Optional[ChannelFinderClient] = None - self.currentTime: Callable[[Optional[str]], str] = getCurrentTime + self.current_time: Callable[[Optional[str]], str] = get_current_time self.lock: DeferredLock = DeferredLock() def startService(self): @@ -552,7 +552,7 @@ def _commitWithThread(self, transaction: CommitTransaction): or transaction.client_infos.get("CF_USERNAME") or self.cf_config.username ), - time=self.currentTime(self.cf_config.timezone), + time=self.current_time(self.cf_config.timezone), port=transaction.source_address.port, channelcount=0, ) @@ -1229,7 +1229,7 @@ def __merge_property_lists( return newProperties -def getCurrentTime(timezone: Optional[str] = None) -> str: +def get_current_time(timezone: Optional[str] = None) -> str: """Get the current time. Args: From 0227c90228f9dca669f43e7c98967e36972d45b7 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:22:35 +0100 Subject: [PATCH 32/65] snake case start service with lock --- server/recceiver/cfstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 42ed2fa6..40cbb573 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -267,14 +267,14 @@ def startService(self): raise RuntimeError("Failed to acquired CF Processor lock for service start") try: - self._startServiceWithLock() + self._start_service_with_lock() except: service.Service.stopService(self) raise finally: self.lock.release() - def _startServiceWithLock(self): + def _start_service_with_lock(self): """Start the CFProcessor service with lock held. Using the default python cf-client. The url, username, and From 64af0b3135e1213d94bb46baa69caf8819de92a0 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:23:22 +0100 Subject: [PATCH 33/65] snake case stop_service_with_lock --- server/recceiver/cfstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 40cbb573..37d4f29f 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -343,9 +343,9 @@ def stopService(self): """Stop the CFProcessor service.""" _log.info("CF_STOP") service.Service.stopService(self) - return self.lock.run(self._stopServiceWithLock) + return self.lock.run(self._stop_service_with_lock) - def _stopServiceWithLock(self): + def _stop_service_with_lock(self): """Stop the CFProcessor service with lock held. If clean_on_stop is enabled, mark all channels as inactive. From ec23e3609ebccd4e9cf87f77cb2fbd97df086649 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:23:57 +0100 Subject: [PATCH 34/65] snake case _commit_with_lock --- server/recceiver/cfstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 37d4f29f..ba4426da 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -361,9 +361,9 @@ def commit(self, transaction_record: interfaces.ITransaction) -> defer.Deferred: Args: transaction_record: The transaction to commit. """ - return self.lock.run(self._commitWithLock, transaction_record) + return self.lock.run(self._commit_with_lock, transaction_record) - def _commitWithLock(self, transaction: interfaces.ITransaction) -> defer.Deferred: + def _commit_with_lock(self, transaction: interfaces.ITransaction) -> defer.Deferred: """Commit a transaction to Channelfinder with lock held. Args: From b4f7977ceba6d81d4884b0321b546b3feed1c610 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:24:32 +0100 Subject: [PATCH 35/65] snake case to_record_infos --- server/recceiver/cfstore.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index ba4426da..017d7c57 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -414,9 +414,7 @@ def chainResult(result): t.addCallbacks(chainResult, chainError) return d - def transaction_to_recordInfosById( - self, ioc_info: IocInfo, transaction: CommitTransaction - ) -> Dict[str, RecordInfo]: + def transaction_to_record_infos(self, ioc_info: IocInfo, transaction: CommitTransaction) -> Dict[str, RecordInfo]: """Convert a CommitTransaction and IocInfo to a dictionary of RecordInfo objects. Combines record additions, info tags, aliases, and environment variables. @@ -557,7 +555,7 @@ def _commitWithThread(self, transaction: CommitTransaction): channelcount=0, ) - recordInfoById = self.transaction_to_recordInfosById(ioc_info, transaction) + recordInfoById = self.transaction_to_record_infos(ioc_info, transaction) records_to_delete = list(transaction.records_to_delete) _log.debug("Delete records: %s", records_to_delete) From 9024e352910b2225de39b957f0e7e1b981253e79 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:24:57 +0100 Subject: [PATCH 36/65] snake case record_infos --- server/recceiver/cfstore.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 017d7c57..1accee68 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -423,34 +423,34 @@ def transaction_to_record_infos(self, ioc_info: IocInfo, transaction: CommitTran ioc_info: Information from the IOC transaction: transaction from reccaster """ - recordInfo: Dict[str, RecordInfo] = {} + record_infos: Dict[str, RecordInfo] = {} for record_id, (record_name, record_type) in transaction.records_to_add.items(): - recordInfo[record_id] = RecordInfo(pv_name=record_name, record_type=None, info_properties=[], aliases=[]) + record_infos[record_id] = RecordInfo(pv_name=record_name, record_type=None, info_properties=[], aliases=[]) if self.cf_config.record_type_enabled: - recordInfo[record_id].record_type = record_type + record_infos[record_id].record_type = record_type for record_id, (record_infos_to_add) in transaction.record_infos_to_add.items(): # find intersection of these sets - if record_id not in recordInfo: + if record_id not in record_infos: _log.warning("IOC: %s: PV not found for recinfo with RID: {record_id}", ioc_info, record_id) continue recinfo_wl = [p for p in self.record_property_names_list if p in record_infos_to_add.keys()] if recinfo_wl: for infotag in recinfo_wl: - recordInfo[record_id].info_properties.append( + record_infos[record_id].info_properties.append( CFProperty(infotag, ioc_info.owner, record_infos_to_add[infotag]) ) for record_id, record_aliases in transaction.aliases.items(): - if record_id not in recordInfo: + if record_id not in record_infos: _log.warning("IOC: %s: PV not found for alias with RID: %s", ioc_info, record_id) continue - recordInfo[record_id].aliases = record_aliases + record_infos[record_id].aliases = record_aliases - for record_id in recordInfo: + for record_id in record_infos: for epics_env_var_name, cf_prop_name in self.env_vars.items(): if transaction.client_infos.get(epics_env_var_name) is not None: - recordInfo[record_id].info_properties.append( + record_infos[record_id].info_properties.append( CFProperty(cf_prop_name, ioc_info.owner, transaction.client_infos.get(epics_env_var_name)) ) else: @@ -459,7 +459,7 @@ def transaction_to_record_infos(self, ioc_info: IocInfo, transaction: CommitTran epics_env_var_name, ioc_info, ) - return recordInfo + return record_infos def record_info_by_name( self, recordInfosByRecordID: Dict[str, RecordInfo], ioc_info: IocInfo From 22a1b1a4a3d2449e81f6cc18b15c643911f5069f Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:27:29 +0100 Subject: [PATCH 37/65] record_info like variables snake case --- server/recceiver/cfstore.py | 118 ++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 60 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 1accee68..ed6c4c5b 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -461,29 +461,27 @@ def transaction_to_record_infos(self, ioc_info: IocInfo, transaction: CommitTran ) return record_infos - def record_info_by_name( - self, recordInfosByRecordID: Dict[str, RecordInfo], ioc_info: IocInfo - ) -> Dict[str, RecordInfo]: + def record_info_by_name(self, record_infos: Dict[str, RecordInfo], ioc_info: IocInfo) -> Dict[str, RecordInfo]: """Create a dictionary of RecordInfo objects keyed by pvName. Args: - recordInfosByRecordID: Dictionary of RecordInfo objects keyed by record_id. + record_infos: Dictionary of RecordInfo objects keyed by record_id. ioc_info: Information from the IOC. """ - recordInfoByName = {} - for record_id, (info) in recordInfosByRecordID.items(): - if info.pv_name in recordInfoByName: + record_info_by_name = {} + for record_id, (info) in record_infos.items(): + if info.pv_name in record_info_by_name: _log.warning("Commit contains multiple records with PV name: %s (%s)", info.pv_name, ioc_info) continue - recordInfoByName[info.pv_name] = info - return recordInfoByName + record_info_by_name[info.pv_name] = info + return record_info_by_name def update_ioc_infos( self, transaction: CommitTransaction, ioc_info: IocInfo, records_to_delete: List[str], - recordInfoByName: Dict[str, RecordInfo], + record_info_by_name: Dict[str, RecordInfo], ) -> None: """Update the internal IOC information based on the transaction. @@ -493,7 +491,7 @@ def update_ioc_infos( transaction: The CommitTransaction being processed. ioc_info: The IocInfo for the IOC in the transaction. records_to_delete: List of record names to delete. - recordInfoByName: Dictionary of RecordInfo objects keyed by pvName. + record_info_by_name: Dictionary of RecordInfo objects keyed by pvName. """ iocid = ioc_info.ioc_id if transaction.initial: @@ -501,13 +499,13 @@ def update_ioc_infos( self.iocs[iocid] = ioc_info if not transaction.connected: records_to_delete.extend(self.channel_ioc_ids.keys()) - for record_name in recordInfoByName.keys(): + for record_name in record_info_by_name.keys(): self.channel_ioc_ids[record_name].append(iocid) self.iocs[iocid].channelcount += 1 # In case, alias exists if self.cf_config.alias_enabled: - if record_name in recordInfoByName: - for record_aliases in recordInfoByName[record_name].aliases: + if record_name in record_info_by_name: + for record_aliases in record_info_by_name[record_name].aliases: self.channel_ioc_ids[record_aliases].append(iocid) # add iocname to pvName in dict self.iocs[iocid].channelcount += 1 for record_name in records_to_delete: @@ -515,8 +513,8 @@ def update_ioc_infos( self.remove_channel(record_name, iocid) # In case, alias exists if self.cf_config.alias_enabled: - if record_name in recordInfoByName: - for record_aliases in recordInfoByName[record_name].aliases: + if record_name in record_info_by_name: + for record_aliases in record_info_by_name[record_name].aliases: self.remove_channel(record_aliases, iocid) def _commitWithThread(self, transaction: CommitTransaction): @@ -555,14 +553,14 @@ def _commitWithThread(self, transaction: CommitTransaction): channelcount=0, ) - recordInfoById = self.transaction_to_record_infos(ioc_info, transaction) + record_infos = self.transaction_to_record_infos(ioc_info, transaction) records_to_delete = list(transaction.records_to_delete) _log.debug("Delete records: %s", records_to_delete) - recordInfoByName = self.record_info_by_name(recordInfoById, ioc_info) - self.update_ioc_infos(transaction, ioc_info, records_to_delete, recordInfoByName) - poll(__updateCF__, self, recordInfoByName, records_to_delete, ioc_info) + record_info_by_name = self.record_info_by_name(record_infos, ioc_info) + self.update_ioc_infos(transaction, ioc_info, records_to_delete, record_info_by_name) + poll(__updateCF__, self, record_info_by_name, records_to_delete, ioc_info) def remove_channel(self, recordName: str, iocid: str) -> None: """Remove channel from self.iocs and self.channel_ioc_ids. @@ -656,7 +654,7 @@ def handle_channel_is_old( processor: CFProcessor, cf_config: CFConfig, channels: List[CFChannel], - recordInfoByName: Dict[str, RecordInfo], + record_info_by_name: Dict[str, RecordInfo], ) -> None: """Handle the case when the channel exists in channelfinder but not in the recceiver. @@ -672,7 +670,7 @@ def handle_channel_is_old( processor: Processor going through transaction cf_config: Configuration used for processor channels: list of the current channel changes - recordInfoByName: Input information from the transaction + record_info_by_name: Input information from the transaction """ last_ioc_id = channel_ioc_ids[cf_channel.name][-1] cf_channel.owner = iocs[last_ioc_id].owner @@ -685,8 +683,8 @@ def handle_channel_is_old( _log.debug("Add existing channel %s to previous IOC %s", cf_channel, last_ioc_id) # In case alias exist, also delete them if cf_config.alias_enabled: - if cf_channel.name in recordInfoByName: - for alias_name in recordInfoByName[cf_channel.name].aliases: + if cf_channel.name in record_info_by_name: + for alias_name in record_info_by_name[cf_channel.name].aliases: # TODO Remove? This code couldn't have been working.... alias_channel = CFChannel(alias_name, "", []) if alias_name in channel_ioc_ids: @@ -712,7 +710,7 @@ def orphan_channel( ioc_info: IocInfo, channels: List[CFChannel], cf_config: CFConfig, - recordInfoByName: Dict[str, RecordInfo], + record_info_by_name: Dict[str, RecordInfo], ) -> None: """Handle a channel that exists in channelfinder but not on this recceiver. @@ -724,7 +722,7 @@ def orphan_channel( ioc_info: Info of the current ioc channels: The current list of channel changes cf_config: Configuration of the proccessor - recordInfoByName: information from the transaction + record_info_by_name: information from the transaction """ cf_channel.properties = __merge_property_lists( [ @@ -737,8 +735,8 @@ def orphan_channel( _log.debug("Add orphaned channel %s with no IOC: %s", cf_channel, ioc_info) # Also orphan any alias if cf_config.alias_enabled: - if cf_channel.name in recordInfoByName: - for alias_name in recordInfoByName[cf_channel.name].aliases: + if cf_channel.name in record_info_by_name: + for alias_name in record_info_by_name[cf_channel.name].aliases: alias_channel = CFChannel(alias_name, "", []) alias_channel.properties = __merge_property_lists( [ @@ -759,7 +757,7 @@ def handle_channel_old_and_new( channels: List[CFChannel], new_channels: Set[str], cf_config: CFConfig, - recordInfoByName: Dict[str, RecordInfo], + record_info_by_name: Dict[str, RecordInfo], old_channels: List[CFChannel], ) -> None: """ @@ -778,7 +776,7 @@ def handle_channel_old_and_new( channels: The current list of channel changes new_channels: The list of new channels cf_config: Configuration of the processor - recordInfoByName: information from the transaction + record_info_by_name: information from the transaction old_channels: The list of old channels """ _log.debug("Channel %s exists in Channelfinder with same iocid %s", cf_channel.name, iocid) @@ -796,8 +794,8 @@ def handle_channel_old_and_new( # In case, alias exist if cf_config.alias_enabled: - if cf_channel.name in recordInfoByName: - for alias_name in recordInfoByName[cf_channel.name].aliases: + if cf_channel.name in record_info_by_name: + for alias_name in record_info_by_name[cf_channel.name].aliases: if alias_name in old_channels: # alias exists in old list alias_channel = CFChannel(alias_name, "", []) @@ -883,7 +881,7 @@ def handle_old_channels( processor: CFProcessor, cf_config: CFConfig, channels: List[CFChannel], - recordInfoByName: Dict[str, RecordInfo], + record_info_by_name: Dict[str, RecordInfo], iocid: str, ) -> None: """Handle channels already present in Channelfinder for this IOC. @@ -911,7 +909,7 @@ def handle_old_channels( channels: The list of channels. alias_enabled: Whether aliases are enabled. record_type_enabled: Whether record types are enabled. - recordInfoByName: The dictionary of record names to information. + record_info_by_name: The dictionary of record names to information. iocid: The IOC ID. cf_config: The configuration for the processor. processor: The processor. @@ -931,10 +929,10 @@ def handle_old_channels( processor, cf_config, channels, - recordInfoByName, + record_info_by_name, ) else: - orphan_channel(cf_channel, ioc_info, channels, cf_config, recordInfoByName) + orphan_channel(cf_channel, ioc_info, channels, cf_config, record_info_by_name) else: if cf_channel.name in new_channels: # case: channel in old and new handle_channel_old_and_new( @@ -945,7 +943,7 @@ def handle_old_channels( channels, new_channels, cf_config, - recordInfoByName, + record_info_by_name, old_channels, ) @@ -957,7 +955,7 @@ def update_existing_channel_diff_iocid( processor: CFProcessor, channels: List[CFChannel], cf_config: CFConfig, - recordInfoByName: Dict[str, RecordInfo], + record_info_by_name: Dict[str, RecordInfo], ioc_info: IocInfo, iocid: str, ) -> None: @@ -973,7 +971,7 @@ def update_existing_channel_diff_iocid( processor: The processor. channels: The list of channels. cf_config: configuration of processor - recordInfoByName: The dictionary of record names to information. + record_info_by_name: The dictionary of record names to information. ioc_info: The IOC information. iocid: The IOC ID. """ @@ -987,11 +985,11 @@ def update_existing_channel_diff_iocid( _log.debug("Add existing channel with different IOC: %s", existingChannel) # in case, alias exists, update their properties too if cf_config.alias_enabled: - if channel_name in recordInfoByName: + if channel_name in record_info_by_name: alProps = [CFProperty.alias(ioc_info.owner, channel_name)] for p in newProps: alProps.append(p) - for alias_name in recordInfoByName[channel_name].aliases: + for alias_name in record_info_by_name[channel_name].aliases: if alias_name in existingChannels: ach = existingChannels[alias_name] ach.properties = __merge_property_lists( @@ -1011,7 +1009,7 @@ def create_new_channel( ioc_info: IocInfo, newProps: List[CFProperty], cf_config: CFConfig, - recordInfoByName: Dict[str, RecordInfo], + record_info_by_name: Dict[str, RecordInfo], ) -> None: """Create a new channel. @@ -1024,23 +1022,23 @@ def create_new_channel( ioc_info: The IOC information. newProps: The new properties. cf_config: configuration of processor - recordInfoByName: The dictionary of record names to information. + record_info_by_name: The dictionary of record names to information. """ channels.append(CFChannel(channel_name, ioc_info.owner, newProps)) _log.debug("Add new channel: %s", channel_name) if cf_config.alias_enabled: - if channel_name in recordInfoByName: + if channel_name in record_info_by_name: alProps = [CFProperty.alias(ioc_info.owner, channel_name)] for p in newProps: alProps.append(p) - for alias in recordInfoByName[channel_name].aliases: + for alias in record_info_by_name[channel_name].aliases: channels.append(CFChannel(alias, ioc_info.owner, alProps)) _log.debug("Add new alias: %s from %s", alias, channel_name) def __updateCF__( - processor: CFProcessor, recordInfoByName: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo + processor: CFProcessor, record_info_by_name: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo ) -> None: """Update Channelfinder with the provided IOC and Record information. @@ -1048,19 +1046,19 @@ def __updateCF__( Args: processor: The processor. - recordInfoByName: The dictionary of record names to information. + record_info_by_name: The dictionary of record names to information. records_to_delete: The list of records to delete. ioc_info: The IOC information. """ _log.info("CF Update IOC: %s", ioc_info) - _log.debug("CF Update IOC: %s recordInfoByName %s", ioc_info, recordInfoByName) + _log.debug("CF Update IOC: %s record_info_by_name %s", ioc_info, record_info_by_name) # Consider making this function a class methed then 'processor' simply becomes 'self' client = processor.client channel_ioc_ids = processor.channel_ioc_ids iocs = processor.iocs cf_config = processor.cf_config recceiverid = processor.cf_config.recceiver_id - new_channels = set(recordInfoByName.keys()) + new_channels = set(record_info_by_name.keys()) iocid = ioc_info.ioc_id if iocid not in iocs: @@ -1092,7 +1090,7 @@ def __updateCF__( processor, cf_config, channels, - recordInfoByName, + record_info_by_name, iocid, ) # now pvNames contains a list of pv's new on this host/ioc @@ -1110,12 +1108,12 @@ def __updateCF__( ) if ( cf_config.record_type_enabled - and channel_name in recordInfoByName - and recordInfoByName[channel_name].record_type + and channel_name in record_info_by_name + and record_info_by_name[channel_name].record_type ): - newProps.append(CFProperty.record_type(ioc_info.owner, recordInfoByName[channel_name].record_type)) - if channel_name in recordInfoByName: - newProps = newProps + recordInfoByName[channel_name].info_properties + newProps.append(CFProperty.record_type(ioc_info.owner, record_info_by_name[channel_name].record_type)) + if channel_name in record_info_by_name: + newProps = newProps + record_info_by_name[channel_name].info_properties if channel_name in existingChannels: _log.debug("update existing channel %s: exists but with a different iocid from %s", channel_name, iocid) @@ -1126,12 +1124,12 @@ def __updateCF__( processor, channels, cf_config, - recordInfoByName, + record_info_by_name, ioc_info, iocid, ) else: - create_new_channel(channels, channel_name, ioc_info, newProps, cf_config, recordInfoByName) + create_new_channel(channels, channel_name, ioc_info, newProps, cf_config, record_info_by_name) _log.info("Total channels to update: %s for ioc: %s", len(channels), ioc_info) if len(channels) != 0: @@ -1255,7 +1253,7 @@ def prepareFindArgs(cf_config: CFConfig, args, size=0) -> List[Tuple[str, str]]: def poll( update_method: Callable[[CFProcessor, Dict[str, RecordInfo], List[str], IocInfo], None], processor: CFProcessor, - recordInfoByName: Dict[str, RecordInfo], + record_info_by_name: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo, ) -> bool: @@ -1264,7 +1262,7 @@ def poll( Args: update_method: The update method. processor: The processor. - recordInfoByName: The record information by name. + record_info_by_name: The record information by name. records_to_delete: The records to delete. ioc_info: The IOC information. """ @@ -1273,7 +1271,7 @@ def poll( success = False while not success: try: - update_method(processor, recordInfoByName, records_to_delete, ioc_info) + update_method(processor, record_info_by_name, records_to_delete, ioc_info) success = True return success except RequestException as e: From 82658ffae31ad099e057403863a6d17a97fc5b2c Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:29:56 +0100 Subject: [PATCH 38/65] existing channels snake case --- server/recceiver/cfstore.py | 58 ++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index ed6c4c5b..5eaa97da 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -845,29 +845,29 @@ def get_existing_channels( cf_config: The configuration for the processor. processor: The processor. """ - existingChannels: Dict[str, CFChannel] = {} + existing_channels: Dict[str, CFChannel] = {} # The list of pv's is searched keeping in mind the limitations on the URL length - searchStrings = [] - searchString = "" + search_strings = [] + search_string = "" for channel_name in new_channels: - if not searchString: - searchString = channel_name - elif len(searchString) + len(channel_name) < 600: - searchString = searchString + "|" + channel_name + if not search_string: + search_string = channel_name + elif len(search_string) + len(channel_name) < 600: + search_string = search_string + "|" + channel_name else: - searchStrings.append(searchString) - searchString = channel_name - if searchString: - searchStrings.append(searchString) - - for eachSearchString in searchStrings: - _log.debug("Find existing channels by name: %s", eachSearchString) - for found_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", eachSearchString)])): - existingChannels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) + search_strings.append(search_string) + search_string = channel_name + if search_string: + search_strings.append(search_string) + + for each_search_string in search_strings: + _log.debug("Find existing channels by name: %s", each_search_string) + for found_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", each_search_string)])): + existing_channels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) if processor.cancelled: raise defer.CancelledError() - return existingChannels + return existing_channels def handle_old_channels( @@ -949,7 +949,7 @@ def handle_old_channels( def update_existing_channel_diff_iocid( - existingChannels: Dict[str, CFChannel], + existing_channels: Dict[str, CFChannel], channel_name: str, newProps: List[CFProperty], processor: CFProcessor, @@ -965,7 +965,7 @@ def update_existing_channel_diff_iocid( channels Args: - existingChannels: The dictionary of existing channels. + existing_channels: The dictionary of existing channels. channel_name: The name of the channel. newProps: The new properties. processor: The processor. @@ -975,14 +975,14 @@ def update_existing_channel_diff_iocid( ioc_info: The IOC information. iocid: The IOC ID. """ - existingChannel = existingChannels[channel_name] - existingChannel.properties = __merge_property_lists( + existing_channel = existing_channels[channel_name] + existing_channel.properties = __merge_property_lists( newProps, - existingChannel, + existing_channel, processor.managed_properties, ) - channels.append(existingChannel) - _log.debug("Add existing channel with different IOC: %s", existingChannel) + channels.append(existing_channel) + _log.debug("Add existing channel with different IOC: %s", existing_channel) # in case, alias exists, update their properties too if cf_config.alias_enabled: if channel_name in record_info_by_name: @@ -990,8 +990,8 @@ def update_existing_channel_diff_iocid( for p in newProps: alProps.append(p) for alias_name in record_info_by_name[channel_name].aliases: - if alias_name in existingChannels: - ach = existingChannels[alias_name] + if alias_name in existing_channels: + ach = existing_channels[alias_name] ach.properties = __merge_property_lists( alProps, ach, @@ -1094,7 +1094,7 @@ def __updateCF__( iocid, ) # now pvNames contains a list of pv's new on this host/ioc - existingChannels = get_existing_channels(new_channels, client, cf_config, processor) + existing_channels = get_existing_channels(new_channels, client, cf_config, processor) for channel_name in new_channels: newProps = create_ioc_properties( @@ -1115,10 +1115,10 @@ def __updateCF__( if channel_name in record_info_by_name: newProps = newProps + record_info_by_name[channel_name].info_properties - if channel_name in existingChannels: + if channel_name in existing_channels: _log.debug("update existing channel %s: exists but with a different iocid from %s", channel_name, iocid) update_existing_channel_diff_iocid( - existingChannels, + existing_channels, channel_name, newProps, processor, From a711d32e6e8375ca19f490c69fcc8ffb6982628d Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:31:43 +0100 Subject: [PATCH 39/65] alias_properties rename --- server/recceiver/cfstore.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 5eaa97da..97e8a833 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -986,20 +986,20 @@ def update_existing_channel_diff_iocid( # in case, alias exists, update their properties too if cf_config.alias_enabled: if channel_name in record_info_by_name: - alProps = [CFProperty.alias(ioc_info.owner, channel_name)] + alias_properties = [CFProperty.alias(ioc_info.owner, channel_name)] for p in newProps: - alProps.append(p) + alias_properties.append(p) for alias_name in record_info_by_name[channel_name].aliases: if alias_name in existing_channels: ach = existing_channels[alias_name] ach.properties = __merge_property_lists( - alProps, + alias_properties, ach, processor.managed_properties, ) channels.append(ach) else: - channels.append(CFChannel(alias_name, ioc_info.owner, alProps)) + channels.append(CFChannel(alias_name, ioc_info.owner, alias_properties)) _log.debug("Add existing alias %s of %s with different IOC from %s", alias_name, channel_name, iocid) @@ -1029,11 +1029,11 @@ def create_new_channel( _log.debug("Add new channel: %s", channel_name) if cf_config.alias_enabled: if channel_name in record_info_by_name: - alProps = [CFProperty.alias(ioc_info.owner, channel_name)] + alias_properties = [CFProperty.alias(ioc_info.owner, channel_name)] for p in newProps: - alProps.append(p) + alias_properties.append(p) for alias in record_info_by_name[channel_name].aliases: - channels.append(CFChannel(alias, ioc_info.owner, alProps)) + channels.append(CFChannel(alias, ioc_info.owner, alias_properties)) _log.debug("Add new alias: %s from %s", alias, channel_name) From 55cc59ed2a30e0ff45636ca7db9d0ffdd149c77a Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:32:07 +0100 Subject: [PATCH 40/65] new_properties snake case --- server/recceiver/cfstore.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 97e8a833..0c24ba7e 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -951,7 +951,7 @@ def handle_old_channels( def update_existing_channel_diff_iocid( existing_channels: Dict[str, CFChannel], channel_name: str, - newProps: List[CFProperty], + new_properties: List[CFProperty], processor: CFProcessor, channels: List[CFChannel], cf_config: CFConfig, @@ -967,7 +967,7 @@ def update_existing_channel_diff_iocid( Args: existing_channels: The dictionary of existing channels. channel_name: The name of the channel. - newProps: The new properties. + new_properties: The new properties. processor: The processor. channels: The list of channels. cf_config: configuration of processor @@ -977,7 +977,7 @@ def update_existing_channel_diff_iocid( """ existing_channel = existing_channels[channel_name] existing_channel.properties = __merge_property_lists( - newProps, + new_properties, existing_channel, processor.managed_properties, ) @@ -987,7 +987,7 @@ def update_existing_channel_diff_iocid( if cf_config.alias_enabled: if channel_name in record_info_by_name: alias_properties = [CFProperty.alias(ioc_info.owner, channel_name)] - for p in newProps: + for p in new_properties: alias_properties.append(p) for alias_name in record_info_by_name[channel_name].aliases: if alias_name in existing_channels: @@ -1007,7 +1007,7 @@ def create_new_channel( channels: List[CFChannel], channel_name: str, ioc_info: IocInfo, - newProps: List[CFProperty], + new_properties: List[CFProperty], cf_config: CFConfig, record_info_by_name: Dict[str, RecordInfo], ) -> None: @@ -1020,17 +1020,17 @@ def create_new_channel( channels: The list of channels. channel_name: The name of the channel. ioc_info: The IOC information. - newProps: The new properties. + new_properties: The new properties. cf_config: configuration of processor record_info_by_name: The dictionary of record names to information. """ - channels.append(CFChannel(channel_name, ioc_info.owner, newProps)) + channels.append(CFChannel(channel_name, ioc_info.owner, new_properties)) _log.debug("Add new channel: %s", channel_name) if cf_config.alias_enabled: if channel_name in record_info_by_name: alias_properties = [CFProperty.alias(ioc_info.owner, channel_name)] - for p in newProps: + for p in new_properties: alias_properties.append(p) for alias in record_info_by_name[channel_name].aliases: channels.append(CFChannel(alias, ioc_info.owner, alias_properties)) @@ -1097,7 +1097,7 @@ def __updateCF__( existing_channels = get_existing_channels(new_channels, client, cf_config, processor) for channel_name in new_channels: - newProps = create_ioc_properties( + new_properties = create_ioc_properties( ioc_info.owner, ioc_info.time, recceiverid, @@ -1111,16 +1111,16 @@ def __updateCF__( and channel_name in record_info_by_name and record_info_by_name[channel_name].record_type ): - newProps.append(CFProperty.record_type(ioc_info.owner, record_info_by_name[channel_name].record_type)) + new_properties.append(CFProperty.record_type(ioc_info.owner, record_info_by_name[channel_name].record_type)) if channel_name in record_info_by_name: - newProps = newProps + record_info_by_name[channel_name].info_properties + new_properties = new_properties + record_info_by_name[channel_name].info_properties if channel_name in existing_channels: _log.debug("update existing channel %s: exists but with a different iocid from %s", channel_name, iocid) update_existing_channel_diff_iocid( existing_channels, channel_name, - newProps, + new_properties, processor, channels, cf_config, @@ -1129,7 +1129,7 @@ def __updateCF__( iocid, ) else: - create_new_channel(channels, channel_name, ioc_info, newProps, cf_config, record_info_by_name) + create_new_channel(channels, channel_name, ioc_info, new_properties, cf_config, record_info_by_name) _log.info("Total channels to update: %s for ioc: %s", len(channels), ioc_info) if len(channels) != 0: From 2b8d0e00c09fc6eb34475992ca7c684fcf9e0d3d Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:54:19 +0100 Subject: [PATCH 41/65] updateCF not dunder --- server/recceiver/cfstore.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 0c24ba7e..5d83dc05 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -560,7 +560,7 @@ def _commitWithThread(self, transaction: CommitTransaction): record_info_by_name = self.record_info_by_name(record_infos, ioc_info) self.update_ioc_infos(transaction, ioc_info, records_to_delete, record_info_by_name) - poll(__updateCF__, self, record_info_by_name, records_to_delete, ioc_info) + poll(_updateCF, self, record_info_by_name, records_to_delete, ioc_info) def remove_channel(self, recordName: str, iocid: str) -> None: """Remove channel from self.iocs and self.channel_ioc_ids. @@ -1037,7 +1037,7 @@ def create_new_channel( _log.debug("Add new alias: %s from %s", alias, channel_name) -def __updateCF__( +def _updateCF( processor: CFProcessor, record_info_by_name: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo ) -> None: """Update Channelfinder with the provided IOC and Record information. @@ -1068,7 +1068,7 @@ def __updateCF__( raise Exception(f"Missing hostName {ioc_info.hostname} or iocName {ioc_info.ioc_name}") if processor.cancelled: - raise defer.CancelledError("Processor cancelled in __updateCF__") + raise defer.CancelledError(f"Processor cancelled in __updateCF__ for {ioc_info}") channels: List[CFChannel] = [] # A list of channels in channelfinder with the associated hostName and iocName @@ -1138,7 +1138,7 @@ def __updateCF__( if old_channels and len(old_channels) != 0: cf_set_chunked(client, channels, cf_config.cf_query_limit) if processor.cancelled: - raise defer.CancelledError() + raise defer.CancelledError(f"Processor cancelled in __updateCF__ for {ioc_info}") def cf_set_chunked(client: ChannelFinderClient, channels: List[CFChannel], chunk_size=10000) -> None: From 00a602ef7e43a75e863821abffcedb9075d4e896 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:57:48 +0100 Subject: [PATCH 42/65] Make sure every raise has debug info --- server/recceiver/cfstore.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 5d83dc05..d871a5da 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -560,7 +560,7 @@ def _commitWithThread(self, transaction: CommitTransaction): record_info_by_name = self.record_info_by_name(record_infos, ioc_info) self.update_ioc_infos(transaction, ioc_info, records_to_delete, record_info_by_name) - poll(_updateCF, self, record_info_by_name, records_to_delete, ioc_info) + poll(__updateCF__, self, record_info_by_name, records_to_delete, ioc_info) def remove_channel(self, recordName: str, iocid: str) -> None: """Remove channel from self.iocs and self.channel_ioc_ids. @@ -866,7 +866,9 @@ def get_existing_channels( for found_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", each_search_string)])): existing_channels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) if processor.cancelled: - raise defer.CancelledError() + raise defer.CancelledError( + f"CF Processor is cancelled, while searching for existing channels: {each_search_string}" + ) return existing_channels @@ -1037,7 +1039,7 @@ def create_new_channel( _log.debug("Add new alias: %s from %s", alias, channel_name) -def _updateCF( +def __updateCF__( processor: CFProcessor, record_info_by_name: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo ) -> None: """Update Channelfinder with the provided IOC and Record information. From dce032457346348fe26941ae4bdd15587d99bbe9 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 15:58:14 +0100 Subject: [PATCH 43/65] not dunder update cf --- server/recceiver/cfstore.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index d871a5da..c8f546a3 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -560,7 +560,7 @@ def _commitWithThread(self, transaction: CommitTransaction): record_info_by_name = self.record_info_by_name(record_infos, ioc_info) self.update_ioc_infos(transaction, ioc_info, records_to_delete, record_info_by_name) - poll(__updateCF__, self, record_info_by_name, records_to_delete, ioc_info) + poll(_update_channelfinder, self, record_info_by_name, records_to_delete, ioc_info) def remove_channel(self, recordName: str, iocid: str) -> None: """Remove channel from self.iocs and self.channel_ioc_ids. @@ -1039,7 +1039,7 @@ def create_new_channel( _log.debug("Add new alias: %s from %s", alias, channel_name) -def __updateCF__( +def _update_channelfinder( processor: CFProcessor, record_info_by_name: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo ) -> None: """Update Channelfinder with the provided IOC and Record information. @@ -1070,7 +1070,7 @@ def __updateCF__( raise Exception(f"Missing hostName {ioc_info.hostname} or iocName {ioc_info.ioc_name}") if processor.cancelled: - raise defer.CancelledError(f"Processor cancelled in __updateCF__ for {ioc_info}") + raise defer.CancelledError(f"Processor cancelled in _update_channelfinder for {ioc_info}") channels: List[CFChannel] = [] # A list of channels in channelfinder with the associated hostName and iocName @@ -1140,7 +1140,7 @@ def __updateCF__( if old_channels and len(old_channels) != 0: cf_set_chunked(client, channels, cf_config.cf_query_limit) if processor.cancelled: - raise defer.CancelledError(f"Processor cancelled in __updateCF__ for {ioc_info}") + raise defer.CancelledError(f"Processor cancelled in _update_channelfinder for {ioc_info}") def cf_set_chunked(client: ChannelFinderClient, channels: List[CFChannel], chunk_size=10000) -> None: From c335796dc47e0746eddc5508666dd5d7ce084493 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 16:01:03 +0100 Subject: [PATCH 44/65] empty string rather than "None" --- server/recceiver/cfstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index c8f546a3..73a47eb7 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -85,7 +85,7 @@ class CFProperty: def as_dict(self) -> Dict[str, str]: """Convert to dictionary for Channelfinder API.""" - return {"name": self.name, "owner": self.owner, "value": str(self.value)} + return {"name": self.name, "owner": self.owner, "value": self.value or ""} @staticmethod def from_channelfinder_dict(prop_dict: Dict[str, str]) -> "CFProperty": From 67dea8d93d541f44c161baaed81080134db033af Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 16:03:10 +0100 Subject: [PATCH 45/65] Update infotags examples --- server/demo.conf | 2 +- server/recceiver_full.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/demo.conf b/server/demo.conf index c850ec5c..e55d2667 100644 --- a/server/demo.conf +++ b/server/demo.conf @@ -66,7 +66,7 @@ [cf] # cf-store application # a space-separated list of infotags to set as CF Properties -#infotags = archive foo bar blah +#infotags = archive, foo, bar, blah # Uncomment line below to turn off the feature to add CA/PVA port info for name server to channelfinder #iocConnectionInfo = False diff --git a/server/recceiver_full.conf b/server/recceiver_full.conf index 894bf6bb..2737d788 100644 --- a/server/recceiver_full.conf +++ b/server/recceiver_full.conf @@ -62,7 +62,7 @@ idkey = 42 # cf-store application # A space-separated list of infotags to set as CF Properties -infotags = archive +infotags = archive, archiver # Feature to add CA/PVA port info for name server to channelfinder iocConnectionInfo = True From 99ce35e66d5ed895d6b9a2f4cac2fd0b7627ce66 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Tue, 2 Dec 2025 16:06:09 +0100 Subject: [PATCH 46/65] remove error format strings --- server/recceiver/cfstore.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 73a47eb7..0b8d1c2e 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -393,7 +393,7 @@ def chainError(err): Note this is not foolproof as the thread may still be running. """ if not err.check(defer.CancelledError): - _log.error("CF_COMMIT FAILURE: {s}".format(s=err)) + _log.error("CF_COMMIT FAILURE: %s", err) if self.cancelled: if not err.check(defer.CancelledError): raise defer.CancelledError() @@ -599,7 +599,7 @@ def clean_service(self) -> None: _log.info("CF Clean Completed") return except RequestException as e: - _log.error("Clean service failed: {s}".format(s=e)) + _log.error("Clean service failed: %s", e) retry_seconds = min(60, sleep) _log.info("Clean service retry in %s seconds", retry_seconds) time.sleep(retry_seconds) @@ -1277,9 +1277,9 @@ def poll( success = True return success except RequestException as e: - _log.error("ChannelFinder update failed: {s}".format(s=e)) + _log.error("ChannelFinder update failed: %s", e) retry_seconds = min(60, sleep) - _log.info("ChannelFinder update retry in {retry_seconds} seconds".format(retry_seconds=retry_seconds)) + _log.info("ChannelFinder update retry in %s seconds", retry_seconds) time.sleep(retry_seconds) sleep *= 1.5 _log.info("Polling %s complete", ioc_info) From 553d26c10437b3061fc72205b155e1ae28c03a8c Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 11:24:40 +0100 Subject: [PATCH 47/65] default query limit --- server/recceiver/cfstore.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 0b8d1c2e..41030b1b 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -28,6 +28,7 @@ RECCEIVERID_DEFAULT = socket.gethostname() DEFAULT_MAX_CHANNEL_NAME_QUERY_LENGTH = 600 +DEFAULT_QUERY_LIMIT = 10_000 class PVStatus(enum.Enum): @@ -52,7 +53,7 @@ class CFConfig: username: str = "cfstore" recceiver_id: str = RECCEIVERID_DEFAULT timezone: Optional[str] = None - cf_query_limit: int = 10000 + cf_query_limit: int = DEFAULT_QUERY_LIMIT @classmethod def loads(cls, conf: ConfigAdapter) -> "CFConfig": @@ -73,7 +74,7 @@ def loads(cls, conf: ConfigAdapter) -> "CFConfig": username=conf.get("username", "cfstore"), recceiver_id=conf.get("recceiverId", RECCEIVERID_DEFAULT), timezone=conf.get("timezone", ""), - cf_query_limit=conf.get("findSizeLimit", 10000), + cf_query_limit=conf.get("findSizeLimit", DEFAULT_QUERY_LIMIT), ) @@ -1143,7 +1144,7 @@ def _update_channelfinder( raise defer.CancelledError(f"Processor cancelled in _update_channelfinder for {ioc_info}") -def cf_set_chunked(client: ChannelFinderClient, channels: List[CFChannel], chunk_size=10000) -> None: +def cf_set_chunked(client: ChannelFinderClient, channels: List[CFChannel], chunk_size=DEFAULT_QUERY_LIMIT) -> None: """Submit a list of channels to channelfinder in a chunked way. Args: From 462a7b834294e5abacba58de91bb19c8ce966c8b Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 11:26:11 +0100 Subject: [PATCH 48/65] CFProperty use classmethods --- server/recceiver/cfstore.py | 42 ++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 41030b1b..f7fc1e51 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -88,76 +88,76 @@ def as_dict(self) -> Dict[str, str]: """Convert to dictionary for Channelfinder API.""" return {"name": self.name, "owner": self.owner, "value": self.value or ""} - @staticmethod - def from_channelfinder_dict(prop_dict: Dict[str, str]) -> "CFProperty": + @classmethod + def from_channelfinder_dict(cls, prop_dict: Dict[str, str]) -> "CFProperty": """Create CFProperty from Channelfinder json output. Args: prop_dict: Dictionary representing a property from Channelfinder. """ - return CFProperty( + return cls( name=prop_dict.get("name", ""), owner=prop_dict.get("owner", ""), value=prop_dict.get("value"), ) - @staticmethod - def record_type(owner: str, record_type: str) -> "CFProperty": + @classmethod + def record_type(cls, owner: str, record_type: str) -> "CFProperty": """Create a Channelfinder recordType property. Args: owner: The owner of the property. recordType: The recordType of the property. """ - return CFProperty(CFPropertyName.recordType.name, owner, record_type) + return cls(CFPropertyName.recordType.name, owner, record_type) - @staticmethod - def alias(owner: str, alias: str) -> "CFProperty": + @classmethod + def alias(cls, owner: str, alias: str) -> "CFProperty": """Create a Channelfinder alias property. Args: owner: The owner of the property. alias: The alias of the property. """ - return CFProperty(CFPropertyName.alias.name, owner, alias) + return cls(CFPropertyName.alias.name, owner, alias) - @staticmethod - def pv_status(owner: str, pv_status: PVStatus) -> "CFProperty": + @classmethod + def pv_status(cls, owner: str, pv_status: PVStatus) -> "CFProperty": """Create a Channelfinder pvStatus property. Args: owner: The owner of the property. pvStatus: The pvStatus of the property. """ - return CFProperty(CFPropertyName.pvStatus.name, owner, pv_status.name) + return cls(CFPropertyName.pvStatus.name, owner, pv_status.name) - @staticmethod - def active(owner: str) -> "CFProperty": + @classmethod + def active(cls, owner: str) -> "CFProperty": """Create a Channelfinder active property. Args: owner: The owner of the property. """ - return CFProperty.pv_status(owner, PVStatus.Active) + return cls.pv_status(owner, PVStatus.Active) - @staticmethod - def inactive(owner: str) -> "CFProperty": + @classmethod + def inactive(cls, owner: str) -> "CFProperty": """Create a Channelfinder inactive property. Args: owner: The owner of the property. """ - return CFProperty.pv_status(owner, PVStatus.Inactive) + return cls.pv_status(owner, PVStatus.Inactive) - @staticmethod - def time(owner: str, time: str) -> "CFProperty": + @classmethod + def time(cls, owner: str, time: str) -> "CFProperty": """Create a Channelfinder time property. Args: owner: The owner of the property. time: The time of the property. """ - return CFProperty(CFPropertyName.time.name, owner, time) + return cls(CFPropertyName.time.name, owner, time) @dataclass From 45378c43e52321c5f06ca7b53735756175009a90 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 11:28:47 +0100 Subject: [PATCH 49/65] CFChannel classmethod --- server/recceiver/cfstore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index f7fc1e51..a2a4eb4b 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -222,14 +222,14 @@ def as_dict(self) -> Dict[str, Any]: "properties": [p.as_dict() for p in self.properties], } - @staticmethod - def from_channelfinder_dict(channel_dict: Dict[str, Any]) -> "CFChannel": + @classmethod + def from_channelfinder_dict(cls, channel_dict: Dict[str, Any]) -> "CFChannel": """Create CFChannel from Channelfinder json output. Args: channel_dict: Dictionary representing a channel from Channelfinder. """ - return CFChannel( + return cls( name=channel_dict.get("name", ""), owner=channel_dict.get("owner", ""), properties=[CFProperty.from_channelfinder_dict(p) for p in channel_dict.get("properties", [])], From d0e2c4a64db5c7c1a3e1fa193d3052528b71ea96 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 11:30:09 +0100 Subject: [PATCH 50/65] rename handle_old_channels to handle_channels --- server/recceiver/cfstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index a2a4eb4b..bc267812 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -873,7 +873,7 @@ def get_existing_channels( return existing_channels -def handle_old_channels( +def handle_channels( old_channels: List[CFChannel], new_channels: Set[str], records_to_delete: List[str], @@ -1082,7 +1082,7 @@ def _update_channelfinder( ] if old_channels is not None: - handle_old_channels( + handle_channels( old_channels, new_channels, records_to_delete, From 1d3204ddc864634f73c7c3997d8edeb2eef801e1 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:09:56 +0100 Subject: [PATCH 51/65] simpler if case --- server/recceiver/cfstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index bc267812..08e13a9a 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -919,7 +919,7 @@ def handle_channels( """ for cf_channel in old_channels: if ( - len(new_channels) == 0 or cf_channel.name in records_to_delete + not new_channels or cf_channel.name in records_to_delete ): # case: empty commit/del, remove all reference to ioc _log.debug("Channel %s exists in Channelfinder not in new_channels", cf_channel) if cf_channel.name in channel_ioc_ids: From 5e5e7bcae34bed8dc0357bd02bcfd6d2f7e8f4ab Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:12:06 +0100 Subject: [PATCH 52/65] keyword args in using prepareFindArgs --- server/recceiver/cfstore.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 08e13a9a..0cb68bc5 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -619,8 +619,8 @@ def get_active_channels(self, recceiverid: str) -> List[CFChannel]: CFChannel.from_channelfinder_dict(ch) for ch in self.client.findByArgs( prepareFindArgs( - self.cf_config, - [ + cf_config=self.cf_config, + args=[ (CFPropertyName.pvStatus.name, PVStatus.Active.name), (CFPropertyName.recceiverID.name, recceiverid), ], @@ -864,7 +864,9 @@ def get_existing_channels( for each_search_string in search_strings: _log.debug("Find existing channels by name: %s", each_search_string) - for found_channel in client.findByArgs(prepareFindArgs(cf_config, [("~name", each_search_string)])): + for found_channel in client.findByArgs( + prepareFindArgs(cf_config=cf_config, args=[("~name", each_search_string)]) + ): existing_channels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) if processor.cancelled: raise defer.CancelledError( @@ -1078,7 +1080,7 @@ def _update_channelfinder( _log.debug("Find existing channels by IOCID: %s", ioc_info) old_channels: List[CFChannel] = [ CFChannel.from_channelfinder_dict(ch) - for ch in client.findByArgs(prepareFindArgs(cf_config, [("iocid", iocid)])) + for ch in client.findByArgs(prepareFindArgs(cf_config=cf_config, args=[("iocid", iocid)])) ] if old_channels is not None: From 1099f0150bbd9ee963ac3e8082ea9337ab51f688 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:13:55 +0100 Subject: [PATCH 53/65] Remove unnecessary None return --- server/recceiver/cfstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 0cb68bc5..19396249 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -574,7 +574,7 @@ def remove_channel(self, recordName: str, iocid: str) -> None: if iocid in self.iocs: self.iocs[iocid].channelcount -= 1 if self.iocs[iocid].channelcount == 0: - self.iocs.pop(iocid, None) + self.iocs.pop(iocid) elif self.iocs[iocid].channelcount < 0: _log.error("Channel count negative: %s", iocid) if len(self.channel_ioc_ids[recordName]) <= 0: # case: channel has no more iocs From 628ead3f5b25de93484ad1b1e9eee01f4ec737a0 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:15:00 +0100 Subject: [PATCH 54/65] default for channelcount --- server/recceiver/cfstore.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 19396249..1b89ffd9 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -197,8 +197,8 @@ class IocInfo: ioc_IP: str owner: str time: str - channelcount: int port: int + channelcount: int = 0 @property def ioc_id(self): @@ -551,7 +551,6 @@ def _commitWithThread(self, transaction: CommitTransaction): ), time=self.current_time(self.cf_config.timezone), port=transaction.source_address.port, - channelcount=0, ) record_infos = self.transaction_to_record_infos(ioc_info, transaction) From e6e5ffacfa3bc319e2c8bd3f00f476bdf07233ca Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:15:35 +0100 Subject: [PATCH 55/65] snake case commit with thread --- server/recceiver/cfstore.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 1b89ffd9..487559d2 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -372,7 +372,7 @@ def _commit_with_lock(self, transaction: interfaces.ITransaction) -> defer.Defer """ self.cancelled = False - t = deferToThread(self._commitWithThread, transaction) + t = deferToThread(self._commit_with_thread, transaction) def cancelCommit(d: defer.Deferred): """Cancel the commit operation.""" @@ -518,7 +518,7 @@ def update_ioc_infos( for record_aliases in record_info_by_name[record_name].aliases: self.remove_channel(record_aliases, iocid) - def _commitWithThread(self, transaction: CommitTransaction): + def _commit_with_thread(self, transaction: CommitTransaction): """Commit the transaction to Channelfinder. Collects the ioc info from the transaction. From cd33433e9b8efc2182434eb64eddff091d0cb0b1 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:18:34 +0100 Subject: [PATCH 56/65] Add comments for overriden methods --- server/recceiver/cfstore.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 487559d2..776dd63e 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -249,7 +249,7 @@ def __init__(self, name: Optional[str], conf: ConfigAdapter): """ self.cf_config = CFConfig.loads(conf) _log.info("CF_INIT %s", self.cf_config) - self.name = name + self.name = name # Override name from service.Service self.channel_ioc_ids: Dict[str, List[str]] = defaultdict(list) self.iocs: Dict[str, IocInfo] = dict() self.client: Optional[ChannelFinderClient] = None @@ -257,7 +257,10 @@ def __init__(self, name: Optional[str], conf: ConfigAdapter): self.lock: DeferredLock = DeferredLock() def startService(self): - """Start the CFProcessor service.""" + """Start the CFProcessor service. + + Overridden method of service.Service.startService() + """ service.Service.startService(self) # Returning a Deferred is not supported by startService(), # so instead attempt to acquire the lock synchonously! @@ -341,7 +344,10 @@ def _start_service_with_lock(self): self.clean_service() def stopService(self): - """Stop the CFProcessor service.""" + """Stop the CFProcessor service. + + Overridden method of service.Service.stopService() + """ _log.info("CF_STOP") service.Service.stopService(self) return self.lock.run(self._stop_service_with_lock) From 5d021fc39dc2757ae0b84c6f961c30c219d56e3a Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:19:29 +0100 Subject: [PATCH 57/65] _commit_with_lock submethods to snake case --- server/recceiver/cfstore.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 776dd63e..a89c309c 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -380,21 +380,21 @@ def _commit_with_lock(self, transaction: interfaces.ITransaction) -> defer.Defer t = deferToThread(self._commit_with_thread, transaction) - def cancelCommit(d: defer.Deferred): + def cancel_commit(d: defer.Deferred): """Cancel the commit operation.""" self.cancelled = True d.callback(None) - d: defer.Deferred = defer.Deferred(cancelCommit) + d: defer.Deferred = defer.Deferred(cancel_commit) - def waitForThread(_ignored): + def wait_for_thread(_ignored): """Wait for the commit thread to finish.""" if self.cancelled: return t - d.addCallback(waitForThread) + d.addCallback(wait_for_thread) - def chainError(err): + def chain_error(err): """Handle errors from the commit thread. Note this is not foolproof as the thread may still be running. @@ -408,7 +408,7 @@ def chainError(err): else: d.callback(None) - def chainResult(result): + def chain_result(result): """Handle successful completion of the commit thread. If the commit was cancelled, raise CancelledError. @@ -418,7 +418,7 @@ def chainResult(result): else: d.callback(None) - t.addCallbacks(chainResult, chainError) + t.addCallbacks(chain_result, chain_error) return d def transaction_to_record_infos(self, ioc_info: IocInfo, transaction: CommitTransaction) -> Dict[str, RecordInfo]: From a93ea43dad2c2b81d9c4990b859778a7f26f2bbf Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:22:21 +0100 Subject: [PATCH 58/65] static record_info_by_name --- server/recceiver/cfstore.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index a89c309c..411d40d8 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -468,7 +468,8 @@ def transaction_to_record_infos(self, ioc_info: IocInfo, transaction: CommitTran ) return record_infos - def record_info_by_name(self, record_infos: Dict[str, RecordInfo], ioc_info: IocInfo) -> Dict[str, RecordInfo]: + @staticmethod + def record_info_by_name(record_infos: Dict[str, RecordInfo], ioc_info: IocInfo) -> Dict[str, RecordInfo]: """Create a dictionary of RecordInfo objects keyed by pvName. Args: @@ -564,7 +565,7 @@ def _commit_with_thread(self, transaction: CommitTransaction): records_to_delete = list(transaction.records_to_delete) _log.debug("Delete records: %s", records_to_delete) - record_info_by_name = self.record_info_by_name(record_infos, ioc_info) + record_info_by_name = CFProcessor.record_info_by_name(record_infos, ioc_info) self.update_ioc_infos(transaction, ioc_info, records_to_delete, record_info_by_name) poll(_update_channelfinder, self, record_info_by_name, records_to_delete, ioc_info) From e35136ed5b8e6af7f0dc1a6fb243b1d954511b06 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Wed, 3 Dec 2025 12:30:06 +0100 Subject: [PATCH 59/65] reduce passing processor in static methods --- server/recceiver/cfstore.py | 49 +++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 411d40d8..36cac313 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -658,7 +658,7 @@ def handle_channel_is_old( iocs: Dict[str, IocInfo], ioc_info: IocInfo, recceiverid: str, - processor: CFProcessor, + managed_properties: Set[str], cf_config: CFConfig, channels: List[CFChannel], record_info_by_name: Dict[str, RecordInfo], @@ -674,7 +674,7 @@ def handle_channel_is_old( iocs: List of all known iocs ioc_info: Current ioc recceiverid: id of current recceiver - processor: Processor going through transaction + managed_properties: List of managed properties cf_config: Configuration used for processor channels: list of the current channel changes record_info_by_name: Input information from the transaction @@ -684,7 +684,7 @@ def handle_channel_is_old( cf_channel.properties = __merge_property_lists( create_default_properties(ioc_info, recceiverid, channel_ioc_ids, iocs, cf_channel), cf_channel, - processor.managed_properties, + managed_properties, ) channels.append(cf_channel) _log.debug("Add existing channel %s to previous IOC %s", cf_channel, last_ioc_id) @@ -706,7 +706,7 @@ def handle_channel_is_old( cf_channel, ), alias_channel, - processor.managed_properties, + managed_properties, ) channels.append(alias_channel) _log.debug("Add existing alias %s to previous IOC: %s", alias_channel, last_alias_ioc_id) @@ -760,7 +760,7 @@ def handle_channel_old_and_new( cf_channel: CFChannel, iocid: str, ioc_info: IocInfo, - processor: CFProcessor, + managed_properties: Set[str], channels: List[CFChannel], new_channels: Set[str], cf_config: CFConfig, @@ -779,7 +779,7 @@ def handle_channel_old_and_new( cf_channel: The channel to update iocid: The IOC ID of the channel ioc_info: Info of the current ioc - processor: Processor going through transaction + managed_properties: List of managed properties channels: The current list of channel changes new_channels: The list of new channels cf_config: Configuration of the processor @@ -793,7 +793,7 @@ def handle_channel_old_and_new( CFProperty.time(ioc_info.owner, ioc_info.time), ], cf_channel, - processor.managed_properties, + managed_properties, ) channels.append(cf_channel) _log.debug("Add existing channel with same IOC: %s", cf_channel) @@ -812,7 +812,7 @@ def handle_channel_old_and_new( CFProperty.time(ioc_info.owner, ioc_info.time), ], alias_channel, - processor.managed_properties, + managed_properties, ) channels.append(alias_channel) new_channels.remove(alias_name) @@ -828,7 +828,7 @@ def handle_channel_old_and_new( ), ], cf_channel, - processor.managed_properties, + managed_properties, ) channels.append( CFChannel( @@ -842,7 +842,7 @@ def handle_channel_old_and_new( def get_existing_channels( - new_channels: Set[str], client: ChannelFinderClient, cf_config: CFConfig, processor: CFProcessor + new_channels: Set[str], client: ChannelFinderClient, cf_config: CFConfig ) -> Dict[str, CFChannel]: """Get the channels existing in channelfinder from the list of new channels. @@ -850,7 +850,6 @@ def get_existing_channels( new_channels: The list of new channels. client: The client to contact channelfinder cf_config: The configuration for the processor. - processor: The processor. """ existing_channels: Dict[str, CFChannel] = {} @@ -874,10 +873,6 @@ def get_existing_channels( prepareFindArgs(cf_config=cf_config, args=[("~name", each_search_string)]) ): existing_channels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) - if processor.cancelled: - raise defer.CancelledError( - f"CF Processor is cancelled, while searching for existing channels: {each_search_string}" - ) return existing_channels @@ -889,7 +884,7 @@ def handle_channels( iocs: Dict[str, IocInfo], ioc_info: IocInfo, recceiverid: str, - processor: CFProcessor, + managed_properties: Set[str], cf_config: CFConfig, channels: List[CFChannel], record_info_by_name: Dict[str, RecordInfo], @@ -923,7 +918,6 @@ def handle_channels( record_info_by_name: The dictionary of record names to information. iocid: The IOC ID. cf_config: The configuration for the processor. - processor: The processor. """ for cf_channel in old_channels: if ( @@ -937,7 +931,7 @@ def handle_channels( iocs, ioc_info, recceiverid, - processor, + managed_properties, cf_config, channels, record_info_by_name, @@ -950,7 +944,7 @@ def handle_channels( cf_channel, iocid, ioc_info, - processor, + managed_properties, channels, new_channels, cf_config, @@ -963,7 +957,7 @@ def update_existing_channel_diff_iocid( existing_channels: Dict[str, CFChannel], channel_name: str, new_properties: List[CFProperty], - processor: CFProcessor, + managed_properties: Set[str], channels: List[CFChannel], cf_config: CFConfig, record_info_by_name: Dict[str, RecordInfo], @@ -979,7 +973,7 @@ def update_existing_channel_diff_iocid( existing_channels: The dictionary of existing channels. channel_name: The name of the channel. new_properties: The new properties. - processor: The processor. + managed_properties: The managed properties. channels: The list of channels. cf_config: configuration of processor record_info_by_name: The dictionary of record names to information. @@ -990,7 +984,7 @@ def update_existing_channel_diff_iocid( existing_channel.properties = __merge_property_lists( new_properties, existing_channel, - processor.managed_properties, + managed_properties, ) channels.append(existing_channel) _log.debug("Add existing channel with different IOC: %s", existing_channel) @@ -1006,7 +1000,7 @@ def update_existing_channel_diff_iocid( ach.properties = __merge_property_lists( alias_properties, ach, - processor.managed_properties, + managed_properties, ) channels.append(ach) else: @@ -1098,14 +1092,17 @@ def _update_channelfinder( iocs, ioc_info, recceiverid, - processor, + processor.managed_properties, cf_config, channels, record_info_by_name, iocid, ) # now pvNames contains a list of pv's new on this host/ioc - existing_channels = get_existing_channels(new_channels, client, cf_config, processor) + existing_channels = get_existing_channels(new_channels, client, cf_config) + + if processor.cancelled: + raise defer.CancelledError(f"CF Processor is cancelled, after fetching existing channels for {ioc_info}") for channel_name in new_channels: new_properties = create_ioc_properties( @@ -1132,7 +1129,7 @@ def _update_channelfinder( existing_channels, channel_name, new_properties, - processor, + processor.managed_properties, channels, cf_config, record_info_by_name, From d915481020ce373aaec01e6eaade989f3763f681 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 5 Dec 2025 09:44:36 +0100 Subject: [PATCH 60/65] from dict not from_cf_dict --- server/recceiver/cfstore.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 36cac313..97d11310 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -89,7 +89,7 @@ def as_dict(self) -> Dict[str, str]: return {"name": self.name, "owner": self.owner, "value": self.value or ""} @classmethod - def from_channelfinder_dict(cls, prop_dict: Dict[str, str]) -> "CFProperty": + def from_dict(cls, prop_dict: Dict[str, str]) -> "CFProperty": """Create CFProperty from Channelfinder json output. Args: @@ -223,7 +223,7 @@ def as_dict(self) -> Dict[str, Any]: } @classmethod - def from_channelfinder_dict(cls, channel_dict: Dict[str, Any]) -> "CFChannel": + def from_dict(cls, channel_dict: Dict[str, Any]) -> "CFChannel": """Create CFChannel from Channelfinder json output. Args: @@ -232,7 +232,7 @@ def from_channelfinder_dict(cls, channel_dict: Dict[str, Any]) -> "CFChannel": return cls( name=channel_dict.get("name", ""), owner=channel_dict.get("owner", ""), - properties=[CFProperty.from_channelfinder_dict(p) for p in channel_dict.get("properties", [])], + properties=[CFProperty.from_dict(p) for p in channel_dict.get("properties", [])], ) @@ -622,7 +622,7 @@ def get_active_channels(self, recceiverid: str) -> List[CFChannel]: recceiverid: The current recceiver id. """ return [ - CFChannel.from_channelfinder_dict(ch) + CFChannel.from_dict(ch) for ch in self.client.findByArgs( prepareFindArgs( cf_config=self.cf_config, @@ -872,7 +872,7 @@ def get_existing_channels( for found_channel in client.findByArgs( prepareFindArgs(cf_config=cf_config, args=[("~name", each_search_string)]) ): - existing_channels[found_channel["name"]] = CFChannel.from_channelfinder_dict(found_channel) + existing_channels[found_channel["name"]] = CFChannel.from_dict(found_channel) return existing_channels @@ -1079,7 +1079,7 @@ def _update_channelfinder( # A list of channels in channelfinder with the associated hostName and iocName _log.debug("Find existing channels by IOCID: %s", ioc_info) old_channels: List[CFChannel] = [ - CFChannel.from_channelfinder_dict(ch) + CFChannel.from_dict(ch) for ch in client.findByArgs(prepareFindArgs(cf_config=cf_config, args=[("iocid", iocid)])) ] From 3b1e59d2ec30d1d5d6d2939b9c8826c1c58e67d7 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 5 Dec 2025 09:46:23 +0100 Subject: [PATCH 61/65] more snake case --- server/recceiver/cfstore.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 97d11310..de90301d 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -624,7 +624,7 @@ def get_active_channels(self, recceiverid: str) -> List[CFChannel]: return [ CFChannel.from_dict(ch) for ch in self.client.findByArgs( - prepareFindArgs( + prepare_find_args( cf_config=self.cf_config, args=[ (CFPropertyName.pvStatus.name, PVStatus.Active.name), @@ -870,7 +870,7 @@ def get_existing_channels( for each_search_string in search_strings: _log.debug("Find existing channels by name: %s", each_search_string) for found_channel in client.findByArgs( - prepareFindArgs(cf_config=cf_config, args=[("~name", each_search_string)]) + prepare_find_args(cf_config=cf_config, args=[("~name", each_search_string)]) ): existing_channels[found_channel["name"]] = CFChannel.from_dict(found_channel) return existing_channels @@ -1080,7 +1080,7 @@ def _update_channelfinder( _log.debug("Find existing channels by IOCID: %s", ioc_info) old_channels: List[CFChannel] = [ CFChannel.from_dict(ch) - for ch in client.findByArgs(prepareFindArgs(cf_config=cf_config, args=[("iocid", iocid)])) + for ch in client.findByArgs(prepare_find_args(cf_config=cf_config, args=[("iocid", iocid)])) ] if old_channels is not None: @@ -1213,7 +1213,7 @@ def create_default_properties( def __merge_property_lists( - newProperties: List[CFProperty], channel: CFChannel, managed_properties: Set[str] = set() + new_properties: List[CFProperty], channel: CFChannel, managed_properties: Set[str] = set() ) -> List[CFProperty]: """Merges two lists of properties. @@ -1222,15 +1222,15 @@ def __merge_property_lists( new property list wins out. Args: - newProperties: The new properties. + new_properties: The new properties. channel: The channel. managed_properties: The managed properties """ - newPropNames = [p.name for p in newProperties] - for oldProperty in channel.properties: - if oldProperty.name not in newPropNames and (oldProperty.name not in managed_properties): - newProperties = newProperties + [oldProperty] - return newProperties + new_property_names = [p.name for p in new_properties] + for old_property in channel.properties: + if old_property.name not in new_property_names and (old_property.name not in managed_properties): + new_properties = new_properties + [old_property] + return new_properties def get_current_time(timezone: Optional[str] = None) -> str: @@ -1244,7 +1244,7 @@ def get_current_time(timezone: Optional[str] = None) -> str: return str(datetime.datetime.now()) -def prepareFindArgs(cf_config: CFConfig, args, size=0) -> List[Tuple[str, str]]: +def prepare_find_args(cf_config: CFConfig, args, size=0) -> List[Tuple[str, str]]: """Prepare the find arguments. Args: From 0cb5e002820e37c0b35d3c5dbc67076d7ac8a0de Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 5 Dec 2025 09:47:59 +0100 Subject: [PATCH 62/65] pvstatus to strenum --- server/recceiver/cfstore.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index de90301d..dde232e6 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -31,11 +31,11 @@ DEFAULT_QUERY_LIMIT = 10_000 -class PVStatus(enum.Enum): +class PVStatus(enum.StrEnum): """PV Status values.""" - Active = enum.auto() - Inactive = enum.auto() + ACTIVE = "Active" + INACTIVE = "Inactive" @dataclass @@ -129,7 +129,7 @@ def pv_status(cls, owner: str, pv_status: PVStatus) -> "CFProperty": owner: The owner of the property. pvStatus: The pvStatus of the property. """ - return cls(CFPropertyName.pvStatus.name, owner, pv_status.name) + return cls(CFPropertyName.pvStatus.name, owner, pv_status.value) @classmethod def active(cls, owner: str) -> "CFProperty": @@ -138,7 +138,7 @@ def active(cls, owner: str) -> "CFProperty": Args: owner: The owner of the property. """ - return cls.pv_status(owner, PVStatus.Active) + return cls.pv_status(owner, PVStatus.ACTIVE) @classmethod def inactive(cls, owner: str) -> "CFProperty": @@ -147,7 +147,7 @@ def inactive(cls, owner: str) -> "CFProperty": Args: owner: The owner of the property. """ - return cls.pv_status(owner, PVStatus.Inactive) + return cls.pv_status(owner, PVStatus.INACTIVE) @classmethod def time(cls, owner: str, time: str) -> "CFProperty": @@ -627,7 +627,7 @@ def get_active_channels(self, recceiverid: str) -> List[CFChannel]: prepare_find_args( cf_config=self.cf_config, args=[ - (CFPropertyName.pvStatus.name, PVStatus.Active.name), + (CFPropertyName.pvStatus.name, PVStatus.ACTIVE.value), (CFPropertyName.recceiverID.name, recceiverid), ], ) From 62ecc7d5ba4a4e3c704afa5acaaec203dcf4af90 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 5 Dec 2025 09:54:37 +0100 Subject: [PATCH 63/65] cfpropertyname as strenum --- server/recceiver/cfstore.py | 72 ++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index dde232e6..51ea2116 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -109,7 +109,7 @@ def record_type(cls, owner: str, record_type: str) -> "CFProperty": owner: The owner of the property. recordType: The recordType of the property. """ - return cls(CFPropertyName.recordType.name, owner, record_type) + return cls(CFPropertyName.RECORD_TYPE.value, owner, record_type) @classmethod def alias(cls, owner: str, alias: str) -> "CFProperty": @@ -119,7 +119,7 @@ def alias(cls, owner: str, alias: str) -> "CFProperty": owner: The owner of the property. alias: The alias of the property. """ - return cls(CFPropertyName.alias.name, owner, alias) + return cls(CFPropertyName.ALIAS.value, owner, alias) @classmethod def pv_status(cls, owner: str, pv_status: PVStatus) -> "CFProperty": @@ -129,7 +129,7 @@ def pv_status(cls, owner: str, pv_status: PVStatus) -> "CFProperty": owner: The owner of the property. pvStatus: The pvStatus of the property. """ - return cls(CFPropertyName.pvStatus.name, owner, pv_status.value) + return cls(CFPropertyName.PV_STATUS.value, owner, pv_status.value) @classmethod def active(cls, owner: str) -> "CFProperty": @@ -157,7 +157,7 @@ def time(cls, owner: str, time: str) -> "CFProperty": owner: The owner of the property. time: The time of the property. """ - return cls(CFPropertyName.time.name, owner, time) + return cls(CFPropertyName.TIME.value, owner, time) @dataclass @@ -170,21 +170,21 @@ class RecordInfo: aliases: List[str] = field(default_factory=list) -class CFPropertyName(enum.Enum): +class CFPropertyName(enum.StrEnum): """Standard property names used in Channelfinder.""" - hostName = enum.auto() - iocName = enum.auto() - iocid = enum.auto() - iocIP = enum.auto() - pvStatus = enum.auto() - time = enum.auto() - recceiverID = enum.auto() - alias = enum.auto() - recordType = enum.auto() - recordDesc = enum.auto() - caPort = enum.auto() - pvaPort = enum.auto() + HOSTNAME = "hostName" + IOC_NAME = "iocName" + IOC_ID = "iocid" + IOC_IP = "iocIP" + PV_STATUS = "pvStatus" + TIME = "time" + RECCEIVER_ID = "recceiverID" + ALIAS = "alias" + RECORD_TYPE = "recordType" + RECORD_DESC = "recordDesc" + CA_PORT = "caPort" + PVA_PORT = "pvaPort" @dataclass @@ -291,19 +291,19 @@ def _start_service_with_lock(self): try: cf_properties = {cf_property["name"] for cf_property in self.client.getAllProperties()} required_properties = { - CFPropertyName.hostName.name, - CFPropertyName.iocName.name, - CFPropertyName.iocid.name, - CFPropertyName.iocIP.name, - CFPropertyName.pvStatus.name, - CFPropertyName.time.name, - CFPropertyName.recceiverID.name, + CFPropertyName.HOSTNAME.value, + CFPropertyName.IOC_NAME.value, + CFPropertyName.IOC_ID.value, + CFPropertyName.IOC_IP.value, + CFPropertyName.PV_STATUS.value, + CFPropertyName.TIME.value, + CFPropertyName.RECCEIVER_ID.value, } if self.cf_config.alias_enabled: - required_properties.add(CFPropertyName.alias.name) + required_properties.add(CFPropertyName.ALIAS.value) if self.cf_config.record_type_enabled: - required_properties.add(CFPropertyName.recordType.name) + required_properties.add(CFPropertyName.RECORD_TYPE.value) env_vars_setting = self.cf_config.environment_variables self.env_vars = {} if env_vars_setting != "" and env_vars_setting is not None: @@ -317,12 +317,12 @@ def _start_service_with_lock(self): if self.cf_config.ioc_connection_info: self.env_vars["RSRV_SERVER_PORT"] = "caPort" self.env_vars["PVAS_SERVER_PORT"] = "pvaPort" - required_properties.add(CFPropertyName.caPort.name) - required_properties.add(CFPropertyName.pvaPort.name) + required_properties.add(CFPropertyName.CA_PORT.value) + required_properties.add(CFPropertyName.PVA_PORT.value) record_property_names_list = [s.strip(", ") for s in self.cf_config.info_tags.split()] if self.cf_config.record_description_enabled: - record_property_names_list.append(CFPropertyName.recordDesc.name) + record_property_names_list.append(CFPropertyName.RECORD_DESC.value) # Are any required properties not already present on CF? properties = required_properties - cf_properties # Are any whitelisted properties not already present on CF? @@ -627,8 +627,8 @@ def get_active_channels(self, recceiverid: str) -> List[CFChannel]: prepare_find_args( cf_config=self.cf_config, args=[ - (CFPropertyName.pvStatus.name, PVStatus.ACTIVE.value), - (CFPropertyName.recceiverID.name, recceiverid), + (CFPropertyName.PV_STATUS.value, PVStatus.ACTIVE.value), + (CFPropertyName.RECCEIVER_ID.value, recceiverid), ], ) ) @@ -1177,13 +1177,13 @@ def create_ioc_properties( iocid: The IOC ID of the properties. """ return [ - CFProperty(CFPropertyName.hostName.name, owner, hostName), - CFProperty(CFPropertyName.iocName.name, owner, iocName), - CFProperty(CFPropertyName.iocid.name, owner, iocid), - CFProperty(CFPropertyName.iocIP.name, owner, iocIP), + CFProperty(CFPropertyName.HOSTNAME.value, owner, hostName), + CFProperty(CFPropertyName.IOC_NAME.value, owner, iocName), + CFProperty(CFPropertyName.IOC_ID.value, owner, iocid), + CFProperty(CFPropertyName.IOC_IP.value, owner, iocIP), CFProperty.active(owner), CFProperty.time(owner, iocTime), - CFProperty(CFPropertyName.recceiverID.name, owner, recceiverid), + CFProperty(CFPropertyName.RECCEIVER_ID.value, owner, recceiverid), ] From 991a55077ab938f83b97f045d097e4525c6205a4 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 5 Dec 2025 09:55:56 +0100 Subject: [PATCH 64/65] record property names list set --- server/recceiver/cfstore.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index 51ea2116..b75dae56 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -320,20 +320,20 @@ def _start_service_with_lock(self): required_properties.add(CFPropertyName.CA_PORT.value) required_properties.add(CFPropertyName.PVA_PORT.value) - record_property_names_list = [s.strip(", ") for s in self.cf_config.info_tags.split()] + record_property_names_list = {s.strip(", ") for s in self.cf_config.info_tags.split()} if self.cf_config.record_description_enabled: - record_property_names_list.append(CFPropertyName.RECORD_DESC.value) + record_property_names_list.add(CFPropertyName.RECORD_DESC.value) # Are any required properties not already present on CF? properties = required_properties - cf_properties # Are any whitelisted properties not already present on CF? # If so, add them too. - properties.update(set(record_property_names_list) - cf_properties) + properties.update(record_property_names_list - cf_properties) owner = self.cf_config.username for cf_property_name in properties: self.client.set(property={"name": cf_property_name, "owner": owner}) - self.record_property_names_list = set(record_property_names_list) + self.record_property_names_list = record_property_names_list self.managed_properties = required_properties.union(record_property_names_list) _log.debug("record_property_names_list = %s", self.record_property_names_list) except ConnectionError: From f74749eb22ba3c1f833fb382dbd903f3f8c7b865 Mon Sep 17 00:00:00 2001 From: Sky Brewer Date: Fri, 5 Dec 2025 10:12:45 +0100 Subject: [PATCH 65/65] Specific error class --- server/recceiver/cfstore.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/server/recceiver/cfstore.py b/server/recceiver/cfstore.py index b75dae56..e0cedef7 100755 --- a/server/recceiver/cfstore.py +++ b/server/recceiver/cfstore.py @@ -1042,6 +1042,14 @@ def create_new_channel( _log.debug("Add new alias: %s from %s", alias, channel_name) +class IOCMissingInfoError(Exception): + """Raised when an IOC is missing required information.""" + + def __init__(self, ioc_info: IocInfo): + super().__init__(f"Missing hostName {ioc_info.hostname} or iocName {ioc_info.ioc_name}") + self.ioc_info = ioc_info + + def _update_channelfinder( processor: CFProcessor, record_info_by_name: Dict[str, RecordInfo], records_to_delete, ioc_info: IocInfo ) -> None: @@ -1070,7 +1078,7 @@ def _update_channelfinder( _log.warning("IOC Env Info %s not found in ioc list: %s", ioc_info, iocs) if ioc_info.hostname is None or ioc_info.ioc_name is None: - raise Exception(f"Missing hostName {ioc_info.hostname} or iocName {ioc_info.ioc_name}") + raise IOCMissingInfoError(ioc_info) if processor.cancelled: raise defer.CancelledError(f"Processor cancelled in _update_channelfinder for {ioc_info}")