Preparing release 0.28

Optional keep alive monitoring for TCP/IP connections

CHANGELOG.rst

@@ -1,6 +1,16 @@
 Change Log
 ----------
 
+**0.28** (2021-02-21)
+- Optional keep alive monitoring for TCP/IP connections (`pull request #73 <https://github.com/ndokter/dsmr_parser/pull/73>`_).
+- Catch parse errors in TelegramParser, ignore lines that can not be parsed (`pull request #74 <https://github.com/ndokter/dsmr_parser/pull/74>`_).
+
+**0.27** (2020-12-24)
+- fix for empty parentheses in ProfileGenericParser (redone) (`pull request #69 <https://github.com/ndokter/dsmr_parser/pull/69>`_).
+
+**0.26** (2020-12-15)
+- reverted fix for empty parentheses in ProfileGenericParser (`pull request #68 <https://github.com/ndokter/dsmr_parser/pull/68>`_).
+
 **0.25** (2020-12-14)
 - fix for empty parentheses in ProfileGenericParser (`pull request #57 <https://github.com/ndokter/dsmr_parser/pull/57>`_).
 

dsmr_parser/__main__.py

@@ -16,8 +16,8 @@ def console():
                         help='alternatively connect using TCP host.')
     parser.add_argument('--port', default=None,
                         help='TCP port to use for connection')
-    parser.add_argument('--version', default='2.2', choices=['2.2', '4'],
-                        help='DSMR version (2.2, 4)')
+    parser.add_argument('--version', default='2.2', choices=['2.2', '4', '5', '5B', '5L'],
+                        help='DSMR version (2.2, 4, 5, 5B, 5L)')
     parser.add_argument('--verbose', '-v', action='count')
 
     args = parser.parse_args()

dsmr_parser/clients/protocol.py

@@ -14,7 +14,7 @@ from dsmr_parser.clients.settings import SERIAL_SETTINGS_V2_2, \
     SERIAL_SETTINGS_V4, SERIAL_SETTINGS_V5
 
 
-def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None):
+def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None, **kwargs):
     """Creates a DSMR asyncio protocol."""
 
     if dsmr_version == '2.2':
@@ -37,7 +37,7 @@ def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None):
                                   dsmr_version)
 
     protocol = partial(DSMRProtocol, loop, TelegramParser(specification),
-                       telegram_callback=telegram_callback)
+                       telegram_callback=telegram_callback, **kwargs)
 
     return protocol, serial_settings
 
@@ -53,12 +53,14 @@ def create_dsmr_reader(port, dsmr_version, telegram_callback, loop=None):
 
 
 def create_tcp_dsmr_reader(host, port, dsmr_version,
-                           telegram_callback, loop=None):
+                           telegram_callback, loop=None,
+                           keep_alive_interval=None):
     """Creates a DSMR asyncio protocol coroutine using TCP connection."""
     if not loop:
         loop = asyncio.get_event_loop()
     protocol, _ = create_dsmr_protocol(
-        dsmr_version, telegram_callback, loop=loop)
+        dsmr_version, telegram_callback, loop=loop,
+        keep_alive_interval=keep_alive_interval)
     conn = loop.create_connection(protocol, host, port)
     return conn
 
@@ -69,7 +71,8 @@ class DSMRProtocol(asyncio.Protocol):
     transport = None
     telegram_callback = None
 
-    def __init__(self, loop, telegram_parser, telegram_callback=None):
+    def __init__(self, loop, telegram_parser,
+                 telegram_callback=None, keep_alive_interval=None):
         """Initialize class."""
         self.loop = loop
         self.log = logging.getLogger(__name__)
@@ -80,21 +83,38 @@ class DSMRProtocol(asyncio.Protocol):
         self.telegram_buffer = TelegramBuffer()
         # keep a lock until the connection is closed
         self._closed = asyncio.Event()
+        self._keep_alive_interval = keep_alive_interval
+        self._active = True
 
     def connection_made(self, transport):
         """Just logging for now."""
         self.transport = transport
         self.log.debug('connected')
+        self._active = False
+        if self.loop and self._keep_alive_interval:
+            self.loop.call_later(self._keep_alive_interval, self.keep_alive)
 
     def data_received(self, data):
         """Add incoming data to buffer."""
         data = data.decode('ascii')
+        self._active = True
         self.log.debug('received data: %s', data)
         self.telegram_buffer.append(data)
 
         for telegram in self.telegram_buffer.get_all():
             self.handle_telegram(telegram)
 
+    def keep_alive(self):
+        if self._active:
+            self.log.debug('keep-alive checked')
+            self._active = False
+            if self.loop:
+                self.loop.call_later(self._keep_alive_interval, self.keep_alive)
+        else:
+            self.log.warning('keep-alive check failed')
+            if self.transport:
+                self.transport.close()
+
     def connection_lost(self, exc):
         """Stop when connection is lost."""
         if exc:

dsmr_parser/obis_references.py

@@ -63,6 +63,6 @@ ELECTRICITY_DELIVERED_TARIFF_ALL = (
 
 # Alternate codes for foreign countries.
 BELGIUM_HOURLY_GAS_METER_READING = r'\d-\d:24\.2\.3.+?\r\n'  # Different code, same format.
-LUXEMBOURG_EQUIPMENT_IDENTIFIER = r'\d-\d:42\.0\.0.+?\r\n' # Logical device name
+LUXEMBOURG_EQUIPMENT_IDENTIFIER = r'\d-\d:42\.0\.0.+?\r\n'  # Logical device name
 LUXEMBOURG_ELECTRICITY_USED_TARIFF_GLOBAL = r'\d-\d:1\.8\.0.+?\r\n'  # Total imported energy register (P+)
 LUXEMBOURG_ELECTRICITY_DELIVERED_TARIFF_GLOBAL = r'\d-\d:2\.8\.0.+?\r\n'  # Total exported energy register (P-)

dsmr_parser/parsers.py

@@ -10,7 +10,6 @@ logger = logging.getLogger(__name__)
 
 
 class TelegramParser(object):
-
     crc16_tab = []
 
     def __init__(self, telegram_specification, apply_checksum_validation=True):
@@ -56,7 +55,11 @@ class TelegramParser(object):
             # Some signatures are optional and may not be present,
             # so only parse lines that match
             if match:
-                telegram[signature] = parser.parse(match.group(0))
+                try:
+                    telegram[signature] = parser.parse(match.group(0))
+                except Exception:
+                    logger.error("ignore line with signature {}, because parsing failed.".format(signature),
+                                 exc_info=True)
 
         return telegram
 
@@ -219,12 +222,17 @@ class ProfileGenericParser(DSMRObjectParser):
     8) Buffer value 2 (oldest entry of buffer attribute without unit)
     9) Unit of buffer values (Unit of capture objects attribute)
     """
+
     def __init__(self, buffer_types, head_parsers, parsers_for_unidentified):
         self.value_formats = head_parsers
         self.buffer_types = buffer_types
         self.parsers_for_unidentified = parsers_for_unidentified
 
     def _is_line_wellformed(self, line, values):
+        if values and (len(values) == 1) and (values[0] == ''):
+            # special case: single empty parentheses (indicated by empty string)
+            return True
+
         if values and (len(values) >= 2) and (values[0].isdigit()):
             buffer_length = int(values[0])
             return (buffer_length <= 10) and (len(values) == (buffer_length * 2 + 2))
@@ -232,6 +240,9 @@ class ProfileGenericParser(DSMRObjectParser):
             return False
 
     def _parse_values(self, values):
+        if values and (len(values) == 1) and (values[0] is None):
+            # special case: single empty parentheses; make sure empty ProfileGenericObject is created
+            values = [0, None]  # buffer_length=0, buffer_value_obis_ID=None
         buffer_length = int(values[0])
         buffer_value_obis_ID = values[1]
         if (buffer_length > 0):
@@ -264,7 +275,6 @@ class ValueParser(object):
         self.coerce_type = coerce_type
 
     def parse(self, value):
-
         unit_of_measurement = None
 
         if value and '*' in value:

setup.py

@@ -3,10 +3,10 @@ from setuptools import setup, find_packages
 setup(
     name='dsmr-parser',
     description='Library to parse Dutch Smart Meter Requirements (DSMR)',
-    author='Nigel Dokter',
+    author='Nigel Dokter and many others',
     author_email='nigel@nldr.net',
     url='https://github.com/ndokter/dsmr_parser',
-    version='0.25',
+    version='0.28',
     packages=find_packages(exclude=('test', 'test.*')),
     install_requires=[
         'pyserial>=3,<4',

test/example_telegrams.py

@@ -127,4 +127,4 @@ TELEGRAM_V5 = (
     '0-2:24.1.0(003)\r\n'
     '0-2:96.1.0()\r\n'
     '!6EEE\r\n'
-)
+)

test/test_parse_v5.py

@@ -241,7 +241,6 @@ class TelegramParserV5Test(unittest.TestCase):
 
     def test_checksum_missing(self):
         # Remove the checksum value causing a ParseError.
-        corrupted_telegram = TELEGRAM_V5.replace('!87B3\r\n', '')
-
+        corrupted_telegram = TELEGRAM_V5.replace('!6EEE\r\n', '')
         with self.assertRaises(ParseError):
             TelegramParser.validate_checksum(corrupted_telegram)

test/test_parser_corner_cases.py

@@ -0,0 +1,89 @@
+import unittest
+
+from dsmr_parser import telegram_specifications
+
+from dsmr_parser.objects import Telegram
+from dsmr_parser.objects import ProfileGenericObject
+from dsmr_parser.parsers import TelegramParser
+from dsmr_parser.parsers import ProfileGenericParser
+from dsmr_parser.profile_generic_specifications import BUFFER_TYPES
+from dsmr_parser.profile_generic_specifications import PG_HEAD_PARSERS
+from dsmr_parser.profile_generic_specifications import PG_UNIDENTIFIED_BUFFERTYPE_PARSERS
+from test.example_telegrams import TELEGRAM_V5
+
+
+class TestParserCornerCases(unittest.TestCase):
+    """ Test instantiation of Telegram object """
+
+    def test_power_event_log_empty_1(self):
+        # POWER_EVENT_FAILURE_LOG (1-0:99.97.0)
+        parser = TelegramParser(telegram_specifications.V5)
+        telegram = Telegram(TELEGRAM_V5, parser, telegram_specifications.V5)
+
+        object_type = ProfileGenericObject
+        testitem = telegram.POWER_EVENT_FAILURE_LOG
+        assert isinstance(testitem, object_type)
+        assert testitem.buffer_length == 0
+        assert testitem.buffer_type == '0-0:96.7.19'
+        buffer = testitem.buffer
+        assert isinstance(testitem.buffer, list)
+        assert len(buffer) == 0
+
+    def test_power_event_log_empty_2(self):
+        pef_parser = ProfileGenericParser(BUFFER_TYPES, PG_HEAD_PARSERS, PG_UNIDENTIFIED_BUFFERTYPE_PARSERS)
+        object_type = ProfileGenericObject
+
+        # Power Event Log with 0 items and no object type
+        pefl_line = r'1-0:99.97.0(0)()\r\n'
+        testitem = pef_parser.parse(pefl_line)
+
+        assert isinstance(testitem, object_type)
+        assert testitem.buffer_length == 0
+        assert testitem.buffer_type is None
+        buffer = testitem.buffer
+        assert isinstance(testitem.buffer, list)
+        assert len(buffer) == 0
+        assert testitem.values == [{'value': 0, 'unit': None}, {'value': None, 'unit': None}]
+        json = testitem.to_json()
+        assert json == '{"buffer_length": 0, "buffer_type": null, "buffer": []}'
+
+    def test_power_event_log_null_values(self):
+        pef_parser = ProfileGenericParser(BUFFER_TYPES, PG_HEAD_PARSERS, PG_UNIDENTIFIED_BUFFERTYPE_PARSERS)
+        object_type = ProfileGenericObject
+
+        # Power Event Log with 1 item and no object type and nno values for the item
+        pefl_line = r'1-0:99.97.0(1)()()()\r\n'
+        testitem = pef_parser.parse(pefl_line)
+
+        assert isinstance(testitem, object_type)
+        assert testitem.buffer_length == 1
+        assert testitem.buffer_type is None
+        buffer = testitem.buffer
+        assert isinstance(testitem.buffer, list)
+        assert len(buffer) == 1
+        assert testitem.values == [{'value': 1, 'unit': None}, {'value': None, 'unit': None},
+                                   {'value': None, 'unit': None}, {'value': None, 'unit': None}]
+        json = testitem.to_json()
+        assert json == \
+               '{"buffer_length": 1, "buffer_type": null, "buffer": [{"datetime": null, "value": null, "unit": null}]}'
+
+    def test_power_event_log_brackets_only(self):
+        # POWER_EVENT_FAILURE_LOG (1-0:99.97.0)
+        # Issue 57
+        # Test of an ill formatted empty POWER_EVENT_FAILURE_LOG, observed on some smartmeters
+        # The idea is that instead of failing, the parser converts it to an empty POWER_EVENT_FAILURE_LOG
+        pef_parser = ProfileGenericParser(BUFFER_TYPES, PG_HEAD_PARSERS, PG_UNIDENTIFIED_BUFFERTYPE_PARSERS)
+        object_type = ProfileGenericObject
+
+        pefl_line = r'1-0:99.97.0()\r\n'
+        testitem = pef_parser.parse(pefl_line)
+
+        assert isinstance(testitem, object_type)
+        assert testitem.buffer_length == 0
+        assert testitem.buffer_type is None
+        buffer = testitem.buffer
+        assert isinstance(testitem.buffer, list)
+        assert len(buffer) == 0
+        assert testitem.values == [{'value': 0, 'unit': None}, {'value': None, 'unit': None}]
+        json = testitem.to_json()
+        assert json == '{"buffer_length": 0, "buffer_type": null, "buffer": []}'

test/test_protocol.py

@@ -5,7 +5,7 @@ import unittest
 from dsmr_parser import obis_references as obis
 from dsmr_parser import telegram_specifications
 from dsmr_parser.parsers import TelegramParser
-from dsmr_parser.clients.protocol import DSMRProtocol
+from dsmr_parser.clients.protocol import create_dsmr_protocol
 
 
 TELEGRAM_V2_2 = (
@@ -35,9 +35,10 @@ TELEGRAM_V2_2 = (
 class ProtocolTest(unittest.TestCase):
 
     def setUp(self):
-        telegram_parser = TelegramParser(telegram_specifications.V2_2)
-        self.protocol = DSMRProtocol(None, telegram_parser,
-                                     telegram_callback=Mock())
+        new_protocol, _ = create_dsmr_protocol('2.2',
+                                               telegram_callback=Mock(),
+                                               keep_alive_interval=1)
+        self.protocol = new_protocol()
 
     def test_complete_packet(self):
         """Protocol should assemble incoming lines into complete packet."""
@@ -52,3 +53,23 @@ class ProtocolTest(unittest.TestCase):
 
         assert float(telegram[obis.GAS_METER_READING].value) == 1.001
         assert telegram[obis.GAS_METER_READING].unit == 'm3'
+
+    def test_receive_packet(self):
+        """Protocol packet reception."""
+
+        mock_transport = Mock()
+        self.protocol.connection_made(mock_transport)
+        assert not self.protocol._active
+
+        self.protocol.data_received(TELEGRAM_V2_2.encode('ascii'))
+        assert self.protocol._active
+
+        # 1st call of keep_alive resets 'active' flag
+        self.protocol.keep_alive()
+        assert not self.protocol._active
+
+        # 2nd call of keep_alive should close the transport
+        self.protocol.keep_alive()
+        assert mock_transport.close.called_once()
+
+        self.protocol.connection_lost(None)