Merge branch 'master' of github.com:ndokter/dsmr_parser

* improve performance parsing speed by about 6% by caching compiled regexes

CHANGELOG.rst

@@ -1,9 +1,19 @@
 Change Log
 ----------
 
+**1.2.2** (2023-04-12)
+
+- Improve performance. Thanks to `ejpalacios <https://github.com/bdraco>`_ (`PR #130 <https://github.com/ndokter/dsmr_parser/pull/130>`_ by `ndokter <https://github.com/ndokter>`_)
+
+**1.2.1** (2023-04-05)
+
+- Bug/duplicate index BELGIUM_MAXIMUM_DEMAND_13_MONTHS (`PR #129 <https://github.com/ndokter/dsmr_parser/pull/129>`_ by `ejpalacios <https://github.com/ejpalacios>`_)
+
 **1.2.0** (2023-02-18)
 
 - Improved gas meter (mbus devices) support and replaced Telegram dictionary with backwards compatible object (`PR #121 <https://github.com/ndokter/dsmr_parser/pull/121>`_ by `ndokter <https://github.com/ndokter>`_)
+- Fix parsing with invalid timestamps (`PR #125 <https://github.com/ndokter/dsmr_parser/pull/125>`_ by `dupondje <https://github.com/dupondje>`_)
+- Add Iskra IE.x meters specification (`PR #126 <https://github.com/ndokter/dsmr_parser/pull/126>`_ by `jchevalier7 <https://github.com/jchevalier7>`_)
 
 **1.1.0** (2023-02-08)
 

dsmr_parser/clients/telegram_buffer.py

@@ -1,5 +1,13 @@
 import re
 
+# - Match all characters after start of telegram except for the start
+# itself again '^\/]+', which eliminates incomplete preceding telegrams.
+# - Do non greedy match using '?' so start is matched up to the first
+# checksum that's found.
+# - The checksum is optional '{0,4}' because not all telegram versions
+# support it.
+_FIND_TELEGRAMS_REGEX = re.compile(r"\/[^\/]+?\![A-F0-9]{0,4}\0?\r\n", re.DOTALL)
+
 
 class TelegramBuffer(object):
     """
@@ -8,14 +16,14 @@ class TelegramBuffer(object):
     """
 
     def __init__(self):
-        self._buffer = ''
+        self._buffer = ""
 
     def get_all(self):
         """
         Remove complete telegrams from buffer and yield them.
         :rtype generator:
         """
-        for telegram in self._find_telegrams():
+        for telegram in _FIND_TELEGRAMS_REGEX.findall(self._buffer):
             self._remove(telegram)
             yield telegram
 
@@ -37,21 +45,3 @@ class TelegramBuffer(object):
         index = self._buffer.index(telegram) + len(telegram)
 
         self._buffer = self._buffer[index:]
-
-    def _find_telegrams(self):
-        """
-        Find complete telegrams in buffer from  start ('/') till ending
-        checksum ('!AB12\r\n').
-        :rtype: list
-        """
-        # - Match all characters after start of telegram except for the start
-        # itself again '^\/]+', which eliminates incomplete preceding telegrams.
-        # - Do non greedy match using '?' so start is matched up to the first
-        # checksum that's found.
-        # - The checksum is optional '{0,4}' because not all telegram versions
-        # support it.
-        return re.findall(
-            r'\/[^\/]+?\![A-F0-9]{0,4}\0?\r\n',
-            self._buffer,
-            re.DOTALL
-        )

dsmr_parser/parsers.py

@@ -25,8 +25,13 @@ class TelegramParser(object):
             telegram DSMR version (v4 and up).
         :type telegram_specification: dict
         """
-        self.telegram_specification = telegram_specification
         self.apply_checksum_validation = apply_checksum_validation
+        self.telegram_specification = telegram_specification
+        # Regexes are compiled once to improve performance
+        self.telegram_specification_regexes = {
+            signature: re.compile(signature, re.DOTALL)
+            for signature in self.telegram_specification['objects'].keys()
+        }
 
     def parse(self, telegram_data, encryption_key="", authentication_key=""):  # noqa: C901
         """
@@ -80,7 +85,7 @@ class TelegramParser(object):
         telegram = Telegram()
 
         for signature, parser in self.telegram_specification['objects'].items():
-            pattern = re.compile(signature, re.DOTALL)
+            pattern = self.telegram_specification_regexes[signature]
             matches = pattern.findall(telegram_data)
 
             # Some signatures are optional and may not be present,
@@ -258,7 +263,7 @@ class MaxDemandParser(DSMRObjectParser):
 
         count = int(values[0])
         for i in range(1, count + 1):
-            timestamp_month = ValueParser(timestamp).parse(values[i * 3 + 1])
+            timestamp_month = ValueParser(timestamp).parse(values[i * 3 + 0])
             timestamp_occurred = ValueParser(timestamp).parse(values[i * 3 + 1])
             value = ValueParser(Decimal).parse(values[i * 3 + 2])
             objects.append(MBusObjectPeak(

setup.py

@@ -7,7 +7,7 @@ setup(
     author_email='mail@nldr.net',
     license='MIT',
     url='https://github.com/ndokter/dsmr_parser',
-    version='1.2.0',
+    version='1.2.2',
     packages=find_packages(exclude=('test', 'test.*')),
     install_requires=[
         'pyserial>=3,<4',

test/test_parse_fluvius.py

@@ -37,7 +37,7 @@ class TelegramParserFluviusTest(unittest.TestCase):
         assert result[obis.P1_MESSAGE_TIMESTAMP].unit is None
         assert isinstance(result[obis.P1_MESSAGE_TIMESTAMP].value, datetime.datetime)
         assert result[obis.P1_MESSAGE_TIMESTAMP].value == \
-            datetime.datetime(2020, 5, 12, 11, 54, 9, tzinfo=pytz.UTC)
+            pytz.timezone("Europe/Brussels").localize(datetime.datetime(2020, 5, 12, 13, 54, 9))
 
         # ELECTRICITY_USED_TARIFF_1 (1-0:1.8.1)
         assert isinstance(result[obis.ELECTRICITY_USED_TARIFF_1], CosemObject)
@@ -80,22 +80,43 @@ class TelegramParserFluviusTest(unittest.TestCase):
         assert result[obis.BELGIUM_MAXIMUM_DEMAND_MONTH].unit == 'kW'
         assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_MONTH].value, Decimal)
         assert result[obis.BELGIUM_MAXIMUM_DEMAND_MONTH].value == Decimal('2.589')
+        assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_MONTH].datetime, datetime.datetime)
+        assert result[obis.BELGIUM_MAXIMUM_DEMAND_MONTH].datetime == \
+            pytz.timezone("Europe/Brussels").localize(datetime.datetime(2020, 5, 9, 13, 45, 58))
 
         # BELGIUM_MAXIMUM_DEMAND_13_MONTHS (0-0:98.1.0) Value 0
         assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][0], MBusObjectPeak)
         assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][0].unit == 'kW'
         assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][0].value, Decimal)
         assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][0].value == Decimal('3.695')
+        assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][0].datetime, datetime.datetime)
+        assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][0].datetime == \
+            pytz.timezone("Europe/Brussels").localize(datetime.datetime(2020, 5, 1, 0, 0, 0))
+        assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][0].occurred, datetime.datetime)
+        assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][0].occurred == \
+            pytz.timezone("Europe/Brussels").localize(datetime.datetime(2020, 4, 23, 19, 25, 38))
         # BELGIUM_MAXIMUM_DEMAND_13_MONTHS (0-0:98.1.0) Value 1
         assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][1], MBusObjectPeak)
         assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][1].unit == 'kW'
         assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][1].value, Decimal)
         assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][1].value == Decimal('5.980')
+        assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][1].datetime, datetime.datetime)
+        assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][1].datetime == \
+            pytz.timezone("Europe/Brussels").localize(datetime.datetime(2020, 4, 1, 0, 0, 0))
+        assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][1].occurred, datetime.datetime)
+        assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][1].occurred == \
+            pytz.timezone("Europe/Brussels").localize(datetime.datetime(2020, 3, 5, 12, 21, 39))
         # BELGIUM_MAXIMUM_DEMAND_13_MONTHS (0-0:98.1.0) Value 2
         assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][2], MBusObjectPeak)
         assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][2].unit == 'kW'
         assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][2].value, Decimal)
         assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][2].value == Decimal('4.318')
+        assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][2].datetime, datetime.datetime)
+        assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][2].datetime == \
+            pytz.timezone("Europe/Brussels").localize(datetime.datetime(2020, 3, 1, 0, 0, 0))
+        assert isinstance(result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][2].occurred, datetime.datetime)
+        assert result[obis.BELGIUM_MAXIMUM_DEMAND_13_MONTHS][2].occurred == \
+            pytz.timezone("Europe/Brussels").localize(datetime.datetime(2020, 2, 10, 3, 54, 21))
 
         # CURRENT_ELECTRICITY_USAGE (1-0:1.7.0)
         assert isinstance(result[obis.CURRENT_ELECTRICITY_USAGE], CosemObject)