Merge branch 'master' into master

.github/workflows/tests.yml

@@ -0,0 +1,46 @@
+name: Tests
+
+on:
+  push: ~
+  pull_request: ~
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10  # Don't run forever when stale
+
+    strategy:
+      matrix:
+        python-version:
+          - '3.6'
+          - '3.7'
+          - '3.8'
+          - '3.9'
+          - '3.10'
+
+    name: Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Cached PIP dependencies
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.tox/python/.pytest_cache
+          key: pip-${{ matrix.python-version }}-${{ hashFiles('setup.py', 'tox.ini') }}
+          restore-keys: pip-${{ matrix.python-version }}-
+
+      - name: Install dependencies
+        run: pip install tox
+
+      - name: Run tests
+        run: tox
+
+      - name: Code coverage upload
+        uses: codecov/codecov-action@v1

.travis.yml

@@ -1,18 +0,0 @@
-language: python
-
-python:
-  - 2.7
-  - 3.5
-  - 3.6
-  - 3.8
-  
-install: pip install tox-travis codecov
-
-script: tox
-
-after_success:
-  - codecov
-
-matrix:
-  allow_failures:
-    - python: 2.7

CHANGELOG.rst

@@ -1,7 +1,46 @@
 Change Log
 ----------
 
+**0.32** (2022-01-04)
+
+- Support DSMR data read via RFXtrx with integrated P1 reader (`pull request #98 <https://github.com/ndokter/dsmr_parser/pull/98>`_).
+
+**0.31** (2021-11-21)
+
+- Support for (German) EasyMeter Q3D using COM-1 Ethernet Gateway (`pull request #92 <https://github.com/ndokter/dsmr_parser/pull/92>`_).
+
+**0.30** (2021-08-18)
+
+- Add support for Swedish smart meters (`pull request #86 <https://github.com/ndokter/dsmr_parser/pull/86>`_).
+
+**0.29** (2021-04-18)
+
+- Add value and unit properties to ProfileGenericObject to make sure that code like iterators that rely on that do not break (`pull request #71 <https://github.com/ndokter/dsmr_parser/pull/71>`_).
+Remove deprecated asyncio coroutine decorator (`pull request #76 <https://github.com/ndokter/dsmr_parser/pull/76>`_).
+
+**0.28** (2021-02-21)
+
+- Optional keep alive monitoring for TCP/IP connections (`pull request #73 <https://github.com/ndokter/dsmr_parser/pull/73>`_).
+- Catch parse errors in TelegramParser, ignore lines that can not be parsed (`pull request #74 <https://github.com/ndokter/dsmr_parser/pull/74>`_).
+
+**0.27** (2020-12-24)
+
+- fix for empty parentheses in ProfileGenericParser (redone) (`pull request #69 <https://github.com/ndokter/dsmr_parser/pull/69>`_).
+
+**0.26** (2020-12-15)
+
+- reverted fix for empty parentheses in ProfileGenericParser (`pull request #68 <https://github.com/ndokter/dsmr_parser/pull/68>`_).
+
+**0.25** (2020-12-14)
+
+- fix for empty parentheses in ProfileGenericParser (`pull request #57 <https://github.com/ndokter/dsmr_parser/pull/57>`_).
+
+**0.24** (2020-11-27)
+
+- Add Luxembourg equipment identifier (`pull request #62 <https://github.com/ndokter/dsmr_parser/pull/62>`_).
+
 **0.23** (2020-11-07)
+
 - Resolved issue with x-x:24.3.0 where it contains non-integer character (`pull request #61 <https://github.com/ndokter/dsmr_parser/pull/61>`_).
 - Tests are not installed anymore (`pull request #59 <https://github.com/ndokter/dsmr_parser/pull/59>`_).
 - Example telegram improvement (`pull request #58 <https://github.com/ndokter/dsmr_parser/pull/58>`_).

README.rst

@@ -4,8 +4,8 @@ DSMR Parser
 .. image:: https://img.shields.io/pypi/v/dsmr-parser.svg
     :target: https://pypi.python.org/pypi/dsmr-parser
 
-.. image:: https://travis-ci.org/ndokter/dsmr_parser.svg?branch=master
+.. image:: https://img.shields.io/github/workflow/status/ndokter/dsmr_parser/Tests/master
-    :target: https://travis-ci.org/ndokter/dsmr_parser
+    :target: https://github.com/ndokter/dsmr_parser/actions/workflows/tests.yml
 
 A library for parsing Dutch Smart Meter Requirements (DSMR) telegram data. It
 also includes client implementation to directly read and parse smart meter data.
@@ -14,7 +14,7 @@ also includes client implementation to directly read and parse smart meter data.
 Features
 --------
 
-DSMR Parser supports DSMR versions 2, 3, 4 and 5. It has been tested with Python 3.4, 3.5 and 3.6.
+DSMR Parser supports DSMR versions 2, 3, 4 and 5. See for the `currently supported/tested Python versions here <https://github.com/ndokter/dsmr_parser/blob/master/.github/workflows/tests.yml#L14>`_.
 
 
 Client module usage
@@ -39,12 +39,30 @@ process because the code is blocking (not asynchronous):
      for telegram in serial_reader.read():
          print(telegram)  # see 'Telegram object' docs below
 
+**Socket client**
+
+Read a remote serial port (for example using ser2net) and work with the parsed telegrams.
+It should be run in a separate process because the code is blocking (not asynchronous):
+
+.. code-block:: python
+
+     from dsmr_parser import telegram_specifications
+     from dsmr_parser.clients import SocketReader
+    
+     socket_reader = SocketReader(
+         host='127.0.0.1',
+         port=2001,
+         telegram_specification=telegram_specifications.V4
+     )
+    
+     for telegram in socket_reader.read():
+         print(telegram)  # see 'Telegram object' docs below
+
 **AsyncIO client**
 
 For a test run using a tcp server (lasting 20 seconds) use the following example:
 
 .. code-block:: python
-
     import asyncio
     import logging
     from dsmr_parser import obis_references
@@ -91,7 +109,6 @@ Moreover, the telegram passed to `telegram_callback(telegram)` is already parsed
 However, if we construct a mock TelegramParser that just returns the already parsed object we can work around this. An example is below:
 
 .. code-block:: python
-
     import asyncio
     import logging
     #from dsmr_parser import obis_references
@@ -151,12 +168,12 @@ However, if we construct a mock TelegramParser that just returns the already par
         except Exception as e:
             logger.error("Unexpected error: "+ e)
 
-    Parsing module usage
+Parsing module usage
-    --------------------
+--------------------
-    The parsing module accepts complete unaltered telegram strings and parses these
+The parsing module accepts complete unaltered telegram strings and parses these
-    into a dictionary.
+into a dictionary.
 
-    .. code-block:: python
+.. code-block:: python
 
     from dsmr_parser import telegram_specifications
     from dsmr_parser.parsers import TelegramParser
@@ -243,7 +260,7 @@ Example to get some of the values:
      gas_reading = telegram[obis_references.HOURLY_GAS_METER_READING]
 
     # See dsmr_reader.obis_references for all readable telegram values.
-    # Note that the avilable values differ per DSMR version.
+    # Note that the available values differ per DSMR version.
 
 Telegram as an Object
 ---------------------

dsmr_parser/__main__.py

@@ -16,8 +16,8 @@ def console():
                         help='alternatively connect using TCP host.')
     parser.add_argument('--port', default=None,
                         help='TCP port to use for connection')
-    parser.add_argument('--version', default='2.2', choices=['2.2', '4'],
+    parser.add_argument('--version', default='2.2', choices=['2.2', '4', '5', '5B', '5L', '5S', 'Q3D'],
-                        help='DSMR version (2.2, 4)')
+                        help='DSMR version (2.2, 4, 5, 5B, 5L, 5S, Q3D)')
     parser.add_argument('--verbose', '-v', action='count')
 
     args = parser.parse_args()

dsmr_parser/clients/__init__.py

@@ -1,5 +1,6 @@
 from dsmr_parser.clients.settings import SERIAL_SETTINGS_V2_2, \
     SERIAL_SETTINGS_V4, SERIAL_SETTINGS_V5
 from dsmr_parser.clients.serial_ import SerialReader, AsyncSerialReader
+from dsmr_parser.clients.socket_ import SocketReader
 from dsmr_parser.clients.protocol import create_dsmr_protocol, \
     create_dsmr_reader, create_tcp_dsmr_reader

dsmr_parser/clients/filereader.py

@@ -64,8 +64,11 @@ class FileReader(object):
         with open(self._file, "rb") as file_handle:
             while True:
                 data = file_handle.readline()
-                str = data.decode()
+
-                self.telegram_buffer.append(str)
+                if not data:
+                    break
+
+                self.telegram_buffer.append(data.decode())
 
                 for telegram in self.telegram_buffer.get_all():
                     try:

dsmr_parser/clients/protocol.py

@@ -14,7 +14,14 @@ from dsmr_parser.clients.settings import SERIAL_SETTINGS_V2_2, \
     SERIAL_SETTINGS_V4, SERIAL_SETTINGS_V5
 
 
-def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None):
+def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None, **kwargs):
+    """Creates a DSMR asyncio protocol."""
+    protocol = _create_dsmr_protocol(dsmr_version, telegram_callback,
+                                     DSMRProtocol, loop, **kwargs)
+    return protocol
+
+
+def _create_dsmr_protocol(dsmr_version, telegram_callback, protocol, loop=None, **kwargs):
     """Creates a DSMR asyncio protocol."""
 
     if dsmr_version == '2.2':
@@ -23,6 +30,9 @@ def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None):
     elif dsmr_version == '4':
         specification = telegram_specifications.V4
         serial_settings = SERIAL_SETTINGS_V4
+    elif dsmr_version == '4+':
+        specification = telegram_specifications.V5
+        serial_settings = SERIAL_SETTINGS_V4
     elif dsmr_version == '5':
         specification = telegram_specifications.V5
         serial_settings = SERIAL_SETTINGS_V5
@@ -32,12 +42,18 @@ def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None):
     elif dsmr_version == "5L":
         specification = telegram_specifications.LUXEMBOURG_SMARTY
         serial_settings = SERIAL_SETTINGS_V5
+    elif dsmr_version == "5S":
+        specification = telegram_specifications.SWEDEN
+        serial_settings = SERIAL_SETTINGS_V5
+    elif dsmr_version == "Q3D":
+        specification = telegram_specifications.Q3D
+        serial_settings = SERIAL_SETTINGS_V5
     else:
         raise NotImplementedError("No telegram parser found for version: %s",
                                   dsmr_version)
 
-    protocol = partial(DSMRProtocol, loop, TelegramParser(specification),
+    protocol = partial(protocol, loop, TelegramParser(specification),
-                       telegram_callback=telegram_callback)
+                       telegram_callback=telegram_callback, **kwargs)
 
     return protocol, serial_settings
 
@@ -53,12 +69,14 @@ def create_dsmr_reader(port, dsmr_version, telegram_callback, loop=None):
 
 
 def create_tcp_dsmr_reader(host, port, dsmr_version,
-                           telegram_callback, loop=None):
+                           telegram_callback, loop=None,
+                           keep_alive_interval=None):
     """Creates a DSMR asyncio protocol coroutine using TCP connection."""
     if not loop:
         loop = asyncio.get_event_loop()
     protocol, _ = create_dsmr_protocol(
-        dsmr_version, telegram_callback, loop=loop)
+        dsmr_version, telegram_callback, loop=loop,
+        keep_alive_interval=keep_alive_interval)
     conn = loop.create_connection(protocol, host, port)
     return conn
 
@@ -69,7 +87,8 @@ class DSMRProtocol(asyncio.Protocol):
     transport = None
     telegram_callback = None
 
-    def __init__(self, loop, telegram_parser, telegram_callback=None):
+    def __init__(self, loop, telegram_parser,
+                 telegram_callback=None, keep_alive_interval=None):
         """Initialize class."""
         self.loop = loop
         self.log = logging.getLogger(__name__)
@@ -80,21 +99,42 @@ class DSMRProtocol(asyncio.Protocol):
         self.telegram_buffer = TelegramBuffer()
         # keep a lock until the connection is closed
         self._closed = asyncio.Event()
+        self._keep_alive_interval = keep_alive_interval
+        self._active = True
 
     def connection_made(self, transport):
         """Just logging for now."""
         self.transport = transport
         self.log.debug('connected')
+        self._active = False
+        if self.loop and self._keep_alive_interval:
+            self.loop.call_later(self._keep_alive_interval, self.keep_alive)
 
     def data_received(self, data):
         """Add incoming data to buffer."""
-        data = data.decode('ascii')
+
+        # accept latin-1 (8-bit) on the line, to allow for non-ascii transport or padding
+        data = data.decode("latin1")
+        self._active = True
         self.log.debug('received data: %s', data)
         self.telegram_buffer.append(data)
 
         for telegram in self.telegram_buffer.get_all():
+            # ensure actual telegram is ascii (7-bit) only (ISO 646:1991 IRV required in section 5.5 of IEC 62056-21)
+            telegram = telegram.encode("latin1").decode("ascii")
             self.handle_telegram(telegram)
 
+    def keep_alive(self):
+        if self._active:
+            self.log.debug('keep-alive checked')
+            self._active = False
+            if self.loop:
+                self.loop.call_later(self._keep_alive_interval, self.keep_alive)
+        else:
+            self.log.warning('keep-alive check failed')
+            if self.transport:
+                self.transport.close()
+
     def connection_lost(self, exc):
         """Stop when connection is lost."""
         if exc:
@@ -116,7 +156,6 @@ class DSMRProtocol(asyncio.Protocol):
         else:
             self.telegram_callback(parsed_telegram)
 
-    @asyncio.coroutine
+    async def wait_closed(self):
-    def wait_closed(self):
         """Wait until connection is closed."""
-        yield from self._closed.wait()
+        await self._closed.wait()

dsmr_parser/clients/rfxtrx_protocol.py

@@ -0,0 +1,62 @@
+"""Asyncio protocol implementation for handling telegrams over a RFXtrx connection ."""
+
+import asyncio
+
+from serial_asyncio import create_serial_connection
+from .protocol import DSMRProtocol, _create_dsmr_protocol
+
+
+def create_rfxtrx_dsmr_protocol(dsmr_version, telegram_callback, loop=None, **kwargs):
+    """Creates a RFXtrxDSMR asyncio protocol."""
+    protocol = _create_dsmr_protocol(dsmr_version, telegram_callback,
+                                     RFXtrxDSMRProtocol, loop, **kwargs)
+    return protocol
+
+
+def create_rfxtrx_dsmr_reader(port, dsmr_version, telegram_callback, loop=None):
+    """Creates a DSMR asyncio protocol coroutine using a RFXtrx serial port."""
+    protocol, serial_settings = create_rfxtrx_dsmr_protocol(
+        dsmr_version, telegram_callback, loop=None)
+    serial_settings['url'] = port
+
+    conn = create_serial_connection(loop, protocol, **serial_settings)
+    return conn
+
+
+def create_rfxtrx_tcp_dsmr_reader(host, port, dsmr_version,
+                                  telegram_callback, loop=None,
+                                  keep_alive_interval=None):
+    """Creates a DSMR asyncio protocol coroutine using a RFXtrx TCP connection."""
+    if not loop:
+        loop = asyncio.get_event_loop()
+    protocol, _ = create_rfxtrx_dsmr_protocol(
+        dsmr_version, telegram_callback, loop=loop,
+        keep_alive_interval=keep_alive_interval)
+    conn = loop.create_connection(protocol, host, port)
+    return conn
+
+
+PACKETTYPE_DSMR = 0x62
+SUBTYPE_P1 = 0x01
+
+
+class RFXtrxDSMRProtocol(DSMRProtocol):
+
+    remaining_data = b''
+
+    def data_received(self, data):
+        """Add incoming data to buffer."""
+
+        data = self.remaining_data + data
+
+        packetlength = data[0] + 1 if len(data) > 0 else 1
+        while packetlength <= len(data):
+            packettype = data[1]
+            subtype = data[2]
+            if (packettype == PACKETTYPE_DSMR and subtype == SUBTYPE_P1):
+                dsmr_data = data[4:packetlength]
+                super().data_received(dsmr_data)
+            data = data[packetlength:]
+            packetlength = data[0] + 1 if len(data) > 0 else 1
+
+        self.remaining_data = data

dsmr_parser/clients/serial_.py

@@ -1,4 +1,3 @@
-import asyncio
 import logging
 import serial
 import serial_asyncio
@@ -68,8 +67,7 @@ class AsyncSerialReader(SerialReader):
 
     PORT_KEY = 'url'
 
-    @asyncio.coroutine
+    async def read(self, queue):
-    def read(self, queue):
         """
         Read complete DSMR telegram's from the serial interface and parse it
         into CosemObject's and MbusObject's.
@@ -81,12 +79,12 @@ class AsyncSerialReader(SerialReader):
         """
         # create Serial StreamReader
         conn = serial_asyncio.open_serial_connection(**self.serial_settings)
-        reader, _ = yield from conn
+        reader, _ = await conn
 
         while True:
             # Read line if available or give control back to loop until new
             # data has arrived.
-            data = yield from reader.readline()
+            data = await reader.readline()
             self.telegram_buffer.append(data.decode('ascii'))
 
             for telegram in self.telegram_buffer.get_all():

dsmr_parser/clients/socket_.py

@@ -0,0 +1,90 @@
+import logging
+import socket
+
+from dsmr_parser.clients.telegram_buffer import TelegramBuffer
+from dsmr_parser.exceptions import ParseError, InvalidChecksumError
+from dsmr_parser.parsers import TelegramParser
+from dsmr_parser.objects import Telegram
+
+
+logger = logging.getLogger(__name__)
+
+
+class SocketReader(object):
+
+    BUFFER_SIZE = 256
+
+    def __init__(self, host, port, telegram_specification):
+        self.host = host
+        self.port = port
+
+        self.telegram_parser = TelegramParser(telegram_specification)
+        self.telegram_buffer = TelegramBuffer()
+        self.telegram_specification = telegram_specification
+
+    def read(self):
+        """
+        Read complete DSMR telegram's from remote interface and parse it
+        into CosemObject's and MbusObject's
+
+        :rtype: generator
+        """
+        buffer = b""
+
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as socket_handle:
+
+            socket_handle.connect((self.host, self.port))
+
+            while True:
+                buffer += socket_handle.recv(self.BUFFER_SIZE)
+
+                lines = buffer.splitlines(keepends=True)
+
+                if len(lines) == 0:
+                    continue
+
+                for data in lines:
+                    self.telegram_buffer.append(data.decode('ascii'))
+
+                for telegram in self.telegram_buffer.get_all():
+                    try:
+                        yield self.telegram_parser.parse(telegram)
+                    except InvalidChecksumError as e:
+                        logger.warning(str(e))
+                    except ParseError as e:
+                        logger.error('Failed to parse telegram: %s', e)
+
+                buffer = b""
+
+    def read_as_object(self):
+        """
+        Read complete DSMR telegram's from remote and return a Telegram object.
+
+        :rtype: generator
+        """
+        buffer = b""
+
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as socket_handle:
+
+            socket_handle.connect((self.host, self.port))
+
+            while True:
+                buffer += socket_handle.recv(self.BUFFER_SIZE)
+
+                lines = buffer.splitlines(keepends=True)
+
+                if len(lines) == 0:
+                    continue
+
+                for data in lines:
+                    self.telegram_buffer.append(data.decode('ascii'))
+
+                    for telegram in self.telegram_buffer.get_all():
+                        try:
+                            yield Telegram(telegram, self.telegram_parser, self.telegram_specification)
+                        except InvalidChecksumError as e:
+                            logger.warning(str(e))
+                        except ParseError as e:
+                            logger.error('Failed to parse telegram: %s', e)
+
+                buffer = b""

dsmr_parser/obis_name_mapping.py

@@ -13,6 +13,7 @@ EN = {
     obis.ELECTRICITY_IMPORTED_TOTAL: 'ELECTRICITY_IMPORTED_TOTAL',
     obis.ELECTRICITY_USED_TARIFF_1: 'ELECTRICITY_USED_TARIFF_1',
     obis.ELECTRICITY_USED_TARIFF_2: 'ELECTRICITY_USED_TARIFF_2',
+    obis.ELECTRICITY_EXPORTED_TOTAL: 'ELECTRICITY_EXPORTED_TOTAL',
     obis.ELECTRICITY_DELIVERED_TARIFF_1: 'ELECTRICITY_DELIVERED_TARIFF_1',
     obis.ELECTRICITY_DELIVERED_TARIFF_2: 'ELECTRICITY_DELIVERED_TARIFF_2',
     obis.ELECTRICITY_ACTIVE_TARIFF: 'ELECTRICITY_ACTIVE_TARIFF',
@@ -48,7 +49,14 @@ EN = {
     obis.GAS_METER_READING: 'GAS_METER_READING',
     obis.ACTUAL_TRESHOLD_ELECTRICITY: 'ACTUAL_TRESHOLD_ELECTRICITY',
     obis.ACTUAL_SWITCH_POSITION: 'ACTUAL_SWITCH_POSITION',
-    obis.VALVE_POSITION_GAS: 'VALVE_POSITION_GAS'
+    obis.VALVE_POSITION_GAS: 'VALVE_POSITION_GAS',
+    obis.BELGIUM_5MIN_GAS_METER_READING: 'BELGIUM_5MIN_GAS_METER_READING',
+    obis.BELGIUM_MAX_POWER_PER_PHASE: 'BELGIUM_MAX_POWER_PER_PHASE',
+    obis.BELGIUM_MAX_CURRENT_PER_PHASE: 'BELGIUM_MAX_CURRENT_PER_PHASE',
+    obis.LUXEMBOURG_EQUIPMENT_IDENTIFIER: 'LUXEMBOURG_EQUIPMENT_IDENTIFIER',
+    obis.Q3D_EQUIPMENT_IDENTIFIER: 'Q3D_EQUIPMENT_IDENTIFIER',
+    obis.Q3D_EQUIPMENT_STATE: 'Q3D_EQUIPMENT_STATE',
+    obis.Q3D_EQUIPMENT_SERIALNUMBER: 'Q3D_EQUIPMENT_SERIALNUMBER',
 }
 
 REVERSE_EN = dict([(v, k) for k, v in EN.items()])

dsmr_parser/obis_references.py

@@ -8,7 +8,6 @@ objects are introduced.
 """
 P1_MESSAGE_HEADER = r'\d-\d:0\.2\.8.+?\r\n'
 P1_MESSAGE_TIMESTAMP = r'\d-\d:1\.0\.0.+?\r\n'
-ELECTRICITY_IMPORTED_TOTAL = r'\d-\d:1\.8\.0.+?\r\n'
 ELECTRICITY_USED_TARIFF_1 = r'\d-\d:1\.8\.1.+?\r\n'
 ELECTRICITY_USED_TARIFF_2 = r'\d-\d:1\.8\.2.+?\r\n'
 ELECTRICITY_DELIVERED_TARIFF_1 = r'\d-\d:2\.8\.1.+?\r\n'
@@ -61,8 +60,15 @@ ELECTRICITY_DELIVERED_TARIFF_ALL = (
     ELECTRICITY_DELIVERED_TARIFF_2
 )
 
-# Alternate codes for foreign countries.
+# International generalized additions
-BELGIUM_HOURLY_GAS_METER_READING = r'\d-\d:24\.2\.3.+?\r\n'  # Different code, same format.
+ELECTRICITY_IMPORTED_TOTAL = r'\d-\d:1\.8\.0.+?\r\n'  # Total imported energy register (P+)
+ELECTRICITY_EXPORTED_TOTAL = r'\d-\d:2\.8\.0.+?\r\n'  # Total exported energy register (P-)
+
+# International non generalized additions (country specific) / risk for necessary refactoring
+BELGIUM_5MIN_GAS_METER_READING = r'\d-\d:24\.2\.3.+?\r\n'  # Different code, same format.
+BELGIUM_MAX_POWER_PER_PHASE = r'\d-\d:17\.0\.0.+?\r\n'  # Applicable when power limitation is active
+BELGIUM_MAX_CURRENT_PER_PHASE = r'\d-\d:31\.4\.0.+?\r\n'  # Applicable when current limitation is active
 LUXEMBOURG_EQUIPMENT_IDENTIFIER = r'\d-\d:42\.0\.0.+?\r\n'  # Logical device name
-LUXEMBOURG_ELECTRICITY_USED_TARIFF_GLOBAL = r'\d-\d:1\.8\.0.+?\r\n'  # Total imported energy register (P+)
+Q3D_EQUIPMENT_IDENTIFIER = r'\d-\d:0\.0\.0.+?\r\n'  # Logical device name
-LUXEMBOURG_ELECTRICITY_DELIVERED_TARIFF_GLOBAL = r'\d-\d:2\.8\.0.+?\r\n'  # Total exported energy register (P-)
+Q3D_EQUIPMENT_STATE = r'\d-\d:96\.5\.5.+?\r\n'  # Device state (hexadecimal)
+Q3D_EQUIPMENT_SERIALNUMBER = r'\d-\d:96\.1\.255.+?\r\n'  # Device Serialnumber

dsmr_parser/objects.py

@@ -155,6 +155,16 @@ class ProfileGenericObject(DSMRObject):
         super().__init__(values)
         self._buffer_list = None
 
+    @property
+    def value(self):
+        # value is added to make sure the telegram iterator does not break
+        return self.values
+
+    @property
+    def unit(self):
+        # value is added to make sure all items have a unit so code that relies on that does not break
+        return None
+
     @property
     def buffer_length(self):
         return self.values[0]['value']
@@ -169,7 +179,7 @@ class ProfileGenericObject(DSMRObject):
             self._buffer_list = []
             values_offset = 2
             for i in range(self.buffer_length):
-                offset = values_offset + i*2
+                offset = values_offset + i * 2
                 self._buffer_list.append(MBusObject([self.values[offset], self.values[offset + 1]]))
         return self._buffer_list
 

dsmr_parser/parsers.py

@@ -10,7 +10,6 @@ logger = logging.getLogger(__name__)
 
 
 class TelegramParser(object):
-
     crc16_tab = []
 
     def __init__(self, telegram_specification, apply_checksum_validation=True):
@@ -56,7 +55,11 @@ class TelegramParser(object):
             # Some signatures are optional and may not be present,
             # so only parse lines that match
             if match:
+                try:
                     telegram[signature] = parser.parse(match.group(0))
+                except Exception:
+                    logger.error("ignore line with signature {}, because parsing failed.".format(signature),
+                                 exc_info=True)
 
         return telegram
 
@@ -219,12 +222,17 @@ class ProfileGenericParser(DSMRObjectParser):
     8) Buffer value 2 (oldest entry of buffer attribute without unit)
     9) Unit of buffer values (Unit of capture objects attribute)
     """
+
     def __init__(self, buffer_types, head_parsers, parsers_for_unidentified):
         self.value_formats = head_parsers
         self.buffer_types = buffer_types
         self.parsers_for_unidentified = parsers_for_unidentified
 
     def _is_line_wellformed(self, line, values):
+        if values and (len(values) == 1) and (values[0] == ''):
+            # special case: single empty parentheses (indicated by empty string)
+            return True
+
         if values and (len(values) >= 2) and (values[0].isdigit()):
             buffer_length = int(values[0])
             return (buffer_length <= 10) and (len(values) == (buffer_length * 2 + 2))
@@ -232,6 +240,9 @@ class ProfileGenericParser(DSMRObjectParser):
             return False
 
     def _parse_values(self, values):
+        if values and (len(values) == 1) and (values[0] is None):
+            # special case: single empty parentheses; make sure empty ProfileGenericObject is created
+            values = [0, None]  # buffer_length=0, buffer_value_obis_ID=None
         buffer_length = int(values[0])
         buffer_value_obis_ID = values[1]
         if (buffer_length > 0):
@@ -264,7 +275,6 @@ class ValueParser(object):
         self.coerce_type = coerce_type
 
     def parse(self, value):
-
         unit_of_measurement = None
 
         if value and '*' in value:

dsmr_parser/profile_generic_specifications.py

@@ -7,4 +7,4 @@ PG_HEAD_PARSERS = [ValueParser(int), ValueParser(str)]
 PG_UNIDENTIFIED_BUFFERTYPE_PARSERS = [ValueParser(str), ValueParser(str)]
 BUFFER_TYPES = {
     PG_FAILURE_EVENT:  [ValueParser(timestamp), ValueParser(int)]
-    }
+}

dsmr_parser/telegram_specifications.py

@@ -144,15 +144,60 @@ ALL = (V2_2, V3, V4, V5)
 
 BELGIUM_FLUVIUS = deepcopy(V5)
 BELGIUM_FLUVIUS['objects'].update({
-    obis.BELGIUM_HOURLY_GAS_METER_READING: MBusParser(
+    obis.BELGIUM_5MIN_GAS_METER_READING: MBusParser(
         ValueParser(timestamp),
         ValueParser(Decimal)
-    )
+    ),
+    obis.BELGIUM_MAX_POWER_PER_PHASE: CosemParser(ValueParser(Decimal)),
+    obis.BELGIUM_MAX_CURRENT_PER_PHASE: CosemParser(ValueParser(Decimal)),
+    obis.ACTUAL_SWITCH_POSITION: CosemParser(ValueParser(str)),
+    obis.VALVE_POSITION_GAS: CosemParser(ValueParser(str)),
 })
 
 LUXEMBOURG_SMARTY = deepcopy(V5)
 LUXEMBOURG_SMARTY['objects'].update({
     obis.LUXEMBOURG_EQUIPMENT_IDENTIFIER: CosemParser(ValueParser(str)),
-    obis.LUXEMBOURG_ELECTRICITY_USED_TARIFF_GLOBAL: CosemParser(ValueParser(Decimal)),
+    obis.ELECTRICITY_IMPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
-    obis.LUXEMBOURG_ELECTRICITY_DELIVERED_TARIFF_GLOBAL: CosemParser(ValueParser(Decimal)),
+    obis.ELECTRICITY_EXPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
 })
+
+# Source: https://www.energiforetagen.se/globalassets/energiforetagen/det-erbjuder-vi/kurser-och-konferenser/elnat/
+#         branschrekommendation-lokalt-granssnitt-v2_0-201912.pdf
+SWEDEN = {
+    'checksum_support': True,
+    'objects': {
+        obis.P1_MESSAGE_HEADER: CosemParser(ValueParser(str)),
+        obis.P1_MESSAGE_TIMESTAMP: CosemParser(ValueParser(timestamp)),
+        obis.ELECTRICITY_IMPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
+        obis.ELECTRICITY_EXPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
+        obis.CURRENT_ELECTRICITY_USAGE: CosemParser(ValueParser(Decimal)),
+        obis.CURRENT_ELECTRICITY_DELIVERY: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L1_POSITIVE: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L2_POSITIVE: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L3_POSITIVE: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L1_NEGATIVE: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L2_NEGATIVE: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L3_NEGATIVE: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_VOLTAGE_L1: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_VOLTAGE_L2: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_VOLTAGE_L3: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_CURRENT_L1: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_CURRENT_L2: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_CURRENT_L3: CosemParser(ValueParser(Decimal)),
+    }
+}
+
+Q3D = {
+    "checksum_support": False,
+    "objects": {
+        obis.Q3D_EQUIPMENT_IDENTIFIER: CosemParser(ValueParser(str)),
+        obis.ELECTRICITY_IMPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
+        obis.ELECTRICITY_EXPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L1_POSITIVE: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L2_POSITIVE: CosemParser(ValueParser(Decimal)),
+        obis.INSTANTANEOUS_ACTIVE_POWER_L3_POSITIVE: CosemParser(ValueParser(Decimal)),
+        obis.CURRENT_ELECTRICITY_USAGE: CosemParser(ValueParser(Decimal)),
+        obis.Q3D_EQUIPMENT_STATE: CosemParser(ValueParser(str)),
+        obis.Q3D_EQUIPMENT_SERIALNUMBER: CosemParser(ValueParser(str)),
+    },
+}

setup.py

@@ -3,10 +3,11 @@ from setuptools import setup, find_packages
 setup(
     name='dsmr-parser',
     description='Library to parse Dutch Smart Meter Requirements (DSMR)',
-    author='Nigel Dokter',
+    author='Nigel Dokter and many others',
     author_email='nigel@nldr.net',
+    license='MIT',
     url='https://github.com/ndokter/dsmr_parser',
-    version='0.23',
+    version='0.32',
     packages=find_packages(exclude=('test', 'test.*')),
     install_requires=[
         'pyserial>=3,<4',

test/example_telegrams.py

@@ -128,3 +128,44 @@ TELEGRAM_V5 = (
     '0-2:96.1.0()\r\n'
     '!6EEE\r\n'
 )
+
+# EasyMeter via COM-1 Ethernet Gateway
+# Q3D Manual (german) https://www.easymeter.com/downloads/products/zaehler/Q3D/Easymeter_Q3D_DE_2016-06-15.pdf
+#  - type code on page 8
+#  - D0-Specs on page 20
+#
+# last two lines are added by the COM-1 Ethernet Gateway
+
+TELEGRAM_ESY5Q3DB1024_V304 = (
+    '/ESY5Q3DB1024 V3.04\r\n'
+    '\r\n'
+    '1-0:0.0.0*255(0272031312565)\r\n'
+    '1-0:1.8.0*255(00052185.7825309*kWh)\r\n'
+    '1-0:2.8.0*255(00019949.3221493*kWh)\r\n'
+    '1-0:21.7.0*255(000747.85*W)\r\n'
+    '1-0:41.7.0*255(000737.28*W)\r\n'
+    '1-0:61.7.0*255(000639.73*W)\r\n'
+    '1-0:1.7.0*255(002124.86*W)\r\n'
+    '1-0:96.5.5*255(80)\r\n'
+    '0-0:96.1.255*255(1ESY1313002565)\r\n'
+    '!\r\n'
+    '  25803103\r\n'
+    '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff'
+    '\xff\xff\xff\xff\xff\r\n'
+)
+
+TELEGRAM_ESY5Q3DA1004_V304 = (
+    '/ESY5Q3DA1004 V3.04\r\n'
+    '\r\n'
+    '1-0:0.0.0*255(1336001560)\r\n'
+    '1-0:1.8.0*255(00032549.5061662*kWh)\r\n'
+    '1-0:21.7.0*255(000557.29*W)\r\n'
+    '1-0:41.7.0*255(000521.62*W)\r\n'
+    '1-0:61.7.0*255(000609.30*W)\r\n'
+    '1-0:1.7.0*255(001688.21*W)\r\n'
+    '1-0:96.5.5*255(80)\r\n'
+    '0-0:96.1.255*255(1ESY1336001560)\r\n'
+    '!\r\n'
+    '  25818685\r\n'
+    'DE0000000000000000000000000000003\r\n'
+)

test/test_filereader.py

@@ -0,0 +1,21 @@
+import unittest
+import tempfile
+
+from dsmr_parser.clients.filereader import FileReader
+from dsmr_parser.telegram_specifications import V5
+from test.example_telegrams import TELEGRAM_V5
+
+
+class FileReaderTest(unittest.TestCase):
+    def test_read_as_object(self):
+        with tempfile.NamedTemporaryFile() as file:
+            with open(file.name, "w") as f:
+                f.write(TELEGRAM_V5)
+
+            telegrams = []
+            reader = FileReader(file=file.name, telegram_specification=V5)
+            # Call
+            for telegram in reader.read_as_object():
+                telegrams.append(telegram)
+
+            self.assertEqual(len(telegrams), 1)

test/test_parse_v5.py

@@ -241,7 +241,6 @@ class TelegramParserV5Test(unittest.TestCase):
 
     def test_checksum_missing(self):
         # Remove the checksum value causing a ParseError.
-        corrupted_telegram = TELEGRAM_V5.replace('!87B3\r\n', '')
+        corrupted_telegram = TELEGRAM_V5.replace('!6EEE\r\n', '')
-
         with self.assertRaises(ParseError):
             TelegramParser.validate_checksum(corrupted_telegram)

test/test_parser_corner_cases.py

@@ -0,0 +1,89 @@
+import unittest
+
+from dsmr_parser import telegram_specifications
+
+from dsmr_parser.objects import Telegram
+from dsmr_parser.objects import ProfileGenericObject
+from dsmr_parser.parsers import TelegramParser
+from dsmr_parser.parsers import ProfileGenericParser
+from dsmr_parser.profile_generic_specifications import BUFFER_TYPES
+from dsmr_parser.profile_generic_specifications import PG_HEAD_PARSERS
+from dsmr_parser.profile_generic_specifications import PG_UNIDENTIFIED_BUFFERTYPE_PARSERS
+from test.example_telegrams import TELEGRAM_V5
+
+
+class TestParserCornerCases(unittest.TestCase):
+    """ Test instantiation of Telegram object """
+
+    def test_power_event_log_empty_1(self):
+        # POWER_EVENT_FAILURE_LOG (1-0:99.97.0)
+        parser = TelegramParser(telegram_specifications.V5)
+        telegram = Telegram(TELEGRAM_V5, parser, telegram_specifications.V5)
+
+        object_type = ProfileGenericObject
+        testitem = telegram.POWER_EVENT_FAILURE_LOG
+        assert isinstance(testitem, object_type)
+        assert testitem.buffer_length == 0
+        assert testitem.buffer_type == '0-0:96.7.19'
+        buffer = testitem.buffer
+        assert isinstance(testitem.buffer, list)
+        assert len(buffer) == 0
+
+    def test_power_event_log_empty_2(self):
+        pef_parser = ProfileGenericParser(BUFFER_TYPES, PG_HEAD_PARSERS, PG_UNIDENTIFIED_BUFFERTYPE_PARSERS)
+        object_type = ProfileGenericObject
+
+        # Power Event Log with 0 items and no object type
+        pefl_line = r'1-0:99.97.0(0)()\r\n'
+        testitem = pef_parser.parse(pefl_line)
+
+        assert isinstance(testitem, object_type)
+        assert testitem.buffer_length == 0
+        assert testitem.buffer_type is None
+        buffer = testitem.buffer
+        assert isinstance(testitem.buffer, list)
+        assert len(buffer) == 0
+        assert testitem.values == [{'value': 0, 'unit': None}, {'value': None, 'unit': None}]
+        json = testitem.to_json()
+        assert json == '{"buffer_length": 0, "buffer_type": null, "buffer": []}'
+
+    def test_power_event_log_null_values(self):
+        pef_parser = ProfileGenericParser(BUFFER_TYPES, PG_HEAD_PARSERS, PG_UNIDENTIFIED_BUFFERTYPE_PARSERS)
+        object_type = ProfileGenericObject
+
+        # Power Event Log with 1 item and no object type and nno values for the item
+        pefl_line = r'1-0:99.97.0(1)()()()\r\n'
+        testitem = pef_parser.parse(pefl_line)
+
+        assert isinstance(testitem, object_type)
+        assert testitem.buffer_length == 1
+        assert testitem.buffer_type is None
+        buffer = testitem.buffer
+        assert isinstance(testitem.buffer, list)
+        assert len(buffer) == 1
+        assert testitem.values == [{'value': 1, 'unit': None}, {'value': None, 'unit': None},
+                                   {'value': None, 'unit': None}, {'value': None, 'unit': None}]
+        json = testitem.to_json()
+        assert json == \
+               '{"buffer_length": 1, "buffer_type": null, "buffer": [{"datetime": null, "value": null, "unit": null}]}'
+
+    def test_power_event_log_brackets_only(self):
+        # POWER_EVENT_FAILURE_LOG (1-0:99.97.0)
+        # Issue 57
+        # Test of an ill formatted empty POWER_EVENT_FAILURE_LOG, observed on some smartmeters
+        # The idea is that instead of failing, the parser converts it to an empty POWER_EVENT_FAILURE_LOG
+        pef_parser = ProfileGenericParser(BUFFER_TYPES, PG_HEAD_PARSERS, PG_UNIDENTIFIED_BUFFERTYPE_PARSERS)
+        object_type = ProfileGenericObject
+
+        pefl_line = r'1-0:99.97.0()\r\n'
+        testitem = pef_parser.parse(pefl_line)
+
+        assert isinstance(testitem, object_type)
+        assert testitem.buffer_length == 0
+        assert testitem.buffer_type is None
+        buffer = testitem.buffer
+        assert isinstance(testitem.buffer, list)
+        assert len(buffer) == 0
+        assert testitem.values == [{'value': 0, 'unit': None}, {'value': None, 'unit': None}]
+        json = testitem.to_json()
+        assert json == '{"buffer_length": 0, "buffer_type": null, "buffer": []}'

test/test_protocol.py

@@ -3,9 +3,7 @@ from unittest.mock import Mock
 import unittest
 
 from dsmr_parser import obis_references as obis
-from dsmr_parser import telegram_specifications
+from dsmr_parser.clients.protocol import create_dsmr_protocol
-from dsmr_parser.parsers import TelegramParser
-from dsmr_parser.clients.protocol import DSMRProtocol
 
 
 TELEGRAM_V2_2 = (
@@ -35,9 +33,10 @@ TELEGRAM_V2_2 = (
 class ProtocolTest(unittest.TestCase):
 
     def setUp(self):
-        telegram_parser = TelegramParser(telegram_specifications.V2_2)
+        new_protocol, _ = create_dsmr_protocol('2.2',
-        self.protocol = DSMRProtocol(None, telegram_parser,
+                                               telegram_callback=Mock(),
-                                     telegram_callback=Mock())
+                                               keep_alive_interval=1)
+        self.protocol = new_protocol()
 
     def test_complete_packet(self):
         """Protocol should assemble incoming lines into complete packet."""
@@ -52,3 +51,23 @@ class ProtocolTest(unittest.TestCase):
 
         assert float(telegram[obis.GAS_METER_READING].value) == 1.001
         assert telegram[obis.GAS_METER_READING].unit == 'm3'
+
+    def test_receive_packet(self):
+        """Protocol packet reception."""
+
+        mock_transport = Mock()
+        self.protocol.connection_made(mock_transport)
+        assert not self.protocol._active
+
+        self.protocol.data_received(TELEGRAM_V2_2.encode('ascii'))
+        assert self.protocol._active
+
+        # 1st call of keep_alive resets 'active' flag
+        self.protocol.keep_alive()
+        assert not self.protocol._active
+
+        # 2nd call of keep_alive should close the transport
+        self.protocol.keep_alive()
+        assert mock_transport.close.called_once()
+
+        self.protocol.connection_lost(None)

test/test_rfxtrx_protocol.py

@@ -0,0 +1,77 @@
+from unittest.mock import Mock
+
+import unittest
+
+from dsmr_parser import obis_references as obis
+from dsmr_parser.clients.rfxtrx_protocol import create_rfxtrx_dsmr_protocol, PACKETTYPE_DSMR, SUBTYPE_P1
+
+
+TELEGRAM_V2_2 = (
+    '/ISk5\2MT382-1004\r\n'
+    '\r\n'
+    '0-0:96.1.1(00000000000000)\r\n'
+    '1-0:1.8.1(00001.001*kWh)\r\n'
+    '1-0:1.8.2(00001.001*kWh)\r\n'
+    '1-0:2.8.1(00001.001*kWh)\r\n'
+    '1-0:2.8.2(00001.001*kWh)\r\n'
+    '0-0:96.14.0(0001)\r\n'
+    '1-0:1.7.0(0001.01*kW)\r\n'
+    '1-0:2.7.0(0000.00*kW)\r\n'
+    '0-0:17.0.0(0999.00*kW)\r\n'
+    '0-0:96.3.10(1)\r\n'
+    '0-0:96.13.1()\r\n'
+    '0-0:96.13.0()\r\n'
+    '0-1:24.1.0(3)\r\n'
+    '0-1:96.1.0(000000000000)\r\n'
+    '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n'
+    '(00001.001)\r\n'
+    '0-1:24.4.0(1)\r\n'
+    '!\r\n'
+)
+
+OTHER_RF_PACKET = b'\x03\x01\x02\x03'
+
+
+def encode_telegram_as_RF_packets(telegram):
+    data = b''
+
+    for line in telegram.split('\n'):
+        packet_data = (line + '\n').encode('ascii')
+        packet_header = bytes(bytearray([
+            len(packet_data) + 3,  # excluding length byte
+            PACKETTYPE_DSMR,
+            SUBTYPE_P1,
+            0  # seq num (ignored)
+        ]))
+
+        data += packet_header + packet_data
+        # other RF packets can pass by on the line
+        data += OTHER_RF_PACKET
+
+    return data
+
+
+class RFXtrxProtocolTest(unittest.TestCase):
+
+    def setUp(self):
+        new_protocol, _ = create_rfxtrx_dsmr_protocol('2.2',
+                                                      telegram_callback=Mock(),
+                                                      keep_alive_interval=1)
+        self.protocol = new_protocol()
+
+    def test_complete_packet(self):
+        """Protocol should assemble incoming lines into complete packet."""
+
+        data = encode_telegram_as_RF_packets(TELEGRAM_V2_2)
+        # send data broken up in two parts
+        self.protocol.data_received(data[0:200])
+        self.protocol.data_received(data[200:])
+
+        telegram = self.protocol.telegram_callback.call_args_list[0][0][0]
+        assert isinstance(telegram, dict)
+
+        assert float(telegram[obis.CURRENT_ELECTRICITY_USAGE].value) == 1.01
+        assert telegram[obis.CURRENT_ELECTRICITY_USAGE].unit == 'kW'
+
+        assert float(telegram[obis.GAS_METER_READING].value) == 1.001
+        assert telegram[obis.GAS_METER_READING].unit == 'm3'

tox.ini

@@ -1,13 +1,9 @@
-[tox]
-envlist = py35,py36,py37,py38
-
 [testenv]
 deps=
   pytest
   pytest-cov
   pylama
   pytest-asyncio
-  pytest-catchlog
   pytest-mock
 commands=
   py.test --cov=dsmr_parser test {posargs}