Merge branch 'master' of github.com:ndokter/dsmr_parser

README.rst

@@ -46,6 +46,25 @@ process because the code is blocking (not asynchronous):
 
 To be documented.
 
+**Socket client**
+
+Read a remote serial port (for example using ser2net) and work with the parsed telegrams.
+It should be run in a separate process because the code is blocking (not asynchronous):
+
+.. code-block:: python
+
+     from dsmr_parser import telegram_specifications
+     from dsmr_parser.clients import SocketReader
+    
+     socket_reader = SocketReader(
+         host='127.0.0.1',
+         port=2001,
+         telegram_specification=telegram_specifications.V4
+     )
+    
+     for telegram in socket_reader.read():
+         print(telegram)  # see 'Telegram object' docs below
+
 
 Parsing module usage
 --------------------

dsmr_parser/clients/__init__.py

@@ -1,5 +1,6 @@
 from dsmr_parser.clients.settings import SERIAL_SETTINGS_V2_2, \
     SERIAL_SETTINGS_V4, SERIAL_SETTINGS_V5
 from dsmr_parser.clients.serial_ import SerialReader, AsyncSerialReader
+from dsmr_parser.clients.socket_ import SocketReader
 from dsmr_parser.clients.protocol import create_dsmr_protocol, \
     create_dsmr_reader, create_tcp_dsmr_reader

dsmr_parser/clients/protocol.py

@@ -16,6 +16,13 @@ from dsmr_parser.clients.settings import SERIAL_SETTINGS_V2_2, \
 
 def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None, **kwargs):
     """Creates a DSMR asyncio protocol."""
+    protocol = _create_dsmr_protocol(dsmr_version, telegram_callback,
+                                     DSMRProtocol, loop, **kwargs)
+    return protocol
+
+
+def _create_dsmr_protocol(dsmr_version, telegram_callback, protocol, loop=None, **kwargs):
+    """Creates a DSMR asyncio protocol."""
 
     if dsmr_version == '2.2':
         specification = telegram_specifications.V2_2
@@ -45,7 +52,7 @@ def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None, **kwargs):
         raise NotImplementedError("No telegram parser found for version: %s",
                                   dsmr_version)
 
-    protocol = partial(DSMRProtocol, loop, TelegramParser(specification),
+    protocol = partial(protocol, loop, TelegramParser(specification),
                        telegram_callback=telegram_callback, **kwargs)
 
     return protocol, serial_settings
@@ -113,7 +120,7 @@ class DSMRProtocol(asyncio.Protocol):
         self.telegram_buffer.append(data)
 
         for telegram in self.telegram_buffer.get_all():
-            # ensure actual telegram is ascii (7-bit) only (IEC 646 required in section 5.4 of IEC 62056-21)
+            # ensure actual telegram is ascii (7-bit) only (ISO 646:1991 IRV required in section 5.5 of IEC 62056-21)
             telegram = telegram.encode("latin1").decode("ascii")
             self.handle_telegram(telegram)
 

dsmr_parser/clients/rfxtrx_protocol.py

@@ -0,0 +1,62 @@
+"""Asyncio protocol implementation for handling telegrams over a RFXtrx connection ."""
+
+import asyncio
+
+from serial_asyncio import create_serial_connection
+from .protocol import DSMRProtocol, _create_dsmr_protocol
+
+
+def create_rfxtrx_dsmr_protocol(dsmr_version, telegram_callback, loop=None, **kwargs):
+    """Creates a RFXtrxDSMR asyncio protocol."""
+    protocol = _create_dsmr_protocol(dsmr_version, telegram_callback,
+                                     RFXtrxDSMRProtocol, loop, **kwargs)
+    return protocol
+
+
+def create_rfxtrx_dsmr_reader(port, dsmr_version, telegram_callback, loop=None):
+    """Creates a DSMR asyncio protocol coroutine using a RFXtrx serial port."""
+    protocol, serial_settings = create_rfxtrx_dsmr_protocol(
+        dsmr_version, telegram_callback, loop=None)
+    serial_settings['url'] = port
+
+    conn = create_serial_connection(loop, protocol, **serial_settings)
+    return conn
+
+
+def create_rfxtrx_tcp_dsmr_reader(host, port, dsmr_version,
+                                  telegram_callback, loop=None,
+                                  keep_alive_interval=None):
+    """Creates a DSMR asyncio protocol coroutine using a RFXtrx TCP connection."""
+    if not loop:
+        loop = asyncio.get_event_loop()
+    protocol, _ = create_rfxtrx_dsmr_protocol(
+        dsmr_version, telegram_callback, loop=loop,
+        keep_alive_interval=keep_alive_interval)
+    conn = loop.create_connection(protocol, host, port)
+    return conn
+
+
+PACKETTYPE_DSMR = 0x62
+SUBTYPE_P1 = 0x01
+
+
+class RFXtrxDSMRProtocol(DSMRProtocol):
+
+    remaining_data = b''
+
+    def data_received(self, data):
+        """Add incoming data to buffer."""
+
+        data = self.remaining_data + data
+
+        packetlength = data[0] + 1 if len(data) > 0 else 1
+        while packetlength <= len(data):
+            packettype = data[1]
+            subtype = data[2]
+            if (packettype == PACKETTYPE_DSMR and subtype == SUBTYPE_P1):
+                dsmr_data = data[4:packetlength]
+                super().data_received(dsmr_data)
+            data = data[packetlength:]
+            packetlength = data[0] + 1 if len(data) > 0 else 1
+
+        self.remaining_data = data

dsmr_parser/clients/socket_.py

@@ -0,0 +1,90 @@
+import logging
+import socket
+
+from dsmr_parser.clients.telegram_buffer import TelegramBuffer
+from dsmr_parser.exceptions import ParseError, InvalidChecksumError
+from dsmr_parser.parsers import TelegramParser
+from dsmr_parser.objects import Telegram
+
+
+logger = logging.getLogger(__name__)
+
+
+class SocketReader(object):
+
+    BUFFER_SIZE = 256
+
+    def __init__(self, host, port, telegram_specification):
+        self.host = host
+        self.port = port
+
+        self.telegram_parser = TelegramParser(telegram_specification)
+        self.telegram_buffer = TelegramBuffer()
+        self.telegram_specification = telegram_specification
+
+    def read(self):
+        """
+        Read complete DSMR telegram's from remote interface and parse it
+        into CosemObject's and MbusObject's
+
+        :rtype: generator
+        """
+        buffer = b""
+
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as socket_handle:
+
+            socket_handle.connect((self.host, self.port))
+
+            while True:
+                buffer += socket_handle.recv(self.BUFFER_SIZE)
+
+                lines = buffer.splitlines(keepends=True)
+
+                if len(lines) == 0:
+                    continue
+
+                for data in lines:
+                    self.telegram_buffer.append(data.decode('ascii'))
+
+                for telegram in self.telegram_buffer.get_all():
+                    try:
+                        yield self.telegram_parser.parse(telegram)
+                    except InvalidChecksumError as e:
+                        logger.warning(str(e))
+                    except ParseError as e:
+                        logger.error('Failed to parse telegram: %s', e)
+
+                buffer = b""
+
+    def read_as_object(self):
+        """
+        Read complete DSMR telegram's from remote and return a Telegram object.
+
+        :rtype: generator
+        """
+        buffer = b""
+
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as socket_handle:
+
+            socket_handle.connect((self.host, self.port))
+
+            while True:
+                buffer += socket_handle.recv(self.BUFFER_SIZE)
+
+                lines = buffer.splitlines(keepends=True)
+
+                if len(lines) == 0:
+                    continue
+
+                for data in lines:
+                    self.telegram_buffer.append(data.decode('ascii'))
+
+                    for telegram in self.telegram_buffer.get_all():
+                        try:
+                            yield Telegram(telegram, self.telegram_parser, self.telegram_specification)
+                        except InvalidChecksumError as e:
+                            logger.warning(str(e))
+                        except ParseError as e:
+                            logger.error('Failed to parse telegram: %s', e)
+
+                buffer = b""

dsmr_parser/obis_name_mapping.py

@@ -52,10 +52,6 @@ EN = {
     obis.VALVE_POSITION_GAS: 'VALVE_POSITION_GAS',
     obis.BELGIUM_HOURLY_GAS_METER_READING: 'BELGIUM_HOURLY_GAS_METER_READING',
     obis.LUXEMBOURG_EQUIPMENT_IDENTIFIER: 'LUXEMBOURG_EQUIPMENT_IDENTIFIER',
-    obis.LUXEMBOURG_ELECTRICITY_USED_TARIFF_GLOBAL: 'LUXEMBOURG_ELECTRICITY_USED_TARIFF_GLOBAL',
-    obis.LUXEMBOURG_ELECTRICITY_DELIVERED_TARIFF_GLOBAL: 'LUXEMBOURG_ELECTRICITY_DELIVERED_TARIFF_GLOBAL',
-    obis.SWEDEN_ELECTRICITY_USED_TARIFF_GLOBAL: 'SWEDEN_ELECTRICITY_USED_TARIFF_GLOBAL',
-    obis.SWEDEN_ELECTRICITY_DELIVERED_TARIFF_GLOBAL: 'SWEDEN_ELECTRICITY_DELIVERED_TARIFF_GLOBAL',
     obis.Q3D_EQUIPMENT_IDENTIFIER: 'Q3D_EQUIPMENT_IDENTIFIER',
     obis.Q3D_EQUIPMENT_STATE: 'Q3D_EQUIPMENT_STATE',
     obis.Q3D_EQUIPMENT_SERIALNUMBER: 'Q3D_EQUIPMENT_SERIALNUMBER',

dsmr_parser/obis_references.py

@@ -8,10 +8,8 @@ objects are introduced.
 """
 P1_MESSAGE_HEADER = r'\d-\d:0\.2\.8.+?\r\n'
 P1_MESSAGE_TIMESTAMP = r'\d-\d:1\.0\.0.+?\r\n'
-ELECTRICITY_IMPORTED_TOTAL = r'\d-\d:1\.8\.0.+?\r\n'
 ELECTRICITY_USED_TARIFF_1 = r'\d-\d:1\.8\.1.+?\r\n'
 ELECTRICITY_USED_TARIFF_2 = r'\d-\d:1\.8\.2.+?\r\n'
-ELECTRICITY_EXPORTED_TOTAL = r'\d-\d:2\.8\.0.+?\r\n'
 ELECTRICITY_DELIVERED_TARIFF_1 = r'\d-\d:2\.8\.1.+?\r\n'
 ELECTRICITY_DELIVERED_TARIFF_2 = r'\d-\d:2\.8\.2.+?\r\n'
 ELECTRICITY_ACTIVE_TARIFF = r'\d-\d:96\.14\.0.+?\r\n'
@@ -62,13 +60,13 @@ ELECTRICITY_DELIVERED_TARIFF_ALL = (
     ELECTRICITY_DELIVERED_TARIFF_2
 )
 
-# Alternate codes for foreign countries.
+# International generalized additions
+ELECTRICITY_IMPORTED_TOTAL = r'\d-\d:1\.8\.0.+?\r\n'  # Total imported energy register (P+)
+ELECTRICITY_EXPORTED_TOTAL = r'\d-\d:2\.8\.0.+?\r\n'  # Total exported energy register (P-)
+
+# International non generalized additions (country specific) / risk for necessary refactoring
 BELGIUM_HOURLY_GAS_METER_READING = r'\d-\d:24\.2\.3.+?\r\n'  # Different code, same format.
 LUXEMBOURG_EQUIPMENT_IDENTIFIER = r'\d-\d:42\.0\.0.+?\r\n'  # Logical device name
-LUXEMBOURG_ELECTRICITY_USED_TARIFF_GLOBAL = r'\d-\d:1\.8\.0.+?\r\n'  # Total imported energy register (P+)
-LUXEMBOURG_ELECTRICITY_DELIVERED_TARIFF_GLOBAL = r'\d-\d:2\.8\.0.+?\r\n'  # Total exported energy register (P-)
-SWEDEN_ELECTRICITY_USED_TARIFF_GLOBAL = r'\d-\d:1\.8\.0.+?\r\n'  # Total imported energy register (P+)
-SWEDEN_ELECTRICITY_DELIVERED_TARIFF_GLOBAL = r'\d-\d:2\.8\.0.+?\r\n'  # Total exported energy register (P-)
 Q3D_EQUIPMENT_IDENTIFIER = r'\d-\d:0\.0\.0.+?\r\n'  # Logical device name
 Q3D_EQUIPMENT_STATE = r'\d-\d:96\.5\.5.+?\r\n'  # Device state (hexadecimal)
 Q3D_EQUIPMENT_SERIALNUMBER = r'\d-\d:96\.1\.255.+?\r\n'  # Device Serialnumber

dsmr_parser/objects.py

@@ -179,7 +179,7 @@ class ProfileGenericObject(DSMRObject):
             self._buffer_list = []
             values_offset = 2
             for i in range(self.buffer_length):
-                offset = values_offset + i*2
+                offset = values_offset + i * 2
                 self._buffer_list.append(MBusObject([self.values[offset], self.values[offset + 1]]))
         return self._buffer_list
 

dsmr_parser/profile_generic_specifications.py

@@ -7,4 +7,4 @@ PG_HEAD_PARSERS = [ValueParser(int), ValueParser(str)]
 PG_UNIDENTIFIED_BUFFERTYPE_PARSERS = [ValueParser(str), ValueParser(str)]
 BUFFER_TYPES = {
     PG_FAILURE_EVENT:  [ValueParser(timestamp), ValueParser(int)]
-    }
+}

dsmr_parser/telegram_specifications.py

@@ -153,8 +153,8 @@ BELGIUM_FLUVIUS['objects'].update({
 LUXEMBOURG_SMARTY = deepcopy(V5)
 LUXEMBOURG_SMARTY['objects'].update({
     obis.LUXEMBOURG_EQUIPMENT_IDENTIFIER: CosemParser(ValueParser(str)),
-    obis.LUXEMBOURG_ELECTRICITY_USED_TARIFF_GLOBAL: CosemParser(ValueParser(Decimal)),
+    obis.ELECTRICITY_IMPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
-    obis.LUXEMBOURG_ELECTRICITY_DELIVERED_TARIFF_GLOBAL: CosemParser(ValueParser(Decimal)),
+    obis.ELECTRICITY_EXPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
 })
 
 # Source: https://www.energiforetagen.se/globalassets/energiforetagen/det-erbjuder-vi/kurser-och-konferenser/elnat/
@@ -164,8 +164,8 @@ SWEDEN = {
     'objects': {
         obis.P1_MESSAGE_HEADER: CosemParser(ValueParser(str)),
         obis.P1_MESSAGE_TIMESTAMP: CosemParser(ValueParser(timestamp)),
-        obis.SWEDEN_ELECTRICITY_USED_TARIFF_GLOBAL: CosemParser(ValueParser(Decimal)),
+        obis.ELECTRICITY_IMPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
-        obis.SWEDEN_ELECTRICITY_DELIVERED_TARIFF_GLOBAL: CosemParser(ValueParser(Decimal)),
+        obis.ELECTRICITY_EXPORTED_TOTAL: CosemParser(ValueParser(Decimal)),
         obis.CURRENT_ELECTRICITY_USAGE: CosemParser(ValueParser(Decimal)),
         obis.CURRENT_ELECTRICITY_DELIVERY: CosemParser(ValueParser(Decimal)),
         obis.INSTANTANEOUS_ACTIVE_POWER_L1_POSITIVE: CosemParser(ValueParser(Decimal)),

test/example_telegrams.py

@@ -136,7 +136,7 @@ TELEGRAM_V5 = (
 #
 # last two lines are added by the COM-1 Ethernet Gateway
 
-TELEGRAM_ESY5Q3DB1024_V304 = ( # Easymeter an Hauptstromzähler
+TELEGRAM_ESY5Q3DB1024_V304 = (
     '/ESY5Q3DB1024 V3.04\r\n'
     '\r\n'
     '1-0:0.0.0*255(0272031312565)\r\n'
@@ -150,10 +150,11 @@ TELEGRAM_ESY5Q3DB1024_V304 = ( # Easymeter an Hauptstromzähler
     '0-0:96.1.255*255(1ESY1313002565)\r\n'
     '!\r\n'
     '  25803103\r\n'
-    '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\r\n'
+    '\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff'
+    '\xff\xff\xff\xff\xff\r\n'
 )
 
-TELEGRAM_ESY5Q3DA1004_V304 = ( # Easymeter an Wärmepumpe
+TELEGRAM_ESY5Q3DA1004_V304 = (
     '/ESY5Q3DA1004 V3.04\r\n'
     '\r\n'
     '1-0:0.0.0*255(1336001560)\r\n'

test/test_rfxtrx_protocol.py

@@ -0,0 +1,77 @@
+from unittest.mock import Mock
+
+import unittest
+
+from dsmr_parser import obis_references as obis
+from dsmr_parser.clients.rfxtrx_protocol import create_rfxtrx_dsmr_protocol, PACKETTYPE_DSMR, SUBTYPE_P1
+
+
+TELEGRAM_V2_2 = (
+    '/ISk5\2MT382-1004\r\n'
+    '\r\n'
+    '0-0:96.1.1(00000000000000)\r\n'
+    '1-0:1.8.1(00001.001*kWh)\r\n'
+    '1-0:1.8.2(00001.001*kWh)\r\n'
+    '1-0:2.8.1(00001.001*kWh)\r\n'
+    '1-0:2.8.2(00001.001*kWh)\r\n'
+    '0-0:96.14.0(0001)\r\n'
+    '1-0:1.7.0(0001.01*kW)\r\n'
+    '1-0:2.7.0(0000.00*kW)\r\n'
+    '0-0:17.0.0(0999.00*kW)\r\n'
+    '0-0:96.3.10(1)\r\n'
+    '0-0:96.13.1()\r\n'
+    '0-0:96.13.0()\r\n'
+    '0-1:24.1.0(3)\r\n'
+    '0-1:96.1.0(000000000000)\r\n'
+    '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n'
+    '(00001.001)\r\n'
+    '0-1:24.4.0(1)\r\n'
+    '!\r\n'
+)
+
+OTHER_RF_PACKET = b'\x03\x01\x02\x03'
+
+
+def encode_telegram_as_RF_packets(telegram):
+    data = b''
+
+    for line in telegram.split('\n'):
+        packet_data = (line + '\n').encode('ascii')
+        packet_header = bytes(bytearray([
+            len(packet_data) + 3,  # excluding length byte
+            PACKETTYPE_DSMR,
+            SUBTYPE_P1,
+            0  # seq num (ignored)
+        ]))
+
+        data += packet_header + packet_data
+        # other RF packets can pass by on the line
+        data += OTHER_RF_PACKET
+
+    return data
+
+
+class RFXtrxProtocolTest(unittest.TestCase):
+
+    def setUp(self):
+        new_protocol, _ = create_rfxtrx_dsmr_protocol('2.2',
+                                                      telegram_callback=Mock(),
+                                                      keep_alive_interval=1)
+        self.protocol = new_protocol()
+
+    def test_complete_packet(self):
+        """Protocol should assemble incoming lines into complete packet."""
+
+        data = encode_telegram_as_RF_packets(TELEGRAM_V2_2)
+        # send data broken up in two parts
+        self.protocol.data_received(data[0:200])
+        self.protocol.data_received(data[200:])
+
+        telegram = self.protocol.telegram_callback.call_args_list[0][0][0]
+        assert isinstance(telegram, dict)
+
+        assert float(telegram[obis.CURRENT_ELECTRICITY_USAGE].value) == 1.01
+        assert telegram[obis.CURRENT_ELECTRICITY_USAGE].unit == 'kW'
+
+        assert float(telegram[obis.GAS_METER_READING].value) == 1.001
+        assert telegram[obis.GAS_METER_READING].unit == 'm3'