update version number to 0.7

Pass lines to parser including line endings

CHANGELOG.rst

@@ -1,6 +1,21 @@
 Change Log
 ----------
 
+**0.7** (2017-01-14)
+
+- Internal refactoring related to the way clients feed their data into the parse module. Clients can now supply the telegram data in single characters, lines (which was common) or complete telegram strings.
+
+**IMPORTANT: this release has the following backwards incompatible changes:**
+
+- Client related imports from dsmr_parser.serial and dsmr_parser.protocol have been moved to dsmr_parser.clients (import these from the clients/__init__.py module)
+- The .parse() method of TelegramParser, TelegramParserV2_2, TelegramParserV4 now accepts a string containing the entire telegram (including \r\n characters) and not a list
+
+
+**0.6** (2017-01-04)
+
+- Fixed bug in CRC checksum verification for the asyncio client (`pull request #15 <https://github.com/ndokter/dsmr_parser/pull/15>`_)
+- Support added for TCP connections using the asyncio client (`pull request #12 <https://github.com/ndokter/dsmr_parser/pull/12/>`_)
+
 **0.5** (2016-12-29)
 
 - CRC checksum verification for DSMR v4 telegrams (`issue #10 <https://github.com/ndokter/dsmr_parser/issues/10>`_)

README.rst

@@ -26,7 +26,7 @@ Using the serial reader to connect to your smart meter and parse it's telegrams:
 
     from dsmr_parser import telegram_specifications
     from dsmr_parser import obis_references
-    from dsmr_parser.serial import SerialReader, SERIAL_SETTINGS_V4
+    from dsmr_parser.clients import SerialReader, SERIAL_SETTINGS_V4
 
     serial_reader = SerialReader(
         device='/dev/ttyUSB0',

dsmr_parser/__main__.py

@@ -1,8 +1,9 @@
+from functools import partial
 import argparse
 import asyncio
 import logging
 
-from .protocol import create_dsmr_reader
+from dsmr_parser.clients import create_dsmr_reader, create_tcp_dsmr_reader
 
 
 def console():
@@ -11,6 +12,10 @@ def console():
     parser = argparse.ArgumentParser(description=console.__doc__)
     parser.add_argument('--device', default='/dev/ttyUSB0',
                         help='port to read DSMR data from')
+    parser.add_argument('--host', default=None,
+                        help='alternatively connect using TCP host.')
+    parser.add_argument('--port', default=None,
+                        help='TCP port to use for connection')
     parser.add_argument('--version', default='2.2', choices=['2.2', '4'],
                         help='DSMR version (2.2, 4)')
     parser.add_argument('--verbose', '-v', action='count')
@@ -32,7 +37,29 @@ def console():
                 print(obj.value, obj.unit)
         print()
 
-    conn = create_dsmr_reader(args.device, args.version, print_callback, loop=loop)
+    # create tcp or serial connection depending on args
+    if args.host and args.port:
+        create_connection = partial(create_tcp_dsmr_reader,
+                                    args.host, args.port, args.version,
+                                    print_callback, loop=loop)
+    else:
+        create_connection = partial(create_dsmr_reader,
+                                    args.device, args.version,
+                                    print_callback, loop=loop)
 
-    loop.create_task(conn)
-    loop.run_forever()
+    try:
+        # connect and keep connected until interrupted by ctrl-c
+        while True:
+            # create serial or tcp connection
+            conn = create_connection()
+            transport, protocol = loop.run_until_complete(conn)
+            # wait until connection it closed
+            loop.run_until_complete(protocol.wait_closed())
+            # wait 5 seconds before attempting reconnect
+            loop.run_until_complete(asyncio.sleep(5))
+    except KeyboardInterrupt:
+        # cleanup connection after user initiated shutdown
+        transport.close()
+        loop.run_until_complete(asyncio.sleep(0))
+    finally:
+        loop.close()

dsmr_parser/clients/__init__.py

@@ -0,0 +1,5 @@
+from dsmr_parser.clients.settings import SERIAL_SETTINGS_V2_2, \
+    SERIAL_SETTINGS_V4
+from dsmr_parser.clients.serial_ import SerialReader, AsyncSerialReader
+from dsmr_parser.clients.protocol import create_dsmr_protocol, \
+    create_dsmr_reader, create_tcp_dsmr_reader

dsmr_parser/clients/protocol.py

@@ -0,0 +1,111 @@
+"""Asyncio protocol implementation for handling telegrams."""
+
+from functools import partial
+import asyncio
+import logging
+
+from serial_asyncio import create_serial_connection
+
+from dsmr_parser import telegram_specifications
+from dsmr_parser.clients.telegram_buffer import TelegramBuffer
+from dsmr_parser.exceptions import ParseError
+from dsmr_parser.parsers import TelegramParserV2_2, TelegramParserV4
+from dsmr_parser.clients.settings import SERIAL_SETTINGS_V2_2, \
+    SERIAL_SETTINGS_V4
+
+
+def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None):
+    """Creates a DSMR asyncio protocol."""
+
+    if dsmr_version == '2.2':
+        specifications = telegram_specifications.V2_2
+        telegram_parser = TelegramParserV2_2
+        serial_settings = SERIAL_SETTINGS_V2_2
+    elif dsmr_version == '4':
+        specifications = telegram_specifications.V4
+        telegram_parser = TelegramParserV4
+        serial_settings = SERIAL_SETTINGS_V4
+    else:
+        raise NotImplementedError("No telegram parser found for version: %s",
+                                  dsmr_version)
+
+    protocol = partial(DSMRProtocol, loop, telegram_parser(specifications),
+                       telegram_callback=telegram_callback)
+
+    return protocol, serial_settings
+
+
+def create_dsmr_reader(port, dsmr_version, telegram_callback, loop=None):
+    """Creates a DSMR asyncio protocol coroutine using serial port."""
+    protocol, serial_settings = create_dsmr_protocol(
+        dsmr_version, telegram_callback, loop=None)
+    serial_settings['url'] = port
+
+    conn = create_serial_connection(loop, protocol, **serial_settings)
+    return conn
+
+
+def create_tcp_dsmr_reader(host, port, dsmr_version,
+                           telegram_callback, loop=None):
+    """Creates a DSMR asyncio protocol coroutine using TCP connection."""
+    protocol, _ = create_dsmr_protocol(
+        dsmr_version, telegram_callback, loop=None)
+    conn = loop.create_connection(protocol, host, port)
+    return conn
+
+
+class DSMRProtocol(asyncio.Protocol):
+    """Assemble and handle incoming data into complete DSM telegrams."""
+
+    transport = None
+    telegram_callback = None
+
+    def __init__(self, loop, telegram_parser, telegram_callback=None):
+        """Initialize class."""
+        self.loop = loop
+        self.log = logging.getLogger(__name__)
+        self.telegram_parser = telegram_parser
+        # callback to call on complete telegram
+        self.telegram_callback = telegram_callback
+        # buffer to keep incomplete incoming data
+        self.telegram_buffer = TelegramBuffer()
+        # keep a lock until the connection is closed
+        self._closed = asyncio.Event()
+
+    def connection_made(self, transport):
+        """Just logging for now."""
+        self.transport = transport
+        self.log.debug('connected')
+
+    def data_received(self, data):
+        """Add incoming data to buffer."""
+        data = data.decode('ascii')
+        self.log.debug('received data: %s', data)
+        self.telegram_buffer.append(data)
+
+        for telegram in self.telegram_buffer.get_all():
+            self.handle_telegram(telegram)
+
+    def connection_lost(self, exc):
+        """Stop when connection is lost."""
+        if exc:
+            self.log.exception('disconnected due to exception')
+        else:
+            self.log.info('disconnected because of close/abort.')
+        self._closed.set()
+
+    def handle_telegram(self, telegram):
+        """Send off parsed telegram to handling callback."""
+        self.log.debug('got telegram: %s', telegram)
+
+        try:
+            parsed_telegram = self.telegram_parser.parse(telegram)
+        except ParseError:
+            self.log.exception("failed to parse telegram")
+        else:
+            self.telegram_callback(parsed_telegram)
+
+    @asyncio.coroutine
+    def wait_closed(self):
+        """Wait until connection is closed."""
+        yield from self._closed.wait()

dsmr_parser/clients/serial_.py

@@ -3,48 +3,17 @@ import logging
 import serial
 import serial_asyncio
 
+from dsmr_parser.clients.telegram_buffer import TelegramBuffer
 from dsmr_parser.exceptions import ParseError
 from dsmr_parser.parsers import TelegramParser, TelegramParserV2_2, \
     TelegramParserV4
+from dsmr_parser.clients.settings import SERIAL_SETTINGS_V2_2, \
+    SERIAL_SETTINGS_V4
+
 
 logger = logging.getLogger(__name__)
 
 
-SERIAL_SETTINGS_V2_2 = {
-    'baudrate': 9600,
-    'bytesize': serial.SEVENBITS,
-    'parity': serial.PARITY_EVEN,
-    'stopbits': serial.STOPBITS_ONE,
-    'xonxoff': 0,
-    'rtscts': 0,
-    'timeout': 20
-}
-
-SERIAL_SETTINGS_V4 = {
-    'baudrate': 115200,
-    'bytesize': serial.SEVENBITS,
-    'parity': serial.PARITY_EVEN,
-    'stopbits': serial.STOPBITS_ONE,
-    'xonxoff': 0,
-    'rtscts': 0,
-    'timeout': 20
-}
-
-
-def is_start_of_telegram(line):
-    """
-    :type line: line
-    """
-    return line.startswith('/')
-
-
-def is_end_of_telegram(line):
-    """
-    :type line: line
-    """
-    return line.startswith('!')
-
-
 class SerialReader(object):
     PORT_KEY = 'port'
 
@@ -60,37 +29,26 @@ class SerialReader(object):
             telegram_parser = TelegramParser
 
         self.telegram_parser = telegram_parser(telegram_specification)
+        self.telegram_buffer = TelegramBuffer()
 
     def read(self):
         """
         Read complete DSMR telegram's from the serial interface and parse it
         into CosemObject's and MbusObject's
 
-        :rtype dict
+        :rtype: generator
         """
         with serial.Serial(**self.serial_settings) as serial_handle:
-            telegram = []
-
             while True:
-                line = serial_handle.readline()
-                line = line.decode('ascii')  # TODO move this to the parser?
-
-                # Telegrams need to be complete because the values belong to a
-                # particular reading and can also be related to eachother.
-                if not telegram and not is_start_of_telegram(line):
-                    continue
-
-                telegram.append(line)
-
-                if is_end_of_telegram(line):
+                data = serial_handle.readline()
+                self.telegram_buffer.append(data.decode('ascii'))
 
+                for telegram in self.telegram_buffer.get_all():
                     try:
                         yield self.telegram_parser.parse(telegram)
                     except ParseError as e:
                         logger.error('Failed to parse telegram: %s', e)
 
-                    telegram = []
-
 
 class AsyncSerialReader(SerialReader):
     """Serial reader using asyncio pyserial."""
@@ -106,33 +64,23 @@ class AsyncSerialReader(SerialReader):
         Instead of being a generator, values are pushed to provided queue for
         asynchronous processing.
 
-        :rtype Generator/Async
+        :rtype: None
         """
         # create Serial StreamReader
         conn = serial_asyncio.open_serial_connection(**self.serial_settings)
         reader, _ = yield from conn
 
-        telegram = []
-
         while True:
-            # read line if available or give control back to loop until
-            # new data has arrived
-            line = yield from reader.readline()
-            line = line.decode('ascii')
+            # Read line if available or give control back to loop until new
+            # data has arrived.
+            data = yield from reader.readline()
+            self.telegram_buffer.append(data.decode('ascii'))
 
-            # Telegrams need to be complete because the values belong to a
-            # particular reading and can also be related to eachother.
-            if not telegram and not is_start_of_telegram(line):
-                continue
-
-            telegram.append(line)
-
-            if is_end_of_telegram(line):
+            for telegram in self.telegram_buffer.get_all():
                 try:
-                    parsed_telegram = self.telegram_parser.parse(telegram)
-                    # push new parsed telegram onto queue
-                    queue.put_nowait(parsed_telegram)
+                    # Push new parsed telegram onto queue.
+                    queue.put_nowait(
+                        self.telegram_parser.parse(telegram)
+                    )
                 except ParseError as e:
                     logger.warning('Failed to parse telegram: %s', e)
-
-                telegram = []

dsmr_parser/clients/settings.py

@@ -0,0 +1,22 @@
+import serial
+
+
+SERIAL_SETTINGS_V2_2 = {
+    'baudrate': 9600,
+    'bytesize': serial.SEVENBITS,
+    'parity': serial.PARITY_EVEN,
+    'stopbits': serial.STOPBITS_ONE,
+    'xonxoff': 0,
+    'rtscts': 0,
+    'timeout': 20
+}
+
+SERIAL_SETTINGS_V4 = {
+    'baudrate': 115200,
+    'bytesize': serial.SEVENBITS,
+    'parity': serial.PARITY_EVEN,
+    'stopbits': serial.STOPBITS_ONE,
+    'xonxoff': 0,
+    'rtscts': 0,
+    'timeout': 20
+}

dsmr_parser/clients/telegram_buffer.py

@@ -0,0 +1,57 @@
+import re
+
+
+class TelegramBuffer(object):
+    """
+    Used as a buffer for a stream of telegram data. Constructs full telegram
+    strings from the buffered data and returns it.
+    """
+
+    def __init__(self):
+        self._buffer = ''
+
+    def get_all(self):
+        """
+        Remove complete telegrams from buffer and yield them.
+        :rtype generator:
+        """
+        for telegram in self._find_telegrams():
+            self._remove(telegram)
+            yield telegram
+
+    def append(self, data):
+        """
+        Add telegram data to buffer.
+        :param str data: chars, lines or full telegram strings of telegram data
+        """
+        self._buffer += data
+
+    def _remove(self, telegram):
+        """
+        Remove telegram from buffer and incomplete data preceding it. This
+        is easier than validating the data before adding it to the buffer.
+        :param str telegram:
+        :return:
+        """
+        # Remove data leading up to the telegram and the telegram itself.
+        index = self._buffer.index(telegram) + len(telegram)
+
+        self._buffer = self._buffer[index:]
+
+    def _find_telegrams(self):
+        """
+        Find complete telegrams in buffer from  start ('/') till ending
+        checksum ('!AB12\r\n').
+        :rtype: list
+        """
+        # - Match all characters after start of telegram except for the start
+        # itself again '^\/]+', which eliminates incomplete preceding telegrams.
+        # - Do non greedy match using '?' so start is matched up to the first
+        # checksum that's found.
+        # - The checksum is optional '{0,4}' because not all telegram versions
+        # support it.
+        return re.findall(
+            r'\/[^\/]+?\![A-F0-9]{0,4}\r\n',
+            self._buffer,
+            re.DOTALL
+        )

dsmr_parser/parsers.py

@@ -3,9 +3,9 @@ import re
 
 from PyCRC.CRC16 import CRC16
 
-from .objects import MBusObject, MBusObjectV2_2, CosemObject
-from .exceptions import ParseError, InvalidChecksumError
-from .obis_references import GAS_METER_READING
+from dsmr_parser.objects import MBusObject, MBusObjectV2_2, CosemObject
+from dsmr_parser.exceptions import ParseError, InvalidChecksumError
+from dsmr_parser.obis_references import GAS_METER_READING
 
 logger = logging.getLogger(__name__)
 
@@ -26,48 +26,58 @@ class TelegramParser(object):
 
         return None, None
 
-    def parse(self, line_values):
-        telegram = {}
+    def parse(self, telegram):
+        """
+        Parse telegram from string to dict.
 
-        for line_value in line_values:
-            # TODO temporarily strip newline characters.
-            line_value = line_value.strip()
+        The telegram str type makes python 2.x integration easier.
 
-            obis_reference, dsmr_object = self.parse_line(line_value)
+        :param str telegram: full telegram from start ('/') to checksum
+            ('!ABCD') including line endings inbetween the telegram's lines
+        :rtype: dict
+        :returns: Shortened example:
+            {
+                ..
+                r'0-0:96\.1\.1': <CosemObject>,  # EQUIPMENT_IDENTIFIER
+                r'1-0:1\.8\.1': <CosemObject>,   # ELECTRICITY_USED_TARIFF_1
+                r'0-\d:24\.3\.0': <MBusObject>,  # GAS_METER_READING
+                ..
+            }
+        """
+        telegram_lines = telegram.splitlines()
+        parsed_lines = map(self.parse_line, telegram_lines)
 
-            telegram[obis_reference] = dsmr_object
+        return {obis_reference: dsmr_object
+                for obis_reference, dsmr_object in parsed_lines}
 
-        return telegram
+    def parse_line(self, line):
+        logger.debug("Parsing line '%s'", line)
 
-    def parse_line(self, line_value):
-        logger.debug('Parsing line \'%s\'', line_value)
+        obis_reference, parser = self._find_line_parser(line)
 
-        obis_reference, parser = self._find_line_parser(line_value)
-
-        if not parser:
-            logger.warning("No line class found for: '%s'", line_value)
+        if not obis_reference:
+            logger.debug("No line class found for: '%s'", line)
             return None, None
 
-        return obis_reference, parser.parse(line_value)
+        return obis_reference, parser.parse(line)
 
 
 class TelegramParserV4(TelegramParser):
 
     @staticmethod
-    def validate_telegram_checksum(line_values):
+    def validate_telegram_checksum(telegram):
         """
-        :type line_values: list
+        :param str telegram:
         :raises ParseError:
         :raises InvalidChecksumError:
         """
 
-        full_telegram = ''.join(line_values)
-
-        # Extract the bytes that count towards the checksum.
-        checksum_contents = re.search(r'\/.+\!', full_telegram, re.DOTALL)
+        # Extract the part for which the checksum applies.
+        checksum_contents = re.search(r'\/.+\!', telegram, re.DOTALL)
 
         # Extract the hexadecimal checksum value itself.
-        checksum_hex = re.search(r'((?<=\!)[0-9A-Z]{4}(?=\r\n))+', full_telegram)
+        # The line ending '\r\n' for the checksum line can be ignored.
+        checksum_hex = re.search(r'((?<=\!)[0-9A-Z]{4})+', telegram)
 
         if not checksum_contents or not checksum_hex:
             raise ParseError(
@@ -76,8 +86,7 @@ class TelegramParserV4(TelegramParser):
             )
 
         calculated_crc = CRC16().calculate(checksum_contents.group(0))
-        expected_crc = checksum_hex.group(0)
-        expected_crc = int(expected_crc, base=16)
+        expected_crc = int(checksum_hex.group(0), base=16)
 
         if calculated_crc != expected_crc:
             raise InvalidChecksumError(
@@ -88,31 +97,44 @@ class TelegramParserV4(TelegramParser):
                 )
             )
 
-    def parse(self, line_values):
-        self.validate_telegram_checksum(line_values)
+    def parse(self, telegram):
+        """
+        :param str telegram:
+        :rtype: dict
+        """
+        self.validate_telegram_checksum(telegram)
 
-        return super().parse(line_values)
+        return super().parse(telegram)
 
 
 class TelegramParserV2_2(TelegramParser):
 
-    def parse(self, line_values):
-        """Join lines for gas meter."""
+    def parse(self, telegram):
+        """
+        :param str telegram:
+        :rtype: dict
+        """
 
-        def join_lines(line_values):
+        # TODO fix this in the specification: telegram_specifications.V2_2
+        def join_lines(telegram):
+            """Join lines for gas meter."""
             join_next = re.compile(GAS_METER_READING)
 
             join = None
-            for line_value in line_values:
+            for line_value in telegram.splitlines():
                 if join:
-                    yield join.strip() + line_value
+                    yield join + line_value
                     join = None
                 elif join_next.match(line_value):
                     join = line_value
                 else:
                     yield line_value
 
-        return super().parse(join_lines(line_values))
+        # TODO temporary workaround
+        lines = join_lines(telegram)
+        telegram = '\r\n'.join(lines)
+
+        return super().parse(telegram)
 
 
 class DSMRObjectParser(object):

dsmr_parser/protocol.py

@@ -1,103 +0,0 @@
-"""Asyncio protocol implementation for handling telegrams."""
-
-import asyncio
-import logging
-from functools import partial
-
-from serial_asyncio import create_serial_connection
-
-from . import telegram_specifications
-from .exceptions import ParseError
-from .parsers import (
-    TelegramParserV2_2,
-    TelegramParserV4
-)
-from .serial import (
-    SERIAL_SETTINGS_V2_2, SERIAL_SETTINGS_V4,
-    is_end_of_telegram,
-    is_start_of_telegram
-)
-
-
-def create_dsmr_reader(port, dsmr_version, telegram_callback, loop=None):
-    """Creates a DSMR asyncio protocol coroutine."""
-
-    if dsmr_version == '2.2':
-        specifications = telegram_specifications.V2_2
-        telegram_parser = TelegramParserV2_2
-        serial_settings = SERIAL_SETTINGS_V2_2
-    elif dsmr_version == '4':
-        specifications = telegram_specifications.V4
-        telegram_parser = TelegramParserV4
-        serial_settings = SERIAL_SETTINGS_V4
-
-    serial_settings['url'] = port
-
-    protocol = partial(DSMRProtocol, loop, telegram_parser(specifications),
-                       telegram_callback=telegram_callback)
-
-    conn = create_serial_connection(loop, protocol, **serial_settings)
-
-    return conn
-
-
-class DSMRProtocol(asyncio.Protocol):
-    """Assemble and handle incoming data into complete DSM telegrams."""
-
-    transport = None
-    telegram_callback = None
-
-    def __init__(self, loop, telegram_parser, telegram_callback=None):
-        """Initialize class."""
-        self.loop = loop
-        self.log = logging.getLogger(__name__)
-        self.telegram_parser = telegram_parser
-        # callback to call on complete telegram
-        self.telegram_callback = telegram_callback
-        # buffer to keep incoming telegram lines
-        self.telegram = []
-        # buffer to keep incomplete incoming data
-        self.buffer = ''
-
-    def connection_made(self, transport):
-        """Just logging for now."""
-        self.transport = transport
-        self.log.debug('connected')
-
-    def data_received(self, data):
-        """Add incoming data to buffer."""
-        data = data.decode()
-        self.log.debug('received data: %s', data.strip())
-        self.buffer += data
-        self.handle_lines()
-
-    def handle_lines(self):
-        """Assemble incoming data into single lines."""
-        while "\r\n" in self.buffer:
-            line, self.buffer = self.buffer.split("\r\n", 1)
-            self.log.debug('got line: %s', line)
-
-            # Telegrams need to be complete because the values belong to a
-            # particular reading and can also be related to eachother.
-            if not self.telegram and not is_start_of_telegram(line):
-                continue
-
-            self.telegram.append(line)
-            if is_end_of_telegram(line):
-                try:
-                    parsed_telegram = self.telegram_parser.parse(self.telegram)
-                    self.handle_telegram(parsed_telegram)
-                except ParseError:
-                    self.log.exception("failed to parse telegram")
-                self.telegram = []
-
-    def connection_lost(self, exc):
-        """Stop when connection is lost."""
-        self.log.error('disconnected')
-
-    def handle_telegram(self, telegram):
-        """Send off parsed telegram to handling callback."""
-        self.log.debug('got telegram: %s', telegram)
-
-        if self.telegram_callback:
-            self.telegram_callback(telegram)

dsmr_parser/telegram_specifications.py

@@ -1,8 +1,8 @@
 from decimal import Decimal
 
-from . import obis_references as obis
-from .parsers import CosemParser, ValueParser, MBusParser
-from .value_types import timestamp
+from dsmr_parser import obis_references as obis
+from dsmr_parser.parsers import CosemParser, ValueParser, MBusParser
+from dsmr_parser.value_types import timestamp
 
 
 """

setup.py

@@ -6,7 +6,7 @@ setup(
     author='Nigel Dokter',
     author_email='nigeldokter@gmail.com',
     url='https://github.com/ndokter/dsmr_parser',
-    version='0.5',
+    version='0.7',
     packages=find_packages(),
     install_requires=[
         'pyserial>=3,<4',

test/example_telegrams.py

@@ -0,0 +1,62 @@
+TELEGRAM_V2_2 = (
+    '/ISk5\2MT382-1004\r\n'
+    '\r\n'
+    '0-0:96.1.1(00000000000000)\r\n'
+    '1-0:1.8.1(00001.001*kWh)\r\n'
+    '1-0:1.8.2(00001.001*kWh)\r\n'
+    '1-0:2.8.1(00001.001*kWh)\r\n'
+    '1-0:2.8.2(00001.001*kWh)\r\n'
+    '0-0:96.14.0(0001)\r\n'
+    '1-0:1.7.0(0001.01*kW)\r\n'
+    '1-0:2.7.0(0000.00*kW)\r\n'
+    '0-0:17.0.0(0999.00*kW)\r\n'
+    '0-0:96.3.10(1)\r\n'
+    '0-0:96.13.1()\r\n'
+    '0-0:96.13.0()\r\n'
+    '0-1:24.1.0(3)\r\n'
+    '0-1:96.1.0(000000000000)\r\n'
+    '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n'
+    '(00001.001)\r\n'
+    '0-1:24.4.0(1)\r\n'
+    '!\r\n'
+)
+
+TELEGRAM_V4_2 = (
+    '/KFM5KAIFA-METER\r\n'
+    '\r\n'
+    '1-3:0.2.8(42)\r\n'
+    '0-0:1.0.0(161113205757W)\r\n'
+    '0-0:96.1.1(3960221976967177082151037881335713)\r\n'
+    '1-0:1.8.1(001581.123*kWh)\r\n'
+    '1-0:1.8.2(001435.706*kWh)\r\n'
+    '1-0:2.8.1(000000.000*kWh)\r\n'
+    '1-0:2.8.2(000000.000*kWh)\r\n'
+    '0-0:96.14.0(0002)\r\n'
+    '1-0:1.7.0(02.027*kW)\r\n'
+    '1-0:2.7.0(00.000*kW)\r\n'
+    '0-0:96.7.21(00015)\r\n'
+    '0-0:96.7.9(00007)\r\n'
+    '1-0:99.97.0(3)(0-0:96.7.19)(000104180320W)(0000237126*s)(000101000001W)'
+    '(2147583646*s)(000102000003W)(2317482647*s)\r\n'
+    '1-0:32.32.0(00000)\r\n'
+    '1-0:52.32.0(00000)\r\n'
+    '1-0:72.32.0(00000)\r\n'
+    '1-0:32.36.0(00000)\r\n'
+    '1-0:52.36.0(00000)\r\n'
+    '1-0:72.36.0(00000)\r\n'
+    '0-0:96.13.1()\r\n'
+    '0-0:96.13.0()\r\n'
+    '1-0:31.7.0(000*A)\r\n'
+    '1-0:51.7.0(006*A)\r\n'
+    '1-0:71.7.0(002*A)\r\n'
+    '1-0:21.7.0(00.170*kW)\r\n'
+    '1-0:22.7.0(00.000*kW)\r\n'
+    '1-0:41.7.0(01.247*kW)\r\n'
+    '1-0:42.7.0(00.000*kW)\r\n'
+    '1-0:61.7.0(00.209*kW)\r\n'
+    '1-0:62.7.0(00.000*kW)\r\n'
+    '0-1:24.1.0(003)\r\n'
+    '0-1:96.1.0(4819243993373755377509728609491464)\r\n'
+    '0-1:24.2.1(161129200000W)(00981.443*m3)\r\n'
+    '!6796\r\n'
+)

test/test_parse_v2_2.py

@@ -1,32 +1,10 @@
 import unittest
 
+from test.example_telegrams import TELEGRAM_V2_2
 from dsmr_parser.parsers import TelegramParserV2_2
 from dsmr_parser import telegram_specifications
 from dsmr_parser import obis_references as obis
 
-TELEGRAM_V2_2 = [
-    '/ISk5\2MT382-1004',
-    '',
-    '0-0:96.1.1(00000000000000)',
-    '1-0:1.8.1(00001.001*kWh)',
-    '1-0:1.8.2(00001.001*kWh)',
-    '1-0:2.8.1(00001.001*kWh)',
-    '1-0:2.8.2(00001.001*kWh)',
-    '0-0:96.14.0(0001)',
-    '1-0:1.7.0(0001.01*kW)',
-    '1-0:2.7.0(0000.00*kW)',
-    '0-0:17.0.0(0999.00*kW)',
-    '0-0:96.3.10(1)',
-    '0-0:96.13.1()',
-    '0-0:96.13.0()',
-    '0-1:24.1.0(3)',
-    '0-1:96.1.0(000000000000)',
-    '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)',
-    '(00001.001)',
-    '0-1:24.4.0(1)',
-    '!',
-]
-
 
 class TelegramParserV2_2Test(unittest.TestCase):
     """ Test parsing of a DSMR v2.2 telegram. """

test/test_parse_v4_2.py

@@ -4,78 +4,38 @@ import unittest
 
 import pytz
 
+from test.example_telegrams import TELEGRAM_V4_2
 from dsmr_parser import obis_references as obis
 from dsmr_parser import telegram_specifications
 from dsmr_parser.exceptions import InvalidChecksumError, ParseError
 from dsmr_parser.objects import CosemObject, MBusObject
 from dsmr_parser.parsers import TelegramParser, TelegramParserV4
 
-TELEGRAM_V4_2 = [
-    '/KFM5KAIFA-METER\r\n',
-    '\r\n',
-    '1-3:0.2.8(42)\r\n',
-    '0-0:1.0.0(161113205757W)\r\n',
-    '0-0:96.1.1(3960221976967177082151037881335713)\r\n',
-    '1-0:1.8.1(001581.123*kWh)\r\n',
-    '1-0:1.8.2(001435.706*kWh)\r\n',
-    '1-0:2.8.1(000000.000*kWh)\r\n',
-    '1-0:2.8.2(000000.000*kWh)\r\n',
-    '0-0:96.14.0(0002)\r\n',
-    '1-0:1.7.0(02.027*kW)\r\n',
-    '1-0:2.7.0(00.000*kW)\r\n',
-    '0-0:96.7.21(00015)\r\n',
-    '0-0:96.7.9(00007)\r\n',
-    '1-0:99.97.0(3)(0-0:96.7.19)(000104180320W)(0000237126*s)(000101000001W)'
-    '(2147583646*s)(000102000003W)(2317482647*s)\r\n',
-    '1-0:32.32.0(00000)\r\n',
-    '1-0:52.32.0(00000)\r\n',
-    '1-0:72.32.0(00000)\r\n',
-    '1-0:32.36.0(00000)\r\n',
-    '1-0:52.36.0(00000)\r\n',
-    '1-0:72.36.0(00000)\r\n',
-    '0-0:96.13.1()\r\n',
-    '0-0:96.13.0()\r\n',
-    '1-0:31.7.0(000*A)\r\n',
-    '1-0:51.7.0(006*A)\r\n',
-    '1-0:71.7.0(002*A)\r\n',
-    '1-0:21.7.0(00.170*kW)\r\n',
-    '1-0:22.7.0(00.000*kW)\r\n',
-    '1-0:41.7.0(01.247*kW)\r\n',
-    '1-0:42.7.0(00.000*kW)\r\n',
-    '1-0:61.7.0(00.209*kW)\r\n',
-    '1-0:62.7.0(00.000*kW)\r\n',
-    '0-1:24.1.0(003)\r\n',
-    '0-1:96.1.0(4819243993373755377509728609491464)\r\n',
-    '0-1:24.2.1(161129200000W)(00981.443*m3)\r\n',
-    '!6796\r\n'
-]
-
 
 class TelegramParserV4_2Test(unittest.TestCase):
     """ Test parsing of a DSMR v4.2 telegram. """
 
     def test_valid(self):
         # No exception is raised.
-        TelegramParserV4.validate_telegram_checksum(
-            TELEGRAM_V4_2
-        )
+        TelegramParserV4.validate_telegram_checksum(TELEGRAM_V4_2)
 
     def test_invalid(self):
-        # Remove one the electricty used data value. This causes the checksum to
+        # Remove the electricty used data value. This causes the checksum to
         # not match anymore.
-        telegram = [line
-                    for line in TELEGRAM_V4_2
-                    if '1-0:1.8.1' not in line]
+        corrupted_telegram = TELEGRAM_V4_2.replace(
+            '1-0:1.8.1(001581.123*kWh)\r\n',
+            ''
+        )
 
         with self.assertRaises(InvalidChecksumError):
-            TelegramParserV4.validate_telegram_checksum(telegram)
+            TelegramParserV4.validate_telegram_checksum(corrupted_telegram)
 
     def test_missing_checksum(self):
         # Remove the checksum value causing a ParseError.
-        telegram = TELEGRAM_V4_2[:-1]
+        corrupted_telegram = TELEGRAM_V4_2.replace('!6796\r\n', '')
 
         with self.assertRaises(ParseError):
-            TelegramParserV4.validate_telegram_checksum(telegram)
+            TelegramParserV4.validate_telegram_checksum(corrupted_telegram)
 
     def test_parse(self):
         parser = TelegramParser(telegram_specifications.V4)

test/test_protocol.py

@@ -1,62 +1,57 @@
-"""Test DSMR serial protocol."""
-
 from unittest.mock import Mock
 
-import pytest
+import unittest
 
 from dsmr_parser import obis_references as obis
 from dsmr_parser import telegram_specifications
 from dsmr_parser.parsers import TelegramParserV2_2
-from dsmr_parser.protocol import DSMRProtocol
+from dsmr_parser.clients.protocol import DSMRProtocol
 
 
-TELEGRAM_V2_2 = [
-    "/ISk5\2MT382-1004",
-    "",
-    "0-0:96.1.1(00000000000000)",
-    "1-0:1.8.1(00001.001*kWh)",
-    "1-0:1.8.2(00001.001*kWh)",
-    "1-0:2.8.1(00001.001*kWh)",
-    "1-0:2.8.2(00001.001*kWh)",
-    "0-0:96.14.0(0001)",
-    "1-0:1.7.0(0001.01*kW)",
-    "1-0:2.7.0(0000.00*kW)",
-    "0-0:17.0.0(0999.00*kW)",
-    "0-0:96.3.10(1)",
-    "0-0:96.13.1()",
-    "0-0:96.13.0()",
-    "0-1:24.1.0(3)",
-    "0-1:96.1.0(000000000000)",
-    "0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)",
-    "(00001.001)",
-    "0-1:24.4.0(1)",
-    "!",
-]
+TELEGRAM_V2_2 = (
+    '/ISk5\2MT382-1004\r\n'
+    '\r\n'
+    '0-0:96.1.1(00000000000000)\r\n'
+    '1-0:1.8.1(00001.001*kWh)\r\n'
+    '1-0:1.8.2(00001.001*kWh)\r\n'
+    '1-0:2.8.1(00001.001*kWh)\r\n'
+    '1-0:2.8.2(00001.001*kWh)\r\n'
+    '0-0:96.14.0(0001)\r\n'
+    '1-0:1.7.0(0001.01*kW)\r\n'
+    '1-0:2.7.0(0000.00*kW)\r\n'
+    '0-0:17.0.0(0999.00*kW)\r\n'
+    '0-0:96.3.10(1)\r\n'
+    '0-0:96.13.1()\r\n'
+    '0-0:96.13.0()\r\n'
+    '0-1:24.1.0(3)\r\n'
+    '0-1:96.1.0(000000000000)\r\n'
+    '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n'
+    '(00001.001)\r\n'
+    '0-1:24.4.0(1)\r\n'
+    '!\r\n'
+)
 
 
-@pytest.fixture
-def protocol():
-    """DSMRprotocol instance with mocked telegram_callback."""
+class ProtocolTest(unittest.TestCase):
 
-    parser = TelegramParserV2_2
-    specification = telegram_specifications.V2_2
+    def setUp(self):
+        parser = TelegramParserV2_2
+        specification = telegram_specifications.V2_2
 
-    telegram_parser = parser(specification)
-    return DSMRProtocol(None, telegram_parser,
-                        telegram_callback=Mock())
+        telegram_parser = parser(specification)
+        self.protocol = DSMRProtocol(None, telegram_parser,
+                                     telegram_callback=Mock())
 
+    def test_complete_packet(self):
+        """Protocol should assemble incoming lines into complete packet."""
 
-def test_complete_packet(protocol):
-    """Protocol should assemble incoming lines into complete packet."""
+        self.protocol.data_received(TELEGRAM_V2_2.encode('ascii'))
 
-    for line in TELEGRAM_V2_2:
-        protocol.data_received(bytes(line + '\r\n', 'ascii'))
+        telegram = self.protocol.telegram_callback.call_args_list[0][0][0]
+        assert isinstance(telegram, dict)
 
-    telegram = protocol.telegram_callback.call_args_list[0][0][0]
-    assert isinstance(telegram, dict)
+        assert float(telegram[obis.CURRENT_ELECTRICITY_USAGE].value) == 1.01
+        assert telegram[obis.CURRENT_ELECTRICITY_USAGE].unit == 'kW'
 
-    assert float(telegram[obis.CURRENT_ELECTRICITY_USAGE].value) == 1.01
-    assert telegram[obis.CURRENT_ELECTRICITY_USAGE].unit == 'kW'
-
-    assert float(telegram[obis.GAS_METER_READING].value) == 1.001
-    assert telegram[obis.GAS_METER_READING].unit == 'm3'
+        assert float(telegram[obis.GAS_METER_READING].value) == 1.001
+        assert telegram[obis.GAS_METER_READING].unit == 'm3'

test/test_telegram_buffer.py

@@ -0,0 +1,105 @@
+import unittest
+
+from dsmr_parser.clients.telegram_buffer import TelegramBuffer
+from test.example_telegrams import TELEGRAM_V2_2, TELEGRAM_V4_2
+
+
+class TelegramBufferTest(unittest.TestCase):
+
+    def setUp(self):
+        self.telegram_buffer = TelegramBuffer()
+
+    def test_v22_telegram(self):
+        self.telegram_buffer.append(TELEGRAM_V2_2)
+
+        telegram = next(self.telegram_buffer.get_all())
+
+        self.assertEqual(telegram, TELEGRAM_V2_2)
+        self.assertEqual(self.telegram_buffer._buffer, '')
+
+    def test_v42_telegram(self):
+        self.telegram_buffer.append(TELEGRAM_V4_2)
+
+        telegram = next(self.telegram_buffer.get_all())
+
+        self.assertEqual(telegram, TELEGRAM_V4_2)
+        self.assertEqual(self.telegram_buffer._buffer, '')
+
+    def test_multiple_mixed_telegrams(self):
+        self.telegram_buffer.append(
+            ''.join((TELEGRAM_V2_2, TELEGRAM_V4_2, TELEGRAM_V2_2))
+        )
+
+        telegrams = list(self.telegram_buffer.get_all())
+
+        self.assertListEqual(
+            telegrams,
+            [
+                TELEGRAM_V2_2,
+                TELEGRAM_V4_2,
+                TELEGRAM_V2_2
+            ]
+        )
+
+        self.assertEqual(self.telegram_buffer._buffer, '')
+
+    def test_v42_telegram_preceded_with_unclosed_telegram(self):
+        # There are unclosed telegrams at the start of the buffer.
+        incomplete_telegram = TELEGRAM_V4_2[:-1]
+
+        self.telegram_buffer.append(incomplete_telegram + TELEGRAM_V4_2)
+
+        telegram = next(self.telegram_buffer.get_all())
+
+        self.assertEqual(telegram, TELEGRAM_V4_2)
+        self.assertEqual(self.telegram_buffer._buffer, '')
+
+    def test_v42_telegram_preceded_with_unopened_telegram(self):
+        # There is unopened telegrams at the start of the buffer indicating that
+        # the buffer was being filled while the telegram was outputted halfway.
+        incomplete_telegram = TELEGRAM_V4_2[1:]
+
+        self.telegram_buffer.append(incomplete_telegram + TELEGRAM_V4_2)
+
+        telegram = next(self.telegram_buffer.get_all())
+
+        self.assertEqual(telegram, TELEGRAM_V4_2)
+        self.assertEqual(self.telegram_buffer._buffer, '')
+
+    def test_v42_telegram_trailed_by_unclosed_telegram(self):
+        incomplete_telegram = TELEGRAM_V4_2[:-1]
+
+        self.telegram_buffer.append(TELEGRAM_V4_2 + incomplete_telegram)
+
+        telegram = next(self.telegram_buffer.get_all())
+
+        self.assertEqual(telegram, TELEGRAM_V4_2)
+        self.assertEqual(self.telegram_buffer._buffer, incomplete_telegram)
+
+    def test_v42_telegram_trailed_by_unopened_telegram(self):
+        incomplete_telegram = TELEGRAM_V4_2[1:]
+
+        self.telegram_buffer.append(TELEGRAM_V4_2 + incomplete_telegram)
+
+        telegram = next(self.telegram_buffer.get_all())
+
+        self.assertEqual(telegram, TELEGRAM_V4_2)
+        self.assertEqual(self.telegram_buffer._buffer, incomplete_telegram)
+
+    def test_v42_telegram_adding_line_by_line(self):
+        for line in TELEGRAM_V4_2.splitlines(keepends=True):
+            self.telegram_buffer.append(line)
+
+        telegram = next(self.telegram_buffer.get_all())
+
+        self.assertEqual(telegram, TELEGRAM_V4_2)
+        self.assertEqual(self.telegram_buffer._buffer, '')
+
+    def test_v42_telegram_adding_char_by_char(self):
+        for char in TELEGRAM_V4_2:
+            self.telegram_buffer.append(char)
+
+        telegram = next(self.telegram_buffer.get_all())
+
+        self.assertEqual(telegram, TELEGRAM_V4_2)
+        self.assertEqual(self.telegram_buffer._buffer, '')

tox.ini

@@ -14,6 +14,9 @@ commands=
   py.test --cov=dsmr_parser test {posargs}
   pylama dsmr_parser test
 
+[pylama:dsmr_parser/clients/__init__.py]
+ignore = W0611
+
 [pylama:pylint]
 max_line_length = 100