From d990a316ad9b15c5b4f7fcee3aec4068e69f0ef0 Mon Sep 17 00:00:00 2001 From: Nigel Dokter Date: Sat, 7 Jan 2017 11:25:43 +0100 Subject: [PATCH] finishing implementation of TelegramBuffer --- dsmr_parser/serial.py | 48 +++++++++++++++++++++ test/example_telegrams.py | 62 ++++++++++++++++++++++++++++ test/telegram_buffer.py | 87 +++++++++++++++++++++++++++++++++++++++ test/test_parse_v2_2.py | 24 +---------- test/test_parse_v4_2.py | 41 +----------------- 5 files changed, 199 insertions(+), 63 deletions(-) create mode 100644 test/example_telegrams.py create mode 100644 test/telegram_buffer.py diff --git a/dsmr_parser/serial.py b/dsmr_parser/serial.py index edd0efd..3e51f48 100644 --- a/dsmr_parser/serial.py +++ b/dsmr_parser/serial.py @@ -1,5 +1,6 @@ import asyncio import logging +import re import serial import serial_asyncio @@ -136,3 +137,50 @@ class AsyncSerialReader(SerialReader): logger.warning('Failed to parse telegram: %s', e) telegram = '' + + +class TelegramBuffer(object): + + def __init__(self, callback): + self._callback = callback + self._buffer = '' + + def append(self, data): + """ + Add telegram data to buffer. The callback is called with a full telegram + when data is complete. + :param str data: chars or lines of telegram data + :return: + """ + self._buffer += data + + for telegram in self.find_telegrams(self._buffer): + self._callback(telegram) + self._remove(telegram) + + def _remove(self, telegram): + """ + Remove telegram from buffer and incomplete data preceding it. This + is easier than validating the data before adding it to the buffer. + :param str telegram: + :return: + """ + # Remove data leading up to the telegram and the telegram itself. + index = self._buffer.index(telegram) + len(telegram) + + self._buffer = self._buffer[index:] + + @staticmethod + def find_telegrams(buffer): + """ + Find complete telegrams from buffer from start ('/') till ending + checksum ('!AB12\r\n'). + :rtype: list + """ + # - Match all characters after start of telegram except for the start + # itself again '^\/]+', which eliminates incomplete preceding telegrams. + # - Do non greedy match using '?' so start is matched up to the first + # checksum that's found. + # - The checksum is optional '{0,4}' because not all telegram versions + # support it. + return re.findall(r'\/[^\/]+?\![A-F0-9]{0,4}\r\n', buffer, re.DOTALL) diff --git a/test/example_telegrams.py b/test/example_telegrams.py new file mode 100644 index 0000000..eb4e8d4 --- /dev/null +++ b/test/example_telegrams.py @@ -0,0 +1,62 @@ +TELEGRAM_V2_2 = ( + '/ISk5\2MT382-1004\r\n' + '\r\n' + '0-0:96.1.1(00000000000000)\r\n' + '1-0:1.8.1(00001.001*kWh)\r\n' + '1-0:1.8.2(00001.001*kWh)\r\n' + '1-0:2.8.1(00001.001*kWh)\r\n' + '1-0:2.8.2(00001.001*kWh)\r\n' + '0-0:96.14.0(0001)\r\n' + '1-0:1.7.0(0001.01*kW)\r\n' + '1-0:2.7.0(0000.00*kW)\r\n' + '0-0:17.0.0(0999.00*kW)\r\n' + '0-0:96.3.10(1)\r\n' + '0-0:96.13.1()\r\n' + '0-0:96.13.0()\r\n' + '0-1:24.1.0(3)\r\n' + '0-1:96.1.0(000000000000)\r\n' + '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n' + '(00001.001)\r\n' + '0-1:24.4.0(1)\r\n' + '!\r\n' +) + +TELEGRAM_V4_2 = ( + '/KFM5KAIFA-METER\r\n' + '\r\n' + '1-3:0.2.8(42)\r\n' + '0-0:1.0.0(161113205757W)\r\n' + '0-0:96.1.1(3960221976967177082151037881335713)\r\n' + '1-0:1.8.1(001581.123*kWh)\r\n' + '1-0:1.8.2(001435.706*kWh)\r\n' + '1-0:2.8.1(000000.000*kWh)\r\n' + '1-0:2.8.2(000000.000*kWh)\r\n' + '0-0:96.14.0(0002)\r\n' + '1-0:1.7.0(02.027*kW)\r\n' + '1-0:2.7.0(00.000*kW)\r\n' + '0-0:96.7.21(00015)\r\n' + '0-0:96.7.9(00007)\r\n' + '1-0:99.97.0(3)(0-0:96.7.19)(000104180320W)(0000237126*s)(000101000001W)' + '(2147583646*s)(000102000003W)(2317482647*s)\r\n' + '1-0:32.32.0(00000)\r\n' + '1-0:52.32.0(00000)\r\n' + '1-0:72.32.0(00000)\r\n' + '1-0:32.36.0(00000)\r\n' + '1-0:52.36.0(00000)\r\n' + '1-0:72.36.0(00000)\r\n' + '0-0:96.13.1()\r\n' + '0-0:96.13.0()\r\n' + '1-0:31.7.0(000*A)\r\n' + '1-0:51.7.0(006*A)\r\n' + '1-0:71.7.0(002*A)\r\n' + '1-0:21.7.0(00.170*kW)\r\n' + '1-0:22.7.0(00.000*kW)\r\n' + '1-0:41.7.0(01.247*kW)\r\n' + '1-0:42.7.0(00.000*kW)\r\n' + '1-0:61.7.0(00.209*kW)\r\n' + '1-0:62.7.0(00.000*kW)\r\n' + '0-1:24.1.0(003)\r\n' + '0-1:96.1.0(4819243993373755377509728609491464)\r\n' + '0-1:24.2.1(161129200000W)(00981.443*m3)\r\n' + '!6796\r\n' +) diff --git a/test/telegram_buffer.py b/test/telegram_buffer.py new file mode 100644 index 0000000..92df5b9 --- /dev/null +++ b/test/telegram_buffer.py @@ -0,0 +1,87 @@ +from unittest import mock, TestCase +from unittest.mock import call + +from dsmr_parser.serial import TelegramBuffer +from test.example_telegrams import TELEGRAM_V2_2, TELEGRAM_V4_2 + + +class TelegramBufferTest(TestCase): + + def setUp(self): + self.callback = mock.MagicMock() + self.telegram_buffer = TelegramBuffer(self.callback) + + def test_v22_telegram(self): + self.telegram_buffer.append(TELEGRAM_V2_2) + + self.callback.assert_called_once_with(TELEGRAM_V2_2) + self.assertEqual(self.telegram_buffer._buffer, '') + + def test_v42_telegram(self): + self.telegram_buffer.append(TELEGRAM_V4_2) + + self.callback.assert_called_once_with(TELEGRAM_V4_2) + self.assertEqual(self.telegram_buffer._buffer, '') + + def test_multiple_mixed_telegrams(self): + self.telegram_buffer.append( + ''.join((TELEGRAM_V2_2, TELEGRAM_V4_2, TELEGRAM_V2_2)) + ) + + self.callback.assert_has_calls([ + call(TELEGRAM_V2_2), + call(TELEGRAM_V4_2), + call(TELEGRAM_V2_2), + ]) + self.assertEqual(self.telegram_buffer._buffer, '') + + def test_v42_telegram_preceded_with_unclosed_telegram(self): + # There are unclosed telegrams at the start of the buffer. + incomplete_telegram = TELEGRAM_V4_2[:-1] + + self.telegram_buffer.append(incomplete_telegram + TELEGRAM_V4_2) + + self.callback.assert_called_once_with(TELEGRAM_V4_2) + self.assertEqual(self.telegram_buffer._buffer, '') + + def test_v42_telegram_preceded_with_unopened_telegram(self): + # There is unopened telegrams at the start of the buffer indicating that + # the buffer was being filled while the telegram was outputted halfway. + incomplete_telegram = TELEGRAM_V4_2[1:] + + self.telegram_buffer.append(incomplete_telegram + TELEGRAM_V4_2) + + self.callback.assert_called_once_with(TELEGRAM_V4_2) + self.assertEqual(self.telegram_buffer._buffer, '') + + def test_v42_telegram_trailed_by_unclosed_telegram(self): + incomplete_telegram = TELEGRAM_V4_2[:-1] + + self.telegram_buffer.append(TELEGRAM_V4_2 + incomplete_telegram) + + self.callback.assert_called_once_with(TELEGRAM_V4_2) + self.assertEqual(self.telegram_buffer._buffer, incomplete_telegram) + + def test_v42_telegram_trailed_by_unopened_telegram(self): + incomplete_telegram = TELEGRAM_V4_2[1:] + + self.telegram_buffer.append(TELEGRAM_V4_2 + incomplete_telegram) + + self.callback.assert_called_once_with(TELEGRAM_V4_2) + self.assertEqual(self.telegram_buffer._buffer, incomplete_telegram) + + def test_v42_telegram_adding_line_by_line(self): + + for line in TELEGRAM_V4_2.splitlines(keepends=True): + self.telegram_buffer.append(line) + + self.callback.assert_called_once_with(TELEGRAM_V4_2) + self.assertEqual(self.telegram_buffer._buffer, '') + + def test_v42_telegram_adding_char_by_char(self): + + for char in TELEGRAM_V4_2: + self.telegram_buffer.append(char) + + self.callback.assert_called_once_with(TELEGRAM_V4_2) + self.assertEqual(self.telegram_buffer._buffer, '') diff --git a/test/test_parse_v2_2.py b/test/test_parse_v2_2.py index 1d6b504..eaaa7ee 100644 --- a/test/test_parse_v2_2.py +++ b/test/test_parse_v2_2.py @@ -1,32 +1,10 @@ import unittest +from .example_telegrams import TELEGRAM_V2_2 from dsmr_parser.parsers import TelegramParserV2_2 from dsmr_parser import telegram_specifications from dsmr_parser import obis_references as obis -TELEGRAM_V2_2 = ( - '/ISk5\2MT382-1004\r\n' - '\r\n' - '0-0:96.1.1(00000000000000)\r\n' - '1-0:1.8.1(00001.001*kWh)\r\n' - '1-0:1.8.2(00001.001*kWh)\r\n' - '1-0:2.8.1(00001.001*kWh)\r\n' - '1-0:2.8.2(00001.001*kWh)\r\n' - '0-0:96.14.0(0001)\r\n' - '1-0:1.7.0(0001.01*kW)\r\n' - '1-0:2.7.0(0000.00*kW)\r\n' - '0-0:17.0.0(0999.00*kW)\r\n' - '0-0:96.3.10(1)\r\n' - '0-0:96.13.1()\r\n' - '0-0:96.13.0()\r\n' - '0-1:24.1.0(3)\r\n' - '0-1:96.1.0(000000000000)\r\n' - '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n' - '(00001.001)\r\n' - '0-1:24.4.0(1)\r\n' - '!\r\n' -) - class TelegramParserV2_2Test(unittest.TestCase): """ Test parsing of a DSMR v2.2 telegram. """ diff --git a/test/test_parse_v4_2.py b/test/test_parse_v4_2.py index 4a2085d..a7904fc 100644 --- a/test/test_parse_v4_2.py +++ b/test/test_parse_v4_2.py @@ -4,52 +4,13 @@ import unittest import pytz +from .example_telegrams import TELEGRAM_V4_2 from dsmr_parser import obis_references as obis from dsmr_parser import telegram_specifications from dsmr_parser.exceptions import InvalidChecksumError, ParseError from dsmr_parser.objects import CosemObject, MBusObject from dsmr_parser.parsers import TelegramParser, TelegramParserV4 -TELEGRAM_V4_2 = ( - '/KFM5KAIFA-METER\r\n' - '\r\n' - '1-3:0.2.8(42)\r\n' - '0-0:1.0.0(161113205757W)\r\n' - '0-0:96.1.1(3960221976967177082151037881335713)\r\n' - '1-0:1.8.1(001581.123*kWh)\r\n' - '1-0:1.8.2(001435.706*kWh)\r\n' - '1-0:2.8.1(000000.000*kWh)\r\n' - '1-0:2.8.2(000000.000*kWh)\r\n' - '0-0:96.14.0(0002)\r\n' - '1-0:1.7.0(02.027*kW)\r\n' - '1-0:2.7.0(00.000*kW)\r\n' - '0-0:96.7.21(00015)\r\n' - '0-0:96.7.9(00007)\r\n' - '1-0:99.97.0(3)(0-0:96.7.19)(000104180320W)(0000237126*s)(000101000001W)' - '(2147583646*s)(000102000003W)(2317482647*s)\r\n' - '1-0:32.32.0(00000)\r\n' - '1-0:52.32.0(00000)\r\n' - '1-0:72.32.0(00000)\r\n' - '1-0:32.36.0(00000)\r\n' - '1-0:52.36.0(00000)\r\n' - '1-0:72.36.0(00000)\r\n' - '0-0:96.13.1()\r\n' - '0-0:96.13.0()\r\n' - '1-0:31.7.0(000*A)\r\n' - '1-0:51.7.0(006*A)\r\n' - '1-0:71.7.0(002*A)\r\n' - '1-0:21.7.0(00.170*kW)\r\n' - '1-0:22.7.0(00.000*kW)\r\n' - '1-0:41.7.0(01.247*kW)\r\n' - '1-0:42.7.0(00.000*kW)\r\n' - '1-0:61.7.0(00.209*kW)\r\n' - '1-0:62.7.0(00.000*kW)\r\n' - '0-1:24.1.0(003)\r\n' - '0-1:96.1.0(4819243993373755377509728609491464)\r\n' - '0-1:24.2.1(161129200000W)(00981.443*m3)\r\n' - '!6796\r\n' -) - class TelegramParserV4_2Test(unittest.TestCase): """ Test parsing of a DSMR v4.2 telegram. """