From f10032f701544453ca21c28fc7b3ba478e2b7192 Mon Sep 17 00:00:00 2001 From: Nigel Dokter Date: Thu, 5 Jan 2017 21:24:41 +0100 Subject: [PATCH] refactored TelegramParser.parse to accept a str instead of list --- dsmr_parser/parsers.py | 88 +++++++++++++++++++++++--------------- dsmr_parser/protocol.py | 16 ++++--- dsmr_parser/serial.py | 42 +++++++++---------- test/test_parse_v2_2.py | 44 +++++++++---------- test/test_parse_v4_2.py | 93 ++++++++++++++++++++--------------------- test/test_protocol.py | 51 +++++++++++----------- 6 files changed, 180 insertions(+), 154 deletions(-) diff --git a/dsmr_parser/parsers.py b/dsmr_parser/parsers.py index cf91e2c..a215a98 100644 --- a/dsmr_parser/parsers.py +++ b/dsmr_parser/parsers.py @@ -26,48 +26,58 @@ class TelegramParser(object): return None, None - def parse(self, line_values): - telegram = {} + def parse(self, telegram): + """ + Parse telegram from string to dict. - for line_value in line_values: - # TODO temporarily strip newline characters. - line_value = line_value.strip() + The telegram str type makes python 2.x integration easier. - obis_reference, dsmr_object = self.parse_line(line_value) + :param str telegram: full telegram from start ('/') to checksum + ('!ABCD') including line endings inbetween the telegram's lines + :rtype: dict + :returns: Shortened example: + { + .. + r'0-0:96\.1\.1': , # EQUIPMENT_IDENTIFIER + r'1-0:1\.8\.1': , # ELECTRICITY_USED_TARIFF_1 + r'0-\d:24\.3\.0': , # GAS_METER_READING + .. + } + """ + telegram_lines = telegram.splitlines() + parsed_lines = map(self.parse_line, telegram_lines) - telegram[obis_reference] = dsmr_object + return {obis_reference: dsmr_object + for obis_reference, dsmr_object in parsed_lines} - return telegram + def parse_line(self, line): + logger.debug("Parsing line '%s'", line) - def parse_line(self, line_value): - logger.debug('Parsing line \'%s\'', line_value) + obis_reference, parser = self._find_line_parser(line) - obis_reference, parser = self._find_line_parser(line_value) - - if not parser: - logger.warning("No line class found for: '%s'", line_value) + if not obis_reference: + logger.debug("No line class found for: '%s'", line) return None, None - return obis_reference, parser.parse(line_value) + return obis_reference, parser.parse(line) class TelegramParserV4(TelegramParser): @staticmethod - def validate_telegram_checksum(line_values): + def validate_telegram_checksum(telegram): """ - :type line_values: list + :param str telegram: :raises ParseError: :raises InvalidChecksumError: """ - full_telegram = ''.join(line_values) - - # Extract the bytes that count towards the checksum. - checksum_contents = re.search(r'\/.+\!', full_telegram, re.DOTALL) + # Extract the part for which the checksum applies. + checksum_contents = re.search(r'\/.+\!', telegram, re.DOTALL) # Extract the hexadecimal checksum value itself. - checksum_hex = re.search(r'((?<=\!)[0-9A-Z]{4}(?=\r\n))+', full_telegram) + # The line ending '\r\n' for the checksum line can be ignored. + checksum_hex = re.search(r'((?<=\!)[0-9A-Z]{4})+', telegram) if not checksum_contents or not checksum_hex: raise ParseError( @@ -76,8 +86,7 @@ class TelegramParserV4(TelegramParser): ) calculated_crc = CRC16().calculate(checksum_contents.group(0)) - expected_crc = checksum_hex.group(0) - expected_crc = int(expected_crc, base=16) + expected_crc = int(checksum_hex.group(0), base=16) if calculated_crc != expected_crc: raise InvalidChecksumError( @@ -88,31 +97,44 @@ class TelegramParserV4(TelegramParser): ) ) - def parse(self, line_values): - self.validate_telegram_checksum(line_values) + def parse(self, telegram): + """ + :param str telegram: + :rtype: dict + """ + self.validate_telegram_checksum(telegram) - return super().parse(line_values) + return super().parse(telegram) class TelegramParserV2_2(TelegramParser): - def parse(self, line_values): - """Join lines for gas meter.""" + def parse(self, telegram): + """ + :param str telegram: + :rtype: dict + """ - def join_lines(line_values): + # TODO fix this in the specification: telegram_specifications.V2_2 + def join_lines(telegram): + """Join lines for gas meter.""" join_next = re.compile(GAS_METER_READING) join = None - for line_value in line_values: + for line_value in telegram.splitlines(): if join: - yield join.strip() + line_value + yield join + line_value join = None elif join_next.match(line_value): join = line_value else: yield line_value - return super().parse(join_lines(line_values)) + # TODO temporary workaround + lines = join_lines(telegram) + telegram = '\r\n'.join(lines) + + return super().parse(telegram) class DSMRObjectParser(object): diff --git a/dsmr_parser/protocol.py b/dsmr_parser/protocol.py index 5d1a8c7..dc5c54e 100644 --- a/dsmr_parser/protocol.py +++ b/dsmr_parser/protocol.py @@ -24,6 +24,9 @@ def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None): specifications = telegram_specifications.V4 telegram_parser = TelegramParserV4 serial_settings = SERIAL_SETTINGS_V4 + else: + raise NotImplementedError("No telegram parser found for version: %s", + dsmr_version) protocol = partial(DSMRProtocol, loop, telegram_parser(specifications), telegram_callback=telegram_callback) @@ -64,7 +67,7 @@ class DSMRProtocol(asyncio.Protocol): # callback to call on complete telegram self.telegram_callback = telegram_callback # buffer to keep incoming telegram lines - self.telegram = [] + self.telegram = '' # buffer to keep incomplete incoming data self.buffer = '' # keep a lock until the connection is closed @@ -77,7 +80,7 @@ class DSMRProtocol(asyncio.Protocol): def data_received(self, data): """Add incoming data to buffer.""" - data = data.decode() + data = data.decode('ascii') self.log.debug('received data: %s', data.strip()) self.buffer += data self.handle_lines() @@ -95,13 +98,16 @@ class DSMRProtocol(asyncio.Protocol): if not self.telegram and not is_start_of_telegram(line): continue - self.telegram.append(line) + self.telegram += line + if is_end_of_telegram(line): try: parsed_telegram = self.telegram_parser.parse(self.telegram) + except ParseError as e: + self.log.error('Failed to parse telegram: %s', e) + else: self.handle_telegram(parsed_telegram) - except ParseError: - self.log.exception("failed to parse telegram") + self.telegram = [] def connection_lost(self, exc): diff --git a/dsmr_parser/serial.py b/dsmr_parser/serial.py index fa70c81..edd0efd 100644 --- a/dsmr_parser/serial.py +++ b/dsmr_parser/serial.py @@ -33,14 +33,16 @@ SERIAL_SETTINGS_V4 = { def is_start_of_telegram(line): """ - :type line: line + :param bytes line: series of bytes representing a line. + Example: b'/KFM5KAIFA-METER\r\n' """ return line.startswith('/') def is_end_of_telegram(line): """ - :type line: line + :param bytes line: series of bytes representing a line. + Example: b'!7B05\r\n' """ return line.startswith('!') @@ -66,30 +68,28 @@ class SerialReader(object): Read complete DSMR telegram's from the serial interface and parse it into CosemObject's and MbusObject's - :rtype dict + :rtype: generator """ with serial.Serial(**self.serial_settings) as serial_handle: - telegram = [] + telegram = '' while True: line = serial_handle.readline() - line = line.decode('ascii') # TODO move this to the parser? + line.decode('ascii') - # Telegrams need to be complete because the values belong to a - # particular reading and can also be related to eachother. + # Build up buffer from the start of the telegram. if not telegram and not is_start_of_telegram(line): continue - telegram.append(line) + telegram += line if is_end_of_telegram(line): - try: yield self.telegram_parser.parse(telegram) except ParseError as e: logger.error('Failed to parse telegram: %s', e) - telegram = [] + telegram = '' class AsyncSerialReader(SerialReader): @@ -106,33 +106,33 @@ class AsyncSerialReader(SerialReader): Instead of being a generator, values are pushed to provided queue for asynchronous processing. - :rtype Generator/Async + :rtype: None """ # create Serial StreamReader conn = serial_asyncio.open_serial_connection(**self.serial_settings) reader, _ = yield from conn - telegram = [] + telegram = '' while True: - # read line if available or give control back to loop until - # new data has arrived + # Read line if available or give control back to loop until new + # data has arrived. line = yield from reader.readline() line = line.decode('ascii') - # Telegrams need to be complete because the values belong to a - # particular reading and can also be related to eachother. + # Build up buffer from the start of the telegram. if not telegram and not is_start_of_telegram(line): continue - telegram.append(line) + telegram += line if is_end_of_telegram(line): try: - parsed_telegram = self.telegram_parser.parse(telegram) - # push new parsed telegram onto queue - queue.put_nowait(parsed_telegram) + # Push new parsed telegram onto queue. + queue.put_nowait( + self.telegram_parser.parse(telegram) + ) except ParseError as e: logger.warning('Failed to parse telegram: %s', e) - telegram = [] + telegram = '' diff --git a/test/test_parse_v2_2.py b/test/test_parse_v2_2.py index b36a466..1d6b504 100644 --- a/test/test_parse_v2_2.py +++ b/test/test_parse_v2_2.py @@ -4,28 +4,28 @@ from dsmr_parser.parsers import TelegramParserV2_2 from dsmr_parser import telegram_specifications from dsmr_parser import obis_references as obis -TELEGRAM_V2_2 = [ - '/ISk5\2MT382-1004', - '', - '0-0:96.1.1(00000000000000)', - '1-0:1.8.1(00001.001*kWh)', - '1-0:1.8.2(00001.001*kWh)', - '1-0:2.8.1(00001.001*kWh)', - '1-0:2.8.2(00001.001*kWh)', - '0-0:96.14.0(0001)', - '1-0:1.7.0(0001.01*kW)', - '1-0:2.7.0(0000.00*kW)', - '0-0:17.0.0(0999.00*kW)', - '0-0:96.3.10(1)', - '0-0:96.13.1()', - '0-0:96.13.0()', - '0-1:24.1.0(3)', - '0-1:96.1.0(000000000000)', - '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)', - '(00001.001)', - '0-1:24.4.0(1)', - '!', -] +TELEGRAM_V2_2 = ( + '/ISk5\2MT382-1004\r\n' + '\r\n' + '0-0:96.1.1(00000000000000)\r\n' + '1-0:1.8.1(00001.001*kWh)\r\n' + '1-0:1.8.2(00001.001*kWh)\r\n' + '1-0:2.8.1(00001.001*kWh)\r\n' + '1-0:2.8.2(00001.001*kWh)\r\n' + '0-0:96.14.0(0001)\r\n' + '1-0:1.7.0(0001.01*kW)\r\n' + '1-0:2.7.0(0000.00*kW)\r\n' + '0-0:17.0.0(0999.00*kW)\r\n' + '0-0:96.3.10(1)\r\n' + '0-0:96.13.1()\r\n' + '0-0:96.13.0()\r\n' + '0-1:24.1.0(3)\r\n' + '0-1:96.1.0(000000000000)\r\n' + '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n' + '(00001.001)\r\n' + '0-1:24.4.0(1)\r\n' + '!\r\n' +) class TelegramParserV2_2Test(unittest.TestCase): diff --git a/test/test_parse_v4_2.py b/test/test_parse_v4_2.py index 58e757e..4a2085d 100644 --- a/test/test_parse_v4_2.py +++ b/test/test_parse_v4_2.py @@ -10,45 +10,45 @@ from dsmr_parser.exceptions import InvalidChecksumError, ParseError from dsmr_parser.objects import CosemObject, MBusObject from dsmr_parser.parsers import TelegramParser, TelegramParserV4 -TELEGRAM_V4_2 = [ - '/KFM5KAIFA-METER\r\n', - '\r\n', - '1-3:0.2.8(42)\r\n', - '0-0:1.0.0(161113205757W)\r\n', - '0-0:96.1.1(3960221976967177082151037881335713)\r\n', - '1-0:1.8.1(001581.123*kWh)\r\n', - '1-0:1.8.2(001435.706*kWh)\r\n', - '1-0:2.8.1(000000.000*kWh)\r\n', - '1-0:2.8.2(000000.000*kWh)\r\n', - '0-0:96.14.0(0002)\r\n', - '1-0:1.7.0(02.027*kW)\r\n', - '1-0:2.7.0(00.000*kW)\r\n', - '0-0:96.7.21(00015)\r\n', - '0-0:96.7.9(00007)\r\n', +TELEGRAM_V4_2 = ( + '/KFM5KAIFA-METER\r\n' + '\r\n' + '1-3:0.2.8(42)\r\n' + '0-0:1.0.0(161113205757W)\r\n' + '0-0:96.1.1(3960221976967177082151037881335713)\r\n' + '1-0:1.8.1(001581.123*kWh)\r\n' + '1-0:1.8.2(001435.706*kWh)\r\n' + '1-0:2.8.1(000000.000*kWh)\r\n' + '1-0:2.8.2(000000.000*kWh)\r\n' + '0-0:96.14.0(0002)\r\n' + '1-0:1.7.0(02.027*kW)\r\n' + '1-0:2.7.0(00.000*kW)\r\n' + '0-0:96.7.21(00015)\r\n' + '0-0:96.7.9(00007)\r\n' '1-0:99.97.0(3)(0-0:96.7.19)(000104180320W)(0000237126*s)(000101000001W)' - '(2147583646*s)(000102000003W)(2317482647*s)\r\n', - '1-0:32.32.0(00000)\r\n', - '1-0:52.32.0(00000)\r\n', - '1-0:72.32.0(00000)\r\n', - '1-0:32.36.0(00000)\r\n', - '1-0:52.36.0(00000)\r\n', - '1-0:72.36.0(00000)\r\n', - '0-0:96.13.1()\r\n', - '0-0:96.13.0()\r\n', - '1-0:31.7.0(000*A)\r\n', - '1-0:51.7.0(006*A)\r\n', - '1-0:71.7.0(002*A)\r\n', - '1-0:21.7.0(00.170*kW)\r\n', - '1-0:22.7.0(00.000*kW)\r\n', - '1-0:41.7.0(01.247*kW)\r\n', - '1-0:42.7.0(00.000*kW)\r\n', - '1-0:61.7.0(00.209*kW)\r\n', - '1-0:62.7.0(00.000*kW)\r\n', - '0-1:24.1.0(003)\r\n', - '0-1:96.1.0(4819243993373755377509728609491464)\r\n', - '0-1:24.2.1(161129200000W)(00981.443*m3)\r\n', + '(2147583646*s)(000102000003W)(2317482647*s)\r\n' + '1-0:32.32.0(00000)\r\n' + '1-0:52.32.0(00000)\r\n' + '1-0:72.32.0(00000)\r\n' + '1-0:32.36.0(00000)\r\n' + '1-0:52.36.0(00000)\r\n' + '1-0:72.36.0(00000)\r\n' + '0-0:96.13.1()\r\n' + '0-0:96.13.0()\r\n' + '1-0:31.7.0(000*A)\r\n' + '1-0:51.7.0(006*A)\r\n' + '1-0:71.7.0(002*A)\r\n' + '1-0:21.7.0(00.170*kW)\r\n' + '1-0:22.7.0(00.000*kW)\r\n' + '1-0:41.7.0(01.247*kW)\r\n' + '1-0:42.7.0(00.000*kW)\r\n' + '1-0:61.7.0(00.209*kW)\r\n' + '1-0:62.7.0(00.000*kW)\r\n' + '0-1:24.1.0(003)\r\n' + '0-1:96.1.0(4819243993373755377509728609491464)\r\n' + '0-1:24.2.1(161129200000W)(00981.443*m3)\r\n' '!6796\r\n' -] +) class TelegramParserV4_2Test(unittest.TestCase): @@ -56,26 +56,25 @@ class TelegramParserV4_2Test(unittest.TestCase): def test_valid(self): # No exception is raised. - TelegramParserV4.validate_telegram_checksum( - TELEGRAM_V4_2 - ) + TelegramParserV4.validate_telegram_checksum(TELEGRAM_V4_2) def test_invalid(self): - # Remove one the electricty used data value. This causes the checksum to + # Remove the electricty used data value. This causes the checksum to # not match anymore. - telegram = [line - for line in TELEGRAM_V4_2 - if '1-0:1.8.1' not in line] + corrupted_telegram = TELEGRAM_V4_2.replace( + '1-0:1.8.1(001581.123*kWh)\r\n', + '' + ) with self.assertRaises(InvalidChecksumError): - TelegramParserV4.validate_telegram_checksum(telegram) + TelegramParserV4.validate_telegram_checksum(corrupted_telegram) def test_missing_checksum(self): # Remove the checksum value causing a ParseError. - telegram = TELEGRAM_V4_2[:-1] + corrupted_telegram = TELEGRAM_V4_2.replace('!6796\r\n', '') with self.assertRaises(ParseError): - TelegramParserV4.validate_telegram_checksum(telegram) + TelegramParserV4.validate_telegram_checksum(corrupted_telegram) def test_parse(self): parser = TelegramParser(telegram_specifications.V4) diff --git a/test/test_protocol.py b/test/test_protocol.py index 41c68b4..d779bc7 100644 --- a/test/test_protocol.py +++ b/test/test_protocol.py @@ -1,35 +1,35 @@ -"""Test DSMR serial protocol.""" -import unittest from unittest.mock import Mock +import unittest + from dsmr_parser import obis_references as obis from dsmr_parser import telegram_specifications from dsmr_parser.parsers import TelegramParserV2_2 from dsmr_parser.protocol import DSMRProtocol -TELEGRAM_V2_2 = [ - "/ISk5\2MT382-1004", - "", - "0-0:96.1.1(00000000000000)", - "1-0:1.8.1(00001.001*kWh)", - "1-0:1.8.2(00001.001*kWh)", - "1-0:2.8.1(00001.001*kWh)", - "1-0:2.8.2(00001.001*kWh)", - "0-0:96.14.0(0001)", - "1-0:1.7.0(0001.01*kW)", - "1-0:2.7.0(0000.00*kW)", - "0-0:17.0.0(0999.00*kW)", - "0-0:96.3.10(1)", - "0-0:96.13.1()", - "0-0:96.13.0()", - "0-1:24.1.0(3)", - "0-1:96.1.0(000000000000)", - "0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)", - "(00001.001)", - "0-1:24.4.0(1)", - "!", -] +TELEGRAM_V2_2 = ( + '/ISk5\2MT382-1004\r\n' + '\r\n' + '0-0:96.1.1(00000000000000)\r\n' + '1-0:1.8.1(00001.001*kWh)\r\n' + '1-0:1.8.2(00001.001*kWh)\r\n' + '1-0:2.8.1(00001.001*kWh)\r\n' + '1-0:2.8.2(00001.001*kWh)\r\n' + '0-0:96.14.0(0001)\r\n' + '1-0:1.7.0(0001.01*kW)\r\n' + '1-0:2.7.0(0000.00*kW)\r\n' + '0-0:17.0.0(0999.00*kW)\r\n' + '0-0:96.3.10(1)\r\n' + '0-0:96.13.1()\r\n' + '0-0:96.13.0()\r\n' + '0-1:24.1.0(3)\r\n' + '0-1:96.1.0(000000000000)\r\n' + '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n' + '(00001.001)\r\n' + '0-1:24.4.0(1)\r\n' + '!\r\n' +) class ProtocolTest(unittest.TestCase): @@ -45,8 +45,7 @@ class ProtocolTest(unittest.TestCase): def test_complete_packet(self): """Protocol should assemble incoming lines into complete packet.""" - for line in TELEGRAM_V2_2: - self.protocol.data_received(bytes(line + '\r\n', 'ascii')) + self.protocol.data_received(TELEGRAM_V2_2.encode('ascii')) telegram = self.protocol.telegram_callback.call_args_list[0][0][0] assert isinstance(telegram, dict)