refactored TelegramParser.parse to accept a str instead of list

Browse Source
This commit is contained in:
Nigel Dokter 2017-01-05 21:24:41 +01:00
parent 1373d570d2
commit f10032f701
6 changed files with 180 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
dsmr_parser/parsers.py
View File

@ -26,48 +26,58 @@ class TelegramParser(object):
return None, None return None, None
def parse(self, line_values): def parse(self, telegram):
telegram = {} """
Parse telegram from string to dict.
for line_value in line_values: The telegram str type makes python 2.x integration easier.
# TODO temporarily strip newline characters.
line_value = line_value.strip()
obis_reference, dsmr_object = self.parse_line(line_value) :param str telegram: full telegram from start ('/') to checksum
('!ABCD') including line endings inbetween the telegram's lines
:rtype: dict
:returns: Shortened example:
{
..
r'0-0:96\.1\.1': <CosemObject>, # EQUIPMENT_IDENTIFIER
r'1-0:1\.8\.1': <CosemObject>, # ELECTRICITY_USED_TARIFF_1
r'0-\d:24\.3\.0': <MBusObject>, # GAS_METER_READING
..
}
"""
telegram_lines = telegram.splitlines()
parsed_lines = map(self.parse_line, telegram_lines)
telegram[obis_reference] = dsmr_object return {obis_reference: dsmr_object
for obis_reference, dsmr_object in parsed_lines}
return telegram def parse_line(self, line):
logger.debug("Parsing line '%s'", line)
def parse_line(self, line_value): obis_reference, parser = self._find_line_parser(line)
logger.debug('Parsing line \'%s\'', line_value)
obis_reference, parser = self._find_line_parser(line_value) if not obis_reference:
logger.debug("No line class found for: '%s'", line)
if not parser:
logger.warning("No line class found for: '%s'", line_value)
return None, None return None, None
return obis_reference, parser.parse(line_value) return obis_reference, parser.parse(line)
class TelegramParserV4(TelegramParser): class TelegramParserV4(TelegramParser):
@staticmethod @staticmethod
def validate_telegram_checksum(line_values): def validate_telegram_checksum(telegram):
""" """
:type line_values: list :param str telegram:
:raises ParseError: :raises ParseError:
:raises InvalidChecksumError: :raises InvalidChecksumError:
""" """
full_telegram = ''.join(line_values) # Extract the part for which the checksum applies.
checksum_contents = re.search(r'\/.+\!', telegram, re.DOTALL)
# Extract the bytes that count towards the checksum.
checksum_contents = re.search(r'\/.+\!', full_telegram, re.DOTALL)
# Extract the hexadecimal checksum value itself. # Extract the hexadecimal checksum value itself.
checksum_hex = re.search(r'((?<=\!)[0-9A-Z]{4}(?=\r\n))+', full_telegram) # The line ending '\r\n' for the checksum line can be ignored.
checksum_hex = re.search(r'((?<=\!)[0-9A-Z]{4})+', telegram)
if not checksum_contents or not checksum_hex: if not checksum_contents or not checksum_hex:
raise ParseError( raise ParseError(
@ -76,8 +86,7 @@ class TelegramParserV4(TelegramParser):
) )
calculated_crc = CRC16().calculate(checksum_contents.group(0)) calculated_crc = CRC16().calculate(checksum_contents.group(0))
expected_crc = checksum_hex.group(0) expected_crc = int(checksum_hex.group(0), base=16)
expected_crc = int(expected_crc, base=16)
if calculated_crc != expected_crc: if calculated_crc != expected_crc:
raise InvalidChecksumError( raise InvalidChecksumError(
@ -88,31 +97,44 @@ class TelegramParserV4(TelegramParser):
) )
) )
def parse(self, line_values): def parse(self, telegram):
self.validate_telegram_checksum(line_values) """
:param str telegram:
:rtype: dict
"""
self.validate_telegram_checksum(telegram)
return super().parse(line_values) return super().parse(telegram)
class TelegramParserV2_2(TelegramParser): class TelegramParserV2_2(TelegramParser):
def parse(self, line_values): def parse(self, telegram):
"""Join lines for gas meter.""" """
:param str telegram:
:rtype: dict
"""
def join_lines(line_values): # TODO fix this in the specification: telegram_specifications.V2_2
def join_lines(telegram):
"""Join lines for gas meter."""
join_next = re.compile(GAS_METER_READING) join_next = re.compile(GAS_METER_READING)
join = None join = None
for line_value in line_values: for line_value in telegram.splitlines():
if join: if join:
yield join.strip() + line_value yield join + line_value
join = None join = None
elif join_next.match(line_value): elif join_next.match(line_value):
join = line_value join = line_value
else: else:
yield line_value yield line_value
return super().parse(join_lines(line_values)) # TODO temporary workaround
lines = join_lines(telegram)
telegram = '\r\n'.join(lines)
return super().parse(telegram)
class DSMRObjectParser(object): class DSMRObjectParser(object):

16
dsmr_parser/protocol.py
View File

@ -24,6 +24,9 @@ def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None):
specifications = telegram_specifications.V4 specifications = telegram_specifications.V4
telegram_parser = TelegramParserV4 telegram_parser = TelegramParserV4
serial_settings = SERIAL_SETTINGS_V4 serial_settings = SERIAL_SETTINGS_V4
else:
raise NotImplementedError("No telegram parser found for version: %s",
dsmr_version)
protocol = partial(DSMRProtocol, loop, telegram_parser(specifications), protocol = partial(DSMRProtocol, loop, telegram_parser(specifications),
telegram_callback=telegram_callback) telegram_callback=telegram_callback)
@ -64,7 +67,7 @@ class DSMRProtocol(asyncio.Protocol):
# callback to call on complete telegram # callback to call on complete telegram
self.telegram_callback = telegram_callback self.telegram_callback = telegram_callback
# buffer to keep incoming telegram lines # buffer to keep incoming telegram lines
self.telegram = [] self.telegram = ''
# buffer to keep incomplete incoming data # buffer to keep incomplete incoming data
self.buffer = '' self.buffer = ''
# keep a lock until the connection is closed # keep a lock until the connection is closed
@ -77,7 +80,7 @@ class DSMRProtocol(asyncio.Protocol):
def data_received(self, data): def data_received(self, data):
"""Add incoming data to buffer.""" """Add incoming data to buffer."""
data = data.decode() data = data.decode('ascii')
self.log.debug('received data: %s', data.strip()) self.log.debug('received data: %s', data.strip())
self.buffer += data self.buffer += data
self.handle_lines() self.handle_lines()
@ -95,13 +98,16 @@ class DSMRProtocol(asyncio.Protocol):
if not self.telegram and not is_start_of_telegram(line): if not self.telegram and not is_start_of_telegram(line):
continue continue
self.telegram.append(line) self.telegram += line
if is_end_of_telegram(line): if is_end_of_telegram(line):
try: try:
parsed_telegram = self.telegram_parser.parse(self.telegram) parsed_telegram = self.telegram_parser.parse(self.telegram)
except ParseError as e:
self.log.error('Failed to parse telegram: %s', e)
else:
self.handle_telegram(parsed_telegram) self.handle_telegram(parsed_telegram)
except ParseError:
self.log.exception("failed to parse telegram")
self.telegram = [] self.telegram = []
def connection_lost(self, exc): def connection_lost(self, exc):

42
dsmr_parser/serial.py
View File

@ -33,14 +33,16 @@ SERIAL_SETTINGS_V4 = {
def is_start_of_telegram(line): def is_start_of_telegram(line):
""" """
:type line: line :param bytes line: series of bytes representing a line.
Example: b'/KFM5KAIFA-METER\r\n'
""" """
return line.startswith('/') return line.startswith('/')
def is_end_of_telegram(line): def is_end_of_telegram(line):
""" """
:type line: line :param bytes line: series of bytes representing a line.
Example: b'!7B05\r\n'
""" """
return line.startswith('!') return line.startswith('!')
@ -66,30 +68,28 @@ class SerialReader(object):
Read complete DSMR telegram's from the serial interface and parse it Read complete DSMR telegram's from the serial interface and parse it
into CosemObject's and MbusObject's into CosemObject's and MbusObject's
:rtype dict :rtype: generator
""" """
with serial.Serial(**self.serial_settings) as serial_handle: with serial.Serial(**self.serial_settings) as serial_handle:
telegram = [] telegram = ''
while True: while True:
line = serial_handle.readline() line = serial_handle.readline()
line = line.decode('ascii') # TODO move this to the parser? line.decode('ascii')
# Telegrams need to be complete because the values belong to a # Build up buffer from the start of the telegram.
# particular reading and can also be related to eachother.
if not telegram and not is_start_of_telegram(line): if not telegram and not is_start_of_telegram(line):
continue continue
telegram.append(line) telegram += line
if is_end_of_telegram(line): if is_end_of_telegram(line):
try: try:
yield self.telegram_parser.parse(telegram) yield self.telegram_parser.parse(telegram)
except ParseError as e: except ParseError as e:
logger.error('Failed to parse telegram: %s', e) logger.error('Failed to parse telegram: %s', e)
telegram = [] telegram = ''
class AsyncSerialReader(SerialReader): class AsyncSerialReader(SerialReader):
@ -106,33 +106,33 @@ class AsyncSerialReader(SerialReader):
Instead of being a generator, values are pushed to provided queue for Instead of being a generator, values are pushed to provided queue for
asynchronous processing. asynchronous processing.
:rtype Generator/Async :rtype: None
""" """
# create Serial StreamReader # create Serial StreamReader
conn = serial_asyncio.open_serial_connection(**self.serial_settings) conn = serial_asyncio.open_serial_connection(**self.serial_settings)
reader, _ = yield from conn reader, _ = yield from conn
telegram = [] telegram = ''
while True: while True:
# read line if available or give control back to loop until # Read line if available or give control back to loop until new
# new data has arrived # data has arrived.
line = yield from reader.readline() line = yield from reader.readline()
line = line.decode('ascii') line = line.decode('ascii')
# Telegrams need to be complete because the values belong to a # Build up buffer from the start of the telegram.
# particular reading and can also be related to eachother.
if not telegram and not is_start_of_telegram(line): if not telegram and not is_start_of_telegram(line):
continue continue
telegram.append(line) telegram += line
if is_end_of_telegram(line): if is_end_of_telegram(line):
try: try:
parsed_telegram = self.telegram_parser.parse(telegram) # Push new parsed telegram onto queue.
# push new parsed telegram onto queue queue.put_nowait(
queue.put_nowait(parsed_telegram) self.telegram_parser.parse(telegram)
)
except ParseError as e: except ParseError as e:
logger.warning('Failed to parse telegram: %s', e) logger.warning('Failed to parse telegram: %s', e)
telegram = [] telegram = ''

44
test/test_parse_v2_2.py
View File

@ -4,28 +4,28 @@ from dsmr_parser.parsers import TelegramParserV2_2
from dsmr_parser import telegram_specifications from dsmr_parser import telegram_specifications
from dsmr_parser import obis_references as obis from dsmr_parser import obis_references as obis
TELEGRAM_V2_2 = [ TELEGRAM_V2_2 = (
'/ISk5\2MT382-1004', '/ISk5\2MT382-1004\r\n'
'', '\r\n'
'0-0:96.1.1(00000000000000)', '0-0:96.1.1(00000000000000)\r\n'
'1-0:1.8.1(00001.001*kWh)', '1-0:1.8.1(00001.001*kWh)\r\n'
'1-0:1.8.2(00001.001*kWh)', '1-0:1.8.2(00001.001*kWh)\r\n'
'1-0:2.8.1(00001.001*kWh)', '1-0:2.8.1(00001.001*kWh)\r\n'
'1-0:2.8.2(00001.001*kWh)', '1-0:2.8.2(00001.001*kWh)\r\n'
'0-0:96.14.0(0001)', '0-0:96.14.0(0001)\r\n'
'1-0:1.7.0(0001.01*kW)', '1-0:1.7.0(0001.01*kW)\r\n'
'1-0:2.7.0(0000.00*kW)', '1-0:2.7.0(0000.00*kW)\r\n'
'0-0:17.0.0(0999.00*kW)', '0-0:17.0.0(0999.00*kW)\r\n'
'0-0:96.3.10(1)', '0-0:96.3.10(1)\r\n'
'0-0:96.13.1()', '0-0:96.13.1()\r\n'
'0-0:96.13.0()', '0-0:96.13.0()\r\n'
'0-1:24.1.0(3)', '0-1:24.1.0(3)\r\n'
'0-1:96.1.0(000000000000)', '0-1:96.1.0(000000000000)\r\n'
'0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)', '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n'
'(00001.001)', '(00001.001)\r\n'
'0-1:24.4.0(1)', '0-1:24.4.0(1)\r\n'
'!', '!\r\n'
] )
class TelegramParserV2_2Test(unittest.TestCase): class TelegramParserV2_2Test(unittest.TestCase):

93
test/test_parse_v4_2.py
View File

@ -10,45 +10,45 @@ from dsmr_parser.exceptions import InvalidChecksumError, ParseError
from dsmr_parser.objects import CosemObject, MBusObject from dsmr_parser.objects import CosemObject, MBusObject
from dsmr_parser.parsers import TelegramParser, TelegramParserV4 from dsmr_parser.parsers import TelegramParser, TelegramParserV4
TELEGRAM_V4_2 = [ TELEGRAM_V4_2 = (
'/KFM5KAIFA-METER\r\n', '/KFM5KAIFA-METER\r\n'
'\r\n', '\r\n'
'1-3:0.2.8(42)\r\n', '1-3:0.2.8(42)\r\n'
'0-0:1.0.0(161113205757W)\r\n', '0-0:1.0.0(161113205757W)\r\n'
'0-0:96.1.1(3960221976967177082151037881335713)\r\n', '0-0:96.1.1(3960221976967177082151037881335713)\r\n'
'1-0:1.8.1(001581.123*kWh)\r\n', '1-0:1.8.1(001581.123*kWh)\r\n'
'1-0:1.8.2(001435.706*kWh)\r\n', '1-0:1.8.2(001435.706*kWh)\r\n'
'1-0:2.8.1(000000.000*kWh)\r\n', '1-0:2.8.1(000000.000*kWh)\r\n'
'1-0:2.8.2(000000.000*kWh)\r\n', '1-0:2.8.2(000000.000*kWh)\r\n'
'0-0:96.14.0(0002)\r\n', '0-0:96.14.0(0002)\r\n'
'1-0:1.7.0(02.027*kW)\r\n', '1-0:1.7.0(02.027*kW)\r\n'
'1-0:2.7.0(00.000*kW)\r\n', '1-0:2.7.0(00.000*kW)\r\n'
'0-0:96.7.21(00015)\r\n', '0-0:96.7.21(00015)\r\n'
'0-0:96.7.9(00007)\r\n', '0-0:96.7.9(00007)\r\n'
'1-0:99.97.0(3)(0-0:96.7.19)(000104180320W)(0000237126*s)(000101000001W)' '1-0:99.97.0(3)(0-0:96.7.19)(000104180320W)(0000237126*s)(000101000001W)'
'(2147583646*s)(000102000003W)(2317482647*s)\r\n', '(2147583646*s)(000102000003W)(2317482647*s)\r\n'
'1-0:32.32.0(00000)\r\n', '1-0:32.32.0(00000)\r\n'
'1-0:52.32.0(00000)\r\n', '1-0:52.32.0(00000)\r\n'
'1-0:72.32.0(00000)\r\n', '1-0:72.32.0(00000)\r\n'
'1-0:32.36.0(00000)\r\n', '1-0:32.36.0(00000)\r\n'
'1-0:52.36.0(00000)\r\n', '1-0:52.36.0(00000)\r\n'
'1-0:72.36.0(00000)\r\n', '1-0:72.36.0(00000)\r\n'
'0-0:96.13.1()\r\n', '0-0:96.13.1()\r\n'
'0-0:96.13.0()\r\n', '0-0:96.13.0()\r\n'
'1-0:31.7.0(000*A)\r\n', '1-0:31.7.0(000*A)\r\n'
'1-0:51.7.0(006*A)\r\n', '1-0:51.7.0(006*A)\r\n'
'1-0:71.7.0(002*A)\r\n', '1-0:71.7.0(002*A)\r\n'
'1-0:21.7.0(00.170*kW)\r\n', '1-0:21.7.0(00.170*kW)\r\n'
'1-0:22.7.0(00.000*kW)\r\n', '1-0:22.7.0(00.000*kW)\r\n'
'1-0:41.7.0(01.247*kW)\r\n', '1-0:41.7.0(01.247*kW)\r\n'
'1-0:42.7.0(00.000*kW)\r\n', '1-0:42.7.0(00.000*kW)\r\n'
'1-0:61.7.0(00.209*kW)\r\n', '1-0:61.7.0(00.209*kW)\r\n'
'1-0:62.7.0(00.000*kW)\r\n', '1-0:62.7.0(00.000*kW)\r\n'
'0-1:24.1.0(003)\r\n', '0-1:24.1.0(003)\r\n'
'0-1:96.1.0(4819243993373755377509728609491464)\r\n', '0-1:96.1.0(4819243993373755377509728609491464)\r\n'
'0-1:24.2.1(161129200000W)(00981.443*m3)\r\n', '0-1:24.2.1(161129200000W)(00981.443*m3)\r\n'
'!6796\r\n' '!6796\r\n'
] )
class TelegramParserV4_2Test(unittest.TestCase): class TelegramParserV4_2Test(unittest.TestCase):
@ -56,26 +56,25 @@ class TelegramParserV4_2Test(unittest.TestCase):
def test_valid(self): def test_valid(self):
# No exception is raised. # No exception is raised.
TelegramParserV4.validate_telegram_checksum( TelegramParserV4.validate_telegram_checksum(TELEGRAM_V4_2)
TELEGRAM_V4_2
)
def test_invalid(self): def test_invalid(self):
# Remove one the electricty used data value. This causes the checksum to # Remove the electricty used data value. This causes the checksum to
# not match anymore. # not match anymore.
telegram = [line corrupted_telegram = TELEGRAM_V4_2.replace(
for line in TELEGRAM_V4_2 '1-0:1.8.1(001581.123*kWh)\r\n',
if '1-0:1.8.1' not in line] ''
)
with self.assertRaises(InvalidChecksumError): with self.assertRaises(InvalidChecksumError):
TelegramParserV4.validate_telegram_checksum(telegram) TelegramParserV4.validate_telegram_checksum(corrupted_telegram)
def test_missing_checksum(self): def test_missing_checksum(self):
# Remove the checksum value causing a ParseError. # Remove the checksum value causing a ParseError.
telegram = TELEGRAM_V4_2[:-1] corrupted_telegram = TELEGRAM_V4_2.replace('!6796\r\n', '')
with self.assertRaises(ParseError): with self.assertRaises(ParseError):
TelegramParserV4.validate_telegram_checksum(telegram) TelegramParserV4.validate_telegram_checksum(corrupted_telegram)
def test_parse(self): def test_parse(self):
parser = TelegramParser(telegram_specifications.V4) parser = TelegramParser(telegram_specifications.V4)

51
test/test_protocol.py
View File

@ -1,35 +1,35 @@
"""Test DSMR serial protocol."""
import unittest
from unittest.mock import Mock from unittest.mock import Mock
import unittest
from dsmr_parser import obis_references as obis from dsmr_parser import obis_references as obis
from dsmr_parser import telegram_specifications from dsmr_parser import telegram_specifications
from dsmr_parser.parsers import TelegramParserV2_2 from dsmr_parser.parsers import TelegramParserV2_2
from dsmr_parser.protocol import DSMRProtocol from dsmr_parser.protocol import DSMRProtocol
TELEGRAM_V2_2 = [ TELEGRAM_V2_2 = (
"/ISk5\2MT382-1004", '/ISk5\2MT382-1004\r\n'
"", '\r\n'
"0-0:96.1.1(00000000000000)", '0-0:96.1.1(00000000000000)\r\n'
"1-0:1.8.1(00001.001*kWh)", '1-0:1.8.1(00001.001*kWh)\r\n'
"1-0:1.8.2(00001.001*kWh)", '1-0:1.8.2(00001.001*kWh)\r\n'
"1-0:2.8.1(00001.001*kWh)", '1-0:2.8.1(00001.001*kWh)\r\n'
"1-0:2.8.2(00001.001*kWh)", '1-0:2.8.2(00001.001*kWh)\r\n'
"0-0:96.14.0(0001)", '0-0:96.14.0(0001)\r\n'
"1-0:1.7.0(0001.01*kW)", '1-0:1.7.0(0001.01*kW)\r\n'
"1-0:2.7.0(0000.00*kW)", '1-0:2.7.0(0000.00*kW)\r\n'
"0-0:17.0.0(0999.00*kW)", '0-0:17.0.0(0999.00*kW)\r\n'
"0-0:96.3.10(1)", '0-0:96.3.10(1)\r\n'
"0-0:96.13.1()", '0-0:96.13.1()\r\n'
"0-0:96.13.0()", '0-0:96.13.0()\r\n'
"0-1:24.1.0(3)", '0-1:24.1.0(3)\r\n'
"0-1:96.1.0(000000000000)", '0-1:96.1.0(000000000000)\r\n'
"0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)", '0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n'
"(00001.001)", '(00001.001)\r\n'
"0-1:24.4.0(1)", '0-1:24.4.0(1)\r\n'
"!", '!\r\n'
] )
class ProtocolTest(unittest.TestCase): class ProtocolTest(unittest.TestCase):
@ -45,8 +45,7 @@ class ProtocolTest(unittest.TestCase):
def test_complete_packet(self): def test_complete_packet(self):
"""Protocol should assemble incoming lines into complete packet.""" """Protocol should assemble incoming lines into complete packet."""
for line in TELEGRAM_V2_2: self.protocol.data_received(TELEGRAM_V2_2.encode('ascii'))
self.protocol.data_received(bytes(line + '\r\n', 'ascii'))
telegram = self.protocol.telegram_callback.call_args_list[0][0][0] telegram = self.protocol.telegram_callback.call_args_list[0][0][0]
assert isinstance(telegram, dict) assert isinstance(telegram, dict)