refactored TelegramParser.parse to accept a str instead of list

This commit is contained in:
Nigel Dokter 2017-01-05 21:24:41 +01:00
parent 1373d570d2
commit f10032f701
6 changed files with 180 additions and 154 deletions

View File

@ -26,48 +26,58 @@ class TelegramParser(object):
return None, None
def parse(self, line_values):
telegram = {}
def parse(self, telegram):
"""
Parse telegram from string to dict.
for line_value in line_values:
# TODO temporarily strip newline characters.
line_value = line_value.strip()
The telegram str type makes python 2.x integration easier.
obis_reference, dsmr_object = self.parse_line(line_value)
:param str telegram: full telegram from start ('/') to checksum
('!ABCD') including line endings inbetween the telegram's lines
:rtype: dict
:returns: Shortened example:
{
..
r'0-0:96\.1\.1': <CosemObject>, # EQUIPMENT_IDENTIFIER
r'1-0:1\.8\.1': <CosemObject>, # ELECTRICITY_USED_TARIFF_1
r'0-\d:24\.3\.0': <MBusObject>, # GAS_METER_READING
..
}
"""
telegram_lines = telegram.splitlines()
parsed_lines = map(self.parse_line, telegram_lines)
telegram[obis_reference] = dsmr_object
return {obis_reference: dsmr_object
for obis_reference, dsmr_object in parsed_lines}
return telegram
def parse_line(self, line):
logger.debug("Parsing line '%s'", line)
def parse_line(self, line_value):
logger.debug('Parsing line \'%s\'', line_value)
obis_reference, parser = self._find_line_parser(line)
obis_reference, parser = self._find_line_parser(line_value)
if not parser:
logger.warning("No line class found for: '%s'", line_value)
if not obis_reference:
logger.debug("No line class found for: '%s'", line)
return None, None
return obis_reference, parser.parse(line_value)
return obis_reference, parser.parse(line)
class TelegramParserV4(TelegramParser):
@staticmethod
def validate_telegram_checksum(line_values):
def validate_telegram_checksum(telegram):
"""
:type line_values: list
:param str telegram:
:raises ParseError:
:raises InvalidChecksumError:
"""
full_telegram = ''.join(line_values)
# Extract the bytes that count towards the checksum.
checksum_contents = re.search(r'\/.+\!', full_telegram, re.DOTALL)
# Extract the part for which the checksum applies.
checksum_contents = re.search(r'\/.+\!', telegram, re.DOTALL)
# Extract the hexadecimal checksum value itself.
checksum_hex = re.search(r'((?<=\!)[0-9A-Z]{4}(?=\r\n))+', full_telegram)
# The line ending '\r\n' for the checksum line can be ignored.
checksum_hex = re.search(r'((?<=\!)[0-9A-Z]{4})+', telegram)
if not checksum_contents or not checksum_hex:
raise ParseError(
@ -76,8 +86,7 @@ class TelegramParserV4(TelegramParser):
)
calculated_crc = CRC16().calculate(checksum_contents.group(0))
expected_crc = checksum_hex.group(0)
expected_crc = int(expected_crc, base=16)
expected_crc = int(checksum_hex.group(0), base=16)
if calculated_crc != expected_crc:
raise InvalidChecksumError(
@ -88,31 +97,44 @@ class TelegramParserV4(TelegramParser):
)
)
def parse(self, line_values):
self.validate_telegram_checksum(line_values)
def parse(self, telegram):
"""
:param str telegram:
:rtype: dict
"""
self.validate_telegram_checksum(telegram)
return super().parse(line_values)
return super().parse(telegram)
class TelegramParserV2_2(TelegramParser):
def parse(self, line_values):
"""Join lines for gas meter."""
def parse(self, telegram):
"""
:param str telegram:
:rtype: dict
"""
def join_lines(line_values):
# TODO fix this in the specification: telegram_specifications.V2_2
def join_lines(telegram):
"""Join lines for gas meter."""
join_next = re.compile(GAS_METER_READING)
join = None
for line_value in line_values:
for line_value in telegram.splitlines():
if join:
yield join.strip() + line_value
yield join + line_value
join = None
elif join_next.match(line_value):
join = line_value
else:
yield line_value
return super().parse(join_lines(line_values))
# TODO temporary workaround
lines = join_lines(telegram)
telegram = '\r\n'.join(lines)
return super().parse(telegram)
class DSMRObjectParser(object):

View File

@ -24,6 +24,9 @@ def create_dsmr_protocol(dsmr_version, telegram_callback, loop=None):
specifications = telegram_specifications.V4
telegram_parser = TelegramParserV4
serial_settings = SERIAL_SETTINGS_V4
else:
raise NotImplementedError("No telegram parser found for version: %s",
dsmr_version)
protocol = partial(DSMRProtocol, loop, telegram_parser(specifications),
telegram_callback=telegram_callback)
@ -64,7 +67,7 @@ class DSMRProtocol(asyncio.Protocol):
# callback to call on complete telegram
self.telegram_callback = telegram_callback
# buffer to keep incoming telegram lines
self.telegram = []
self.telegram = ''
# buffer to keep incomplete incoming data
self.buffer = ''
# keep a lock until the connection is closed
@ -77,7 +80,7 @@ class DSMRProtocol(asyncio.Protocol):
def data_received(self, data):
"""Add incoming data to buffer."""
data = data.decode()
data = data.decode('ascii')
self.log.debug('received data: %s', data.strip())
self.buffer += data
self.handle_lines()
@ -95,13 +98,16 @@ class DSMRProtocol(asyncio.Protocol):
if not self.telegram and not is_start_of_telegram(line):
continue
self.telegram.append(line)
self.telegram += line
if is_end_of_telegram(line):
try:
parsed_telegram = self.telegram_parser.parse(self.telegram)
except ParseError as e:
self.log.error('Failed to parse telegram: %s', e)
else:
self.handle_telegram(parsed_telegram)
except ParseError:
self.log.exception("failed to parse telegram")
self.telegram = []
def connection_lost(self, exc):

View File

@ -33,14 +33,16 @@ SERIAL_SETTINGS_V4 = {
def is_start_of_telegram(line):
"""
:type line: line
:param bytes line: series of bytes representing a line.
Example: b'/KFM5KAIFA-METER\r\n'
"""
return line.startswith('/')
def is_end_of_telegram(line):
"""
:type line: line
:param bytes line: series of bytes representing a line.
Example: b'!7B05\r\n'
"""
return line.startswith('!')
@ -66,30 +68,28 @@ class SerialReader(object):
Read complete DSMR telegram's from the serial interface and parse it
into CosemObject's and MbusObject's
:rtype dict
:rtype: generator
"""
with serial.Serial(**self.serial_settings) as serial_handle:
telegram = []
telegram = ''
while True:
line = serial_handle.readline()
line = line.decode('ascii') # TODO move this to the parser?
line.decode('ascii')
# Telegrams need to be complete because the values belong to a
# particular reading and can also be related to eachother.
# Build up buffer from the start of the telegram.
if not telegram and not is_start_of_telegram(line):
continue
telegram.append(line)
telegram += line
if is_end_of_telegram(line):
try:
yield self.telegram_parser.parse(telegram)
except ParseError as e:
logger.error('Failed to parse telegram: %s', e)
telegram = []
telegram = ''
class AsyncSerialReader(SerialReader):
@ -106,33 +106,33 @@ class AsyncSerialReader(SerialReader):
Instead of being a generator, values are pushed to provided queue for
asynchronous processing.
:rtype Generator/Async
:rtype: None
"""
# create Serial StreamReader
conn = serial_asyncio.open_serial_connection(**self.serial_settings)
reader, _ = yield from conn
telegram = []
telegram = ''
while True:
# read line if available or give control back to loop until
# new data has arrived
# Read line if available or give control back to loop until new
# data has arrived.
line = yield from reader.readline()
line = line.decode('ascii')
# Telegrams need to be complete because the values belong to a
# particular reading and can also be related to eachother.
# Build up buffer from the start of the telegram.
if not telegram and not is_start_of_telegram(line):
continue
telegram.append(line)
telegram += line
if is_end_of_telegram(line):
try:
parsed_telegram = self.telegram_parser.parse(telegram)
# push new parsed telegram onto queue
queue.put_nowait(parsed_telegram)
# Push new parsed telegram onto queue.
queue.put_nowait(
self.telegram_parser.parse(telegram)
)
except ParseError as e:
logger.warning('Failed to parse telegram: %s', e)
telegram = []
telegram = ''

View File

@ -4,28 +4,28 @@ from dsmr_parser.parsers import TelegramParserV2_2
from dsmr_parser import telegram_specifications
from dsmr_parser import obis_references as obis
TELEGRAM_V2_2 = [
'/ISk5\2MT382-1004',
'',
'0-0:96.1.1(00000000000000)',
'1-0:1.8.1(00001.001*kWh)',
'1-0:1.8.2(00001.001*kWh)',
'1-0:2.8.1(00001.001*kWh)',
'1-0:2.8.2(00001.001*kWh)',
'0-0:96.14.0(0001)',
'1-0:1.7.0(0001.01*kW)',
'1-0:2.7.0(0000.00*kW)',
'0-0:17.0.0(0999.00*kW)',
'0-0:96.3.10(1)',
'0-0:96.13.1()',
'0-0:96.13.0()',
'0-1:24.1.0(3)',
'0-1:96.1.0(000000000000)',
'0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)',
'(00001.001)',
'0-1:24.4.0(1)',
'!',
]
TELEGRAM_V2_2 = (
'/ISk5\2MT382-1004\r\n'
'\r\n'
'0-0:96.1.1(00000000000000)\r\n'
'1-0:1.8.1(00001.001*kWh)\r\n'
'1-0:1.8.2(00001.001*kWh)\r\n'
'1-0:2.8.1(00001.001*kWh)\r\n'
'1-0:2.8.2(00001.001*kWh)\r\n'
'0-0:96.14.0(0001)\r\n'
'1-0:1.7.0(0001.01*kW)\r\n'
'1-0:2.7.0(0000.00*kW)\r\n'
'0-0:17.0.0(0999.00*kW)\r\n'
'0-0:96.3.10(1)\r\n'
'0-0:96.13.1()\r\n'
'0-0:96.13.0()\r\n'
'0-1:24.1.0(3)\r\n'
'0-1:96.1.0(000000000000)\r\n'
'0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n'
'(00001.001)\r\n'
'0-1:24.4.0(1)\r\n'
'!\r\n'
)
class TelegramParserV2_2Test(unittest.TestCase):

View File

@ -10,45 +10,45 @@ from dsmr_parser.exceptions import InvalidChecksumError, ParseError
from dsmr_parser.objects import CosemObject, MBusObject
from dsmr_parser.parsers import TelegramParser, TelegramParserV4
TELEGRAM_V4_2 = [
'/KFM5KAIFA-METER\r\n',
'\r\n',
'1-3:0.2.8(42)\r\n',
'0-0:1.0.0(161113205757W)\r\n',
'0-0:96.1.1(3960221976967177082151037881335713)\r\n',
'1-0:1.8.1(001581.123*kWh)\r\n',
'1-0:1.8.2(001435.706*kWh)\r\n',
'1-0:2.8.1(000000.000*kWh)\r\n',
'1-0:2.8.2(000000.000*kWh)\r\n',
'0-0:96.14.0(0002)\r\n',
'1-0:1.7.0(02.027*kW)\r\n',
'1-0:2.7.0(00.000*kW)\r\n',
'0-0:96.7.21(00015)\r\n',
'0-0:96.7.9(00007)\r\n',
TELEGRAM_V4_2 = (
'/KFM5KAIFA-METER\r\n'
'\r\n'
'1-3:0.2.8(42)\r\n'
'0-0:1.0.0(161113205757W)\r\n'
'0-0:96.1.1(3960221976967177082151037881335713)\r\n'
'1-0:1.8.1(001581.123*kWh)\r\n'
'1-0:1.8.2(001435.706*kWh)\r\n'
'1-0:2.8.1(000000.000*kWh)\r\n'
'1-0:2.8.2(000000.000*kWh)\r\n'
'0-0:96.14.0(0002)\r\n'
'1-0:1.7.0(02.027*kW)\r\n'
'1-0:2.7.0(00.000*kW)\r\n'
'0-0:96.7.21(00015)\r\n'
'0-0:96.7.9(00007)\r\n'
'1-0:99.97.0(3)(0-0:96.7.19)(000104180320W)(0000237126*s)(000101000001W)'
'(2147583646*s)(000102000003W)(2317482647*s)\r\n',
'1-0:32.32.0(00000)\r\n',
'1-0:52.32.0(00000)\r\n',
'1-0:72.32.0(00000)\r\n',
'1-0:32.36.0(00000)\r\n',
'1-0:52.36.0(00000)\r\n',
'1-0:72.36.0(00000)\r\n',
'0-0:96.13.1()\r\n',
'0-0:96.13.0()\r\n',
'1-0:31.7.0(000*A)\r\n',
'1-0:51.7.0(006*A)\r\n',
'1-0:71.7.0(002*A)\r\n',
'1-0:21.7.0(00.170*kW)\r\n',
'1-0:22.7.0(00.000*kW)\r\n',
'1-0:41.7.0(01.247*kW)\r\n',
'1-0:42.7.0(00.000*kW)\r\n',
'1-0:61.7.0(00.209*kW)\r\n',
'1-0:62.7.0(00.000*kW)\r\n',
'0-1:24.1.0(003)\r\n',
'0-1:96.1.0(4819243993373755377509728609491464)\r\n',
'0-1:24.2.1(161129200000W)(00981.443*m3)\r\n',
'(2147583646*s)(000102000003W)(2317482647*s)\r\n'
'1-0:32.32.0(00000)\r\n'
'1-0:52.32.0(00000)\r\n'
'1-0:72.32.0(00000)\r\n'
'1-0:32.36.0(00000)\r\n'
'1-0:52.36.0(00000)\r\n'
'1-0:72.36.0(00000)\r\n'
'0-0:96.13.1()\r\n'
'0-0:96.13.0()\r\n'
'1-0:31.7.0(000*A)\r\n'
'1-0:51.7.0(006*A)\r\n'
'1-0:71.7.0(002*A)\r\n'
'1-0:21.7.0(00.170*kW)\r\n'
'1-0:22.7.0(00.000*kW)\r\n'
'1-0:41.7.0(01.247*kW)\r\n'
'1-0:42.7.0(00.000*kW)\r\n'
'1-0:61.7.0(00.209*kW)\r\n'
'1-0:62.7.0(00.000*kW)\r\n'
'0-1:24.1.0(003)\r\n'
'0-1:96.1.0(4819243993373755377509728609491464)\r\n'
'0-1:24.2.1(161129200000W)(00981.443*m3)\r\n'
'!6796\r\n'
]
)
class TelegramParserV4_2Test(unittest.TestCase):
@ -56,26 +56,25 @@ class TelegramParserV4_2Test(unittest.TestCase):
def test_valid(self):
# No exception is raised.
TelegramParserV4.validate_telegram_checksum(
TELEGRAM_V4_2
)
TelegramParserV4.validate_telegram_checksum(TELEGRAM_V4_2)
def test_invalid(self):
# Remove one the electricty used data value. This causes the checksum to
# Remove the electricty used data value. This causes the checksum to
# not match anymore.
telegram = [line
for line in TELEGRAM_V4_2
if '1-0:1.8.1' not in line]
corrupted_telegram = TELEGRAM_V4_2.replace(
'1-0:1.8.1(001581.123*kWh)\r\n',
''
)
with self.assertRaises(InvalidChecksumError):
TelegramParserV4.validate_telegram_checksum(telegram)
TelegramParserV4.validate_telegram_checksum(corrupted_telegram)
def test_missing_checksum(self):
# Remove the checksum value causing a ParseError.
telegram = TELEGRAM_V4_2[:-1]
corrupted_telegram = TELEGRAM_V4_2.replace('!6796\r\n', '')
with self.assertRaises(ParseError):
TelegramParserV4.validate_telegram_checksum(telegram)
TelegramParserV4.validate_telegram_checksum(corrupted_telegram)
def test_parse(self):
parser = TelegramParser(telegram_specifications.V4)

View File

@ -1,35 +1,35 @@
"""Test DSMR serial protocol."""
import unittest
from unittest.mock import Mock
import unittest
from dsmr_parser import obis_references as obis
from dsmr_parser import telegram_specifications
from dsmr_parser.parsers import TelegramParserV2_2
from dsmr_parser.protocol import DSMRProtocol
TELEGRAM_V2_2 = [
"/ISk5\2MT382-1004",
"",
"0-0:96.1.1(00000000000000)",
"1-0:1.8.1(00001.001*kWh)",
"1-0:1.8.2(00001.001*kWh)",
"1-0:2.8.1(00001.001*kWh)",
"1-0:2.8.2(00001.001*kWh)",
"0-0:96.14.0(0001)",
"1-0:1.7.0(0001.01*kW)",
"1-0:2.7.0(0000.00*kW)",
"0-0:17.0.0(0999.00*kW)",
"0-0:96.3.10(1)",
"0-0:96.13.1()",
"0-0:96.13.0()",
"0-1:24.1.0(3)",
"0-1:96.1.0(000000000000)",
"0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)",
"(00001.001)",
"0-1:24.4.0(1)",
"!",
]
TELEGRAM_V2_2 = (
'/ISk5\2MT382-1004\r\n'
'\r\n'
'0-0:96.1.1(00000000000000)\r\n'
'1-0:1.8.1(00001.001*kWh)\r\n'
'1-0:1.8.2(00001.001*kWh)\r\n'
'1-0:2.8.1(00001.001*kWh)\r\n'
'1-0:2.8.2(00001.001*kWh)\r\n'
'0-0:96.14.0(0001)\r\n'
'1-0:1.7.0(0001.01*kW)\r\n'
'1-0:2.7.0(0000.00*kW)\r\n'
'0-0:17.0.0(0999.00*kW)\r\n'
'0-0:96.3.10(1)\r\n'
'0-0:96.13.1()\r\n'
'0-0:96.13.0()\r\n'
'0-1:24.1.0(3)\r\n'
'0-1:96.1.0(000000000000)\r\n'
'0-1:24.3.0(161107190000)(00)(60)(1)(0-1:24.2.1)(m3)\r\n'
'(00001.001)\r\n'
'0-1:24.4.0(1)\r\n'
'!\r\n'
)
class ProtocolTest(unittest.TestCase):
@ -45,8 +45,7 @@ class ProtocolTest(unittest.TestCase):
def test_complete_packet(self):
"""Protocol should assemble incoming lines into complete packet."""
for line in TELEGRAM_V2_2:
self.protocol.data_received(bytes(line + '\r\n', 'ascii'))
self.protocol.data_received(TELEGRAM_V2_2.encode('ascii'))
telegram = self.protocol.telegram_callback.call_args_list[0][0][0]
assert isinstance(telegram, dict)