Skill Tests
Testing is an important part of skill development. Here are some recommendations for how to implement good tests for a skill.
Unit Tests
Like any other module, testing each method in you skill can help prevent debugging headaches when something goes wrong. These tests are specific to a skill and it is up to the skill author to test that methods act as expected.
OSM Installation Tests
This test case may be used to test skill installation via OSM:
import unittest
from os.path import exists
from shutil import rmtree
from ovos_skills_manager import SkillEntry
branch = "dev" # TODO: Specify default branch
repo = "skill-ovos-volume" # TODO: Specify repository name
author = "OpenVoiceOS" # TODO: Specify repository author
url = f"https://github.com/{author}/{repo}@{branch}"
class TestOSM(unittest.TestCase):
@classmethod
def setUpClass(self):
self.skill_id = "{repo.lower()}.{author.lower()}"
def test_osm_install(self):
skill = SkillEntry.from_github_url(url)
tmp_skills = "/tmp/osm_installed_skills"
skill_folder = f"{tmp_skills}/{skill.uuid}"
if exists(skill_folder):
rmtree(skill_folder)
updated = skill.install(folder=tmp_skills, default_branch=branch)
self.assertEqual(updated, True)
self.assertTrue(exists(skill_folder))
updated = skill.install(folder=tmp_skills, default_branch=branch)
self.assertEqual(updated, False)
Skill Load Tests
This test case may be used to test skill resource files and event registration. Note that intent names, resource files, supported languages, and the skill class must be manually specified in the test class.
class TestSkillLoading(unittest.TestCase):
"""
Test skill loading, intent registration, and langauge support. Test cases
are generic, only class variables should be modified per-skill.
"""
# Static parameters
bus = FakeBus()
messages = list()
test_skill_id = 'test_skill.test'
# Default Core Events
default_events = ["mycroft.skill.enable_intent",
"mycroft.skill.disable_intent",
"mycroft.skill.set_cross_context",
"mycroft.skill.remove_cross_context",
"intent.service.skills.deactivated",
"intent.service.skills.activated",
"mycroft.skills.settings.changed",
"skill.converse.ping",
"skill.converse.request",
f"{test_skill_id}.activate",
f"{test_skill_id}.deactivate"
]
# Import and initialize installed skill
from skill_ip_address import IPSkill # TODO: Specify skill class
skill = IPSkill()
# Specify valid languages to test
supported_languages = ["en-us"]
# Specify skill intents as sets
adapt_intents = {'IPIntent'}
padatious_intents = set()
# regex entities, not necessarily filenames
regex = set()
# vocab is lowercase .voc file basenames
vocab = {"query", "ip", "public"}
# dialog is .dialog file basenames (case-sensitive)
dialog = {"dot", "no network connection", "my address is",
"my address on X is Y"}
@classmethod
def setUpClass(cls) -> None:
cls.bus.on("message", cls._on_message)
cls.skill.config_core["secondary_langs"] = cls.supported_languages
cls.skill._startup(cls.bus, cls.test_skill_id)
cls.adapt_intents = {f'{cls.test_skill_id}:{intent}'
for intent in cls.adapt_intents}
cls.padatious_intents = {f'{cls.test_skill_id}:{intent}'
for intent in cls.padatious_intents}
@classmethod
def _on_message(cls, message):
cls.messages.append(json.loads(message))
def test_skill_setup(self):
self.assertEqual(self.skill.skill_id, self.test_skill_id)
for msg in self.messages:
self.assertEqual(msg["context"]["skill_id"], self.test_skill_id)
def test_intent_registration(self):
registered_adapt = list()
registered_padatious = dict()
registered_vocab = dict()
registered_regex = dict()
for msg in self.messages:
if msg["type"] == "register_intent":
registered_adapt.append(msg["data"]["name"])
elif msg["type"] == "padatious:register_intent":
lang = msg["data"]["lang"]
registered_padatious.setdefault(lang, list())
registered_padatious[lang].append(msg["data"]["name"])
elif msg["type"] == "register_vocab":
lang = msg["data"]["lang"]
if msg['data'].get('regex'):
registered_regex.setdefault(lang, dict())
regex = msg["data"]["regex"].split(
'<', 1)[1].split('>', 1)[0].replace(
self.test_skill_id.replace('.', '_'), '').lower()
registered_regex[lang].setdefault(regex, list())
registered_regex[lang][regex].append(msg["data"]["regex"])
else:
registered_vocab.setdefault(lang, dict())
voc_filename = msg["data"]["entity_type"].replace(
self.test_skill_id.replace('.', '_'), '').lower()
registered_vocab[lang].setdefault(voc_filename, list())
registered_vocab[lang][voc_filename].append(
msg["data"]["entity_value"])
self.assertEqual(set(registered_adapt), self.adapt_intents)
for lang in self.supported_languages:
if self.padatious_intents:
self.assertEqual(set(registered_padatious[lang]),
self.padatious_intents)
if self.vocab:
self.assertEqual(set(registered_vocab[lang].keys()), self.vocab)
if self.regex:
self.assertEqual(set(registered_regex[lang].keys()), self.regex)
for voc in self.vocab:
# Ensure every vocab file has at least one entry
self.assertGreater(len(registered_vocab[lang][voc]), 0)
for rx in self.regex:
# Ensure every vocab file has exactly one entry
self.assertTrue(all((rx in line for line in
registered_regex[lang][rx])))
def test_skill_events(self):
events = self.default_events + list(self.adapt_intents)
for event in events:
self.assertIn(event, [e[0] for e in self.skill.events])
def test_dialog_files(self):
for lang in self.supported_languages:
for dialog in self.dialog:
file = self.skill.find_resource(f"{dialog}.dialog", "dialog",
lang)
self.assertTrue(os.path.isfile(file))
Intent Tests
The following test case may be added to test intent matches:
class TestSkillIntentMatching(unittest.TestCase):
# Import and initialize installed skill
from skill_ip_address import IPSkill
skill = IPSkill()
import yaml
test_intents = join(dirname(__file__), 'test_intents.yaml')
with open(test_intents) as f:
valid_intents = yaml.safe_load(f)
from mycroft.skills.intent_service import IntentService
bus = FakeBus()
intent_service = IntentService(bus)
test_skill_id = 'test_skill.test'
@classmethod
def setUpClass(cls) -> None:
cls.skill.config_core["secondary_langs"] = list(cls.valid_intents.keys())
cls.skill._startup(cls.bus, cls.test_skill_id)
def test_intents(self):
for lang in self.valid_intents.keys():
for intent, examples in self.valid_intents[lang].items():
intent_event = f'{self.test_skill_id}:{intent}'
self.skill.events.remove(intent_event)
intent_handler = Mock()
self.skill.events.add(intent_event, intent_handler)
for utt in examples:
if isinstance(utt, dict):
data = list(utt.values())[0]
utt = list(utt.keys())[0]
else:
data = list()
message = Message('test_utterance',
{"utterances": [utt], "lang": lang})
self.intent_service.handle_utterance(message)
intent_handler.assert_called_once()
intent_message = intent_handler.call_args[0][0]
self.assertIsInstance(intent_message, Message)
self.assertEqual(intent_message.msg_type, intent_event)
for datum in data:
if isinstance(datum, dict):
name = list(datum.keys())[0]
value = list(datum.values())[0]
else:
name = datum
value = None
if name in intent_message.data:
# This is an entity
voc_id = name
else:
# We mocked the handler, data is munged
voc_id = f'{self.test_skill_id.replace(".", "_")}' \
f'{name}'
self.assertIsInstance(intent_message.data.get(voc_id),
str, intent_message.data)
if value:
self.assertEqual(intent_message.data.get(voc_id),
value)
intent_handler.reset_mock()
test_intents.yaml
The intent tests may be specified a few ways in test_intents.yaml:
# Specify intents to test here. Valid test cases are as follows:
# Basic intent match tests only:
#lang:
# intent_name:
# - example utterance
# - other example utterance
# Intent tests with expected vocab/entity matches:
#lang:
# intent_name:
# - example_utterance:
# - expected vocab name
# - other expected vocab name
# Intent tests with specific vocab/entity extraction tests:
#lang:
# intent_name:
# - example_utterance:
# - expected_vocab_key: expected_vocab_value
# - expected_entity_key: expected_entity_value
en-us:
IPIntent:
- what is your ip address
- what is my ip address:
- IP
- what is my i.p. address
- What is your I.P. address?
- what is my public IP address?:
- public: public