Additional flow-specific test exceptions, reorg

* detects when `View.run_screen` did not execute.
* detects when the supplied button_data_selection isn't a valid option in `Screen.button_data`.
* Renamed internal mock functions in `FlowTest.run_sequence()` for better clarity.
This commit is contained in:
kdmukai
2023-07-07 08:00:12 -05:00
parent 13537471f7
commit 9b3959050c
4 changed files with 143 additions and 79 deletions
+4 -6
View File
@@ -40,10 +40,9 @@ class StopFlowBasedTest(Exception):
pass pass
class FlowBasedTestUnexpectedViewError(Exception): class FlowBasedTestException(Exception):
""" """
This is a special exception that is only raised by the test suite to indicate This is a special exception that is only raised by the test suite.
that the Controller's current View does not match the expected TestFlow sequence.
It should not be raised by any other code. It should not be raised by any other code.
""" """
pass pass
@@ -276,9 +275,8 @@ class Controller(Singleton):
# to stop the Controller loop and exit the test. # to stop the Controller loop and exit the test.
return return
except FlowBasedTestUnexpectedViewError as e: except FlowBasedTestException as e:
# This is a special exception that is only raised by the test suite # This is a special exception that is only raised by the test suite.
# when a flow-based test fails to follow the expected sequence.
# Re-raise so the test suite can handle it. # Re-raise so the test suite can handle it.
raise e raise e
+3 -2
View File
@@ -814,9 +814,10 @@ class SeedWordsWarningView(View):
# Forward straight to showing the words # Forward straight to showing the words
return destination return destination
selected_menu_num = DireWarningScreen( selected_menu_num = self.run_screen(
DireWarningScreen,
text="""You must keep your seed words private & away from all online devices.""", text="""You must keep your seed words private & away from all online devices.""",
).display() )
if selected_menu_num == 0: if selected_menu_num == 0:
# User clicked "I Understand" # User clicked "I Understand"
+92 -60
View File
@@ -12,7 +12,8 @@ sys.modules['seedsigner.hardware.buttons'] = MagicMock()
sys.modules['seedsigner.hardware.camera'] = MagicMock() sys.modules['seedsigner.hardware.camera'] = MagicMock()
sys.modules['seedsigner.hardware.microsd'] = MagicMock() sys.modules['seedsigner.hardware.microsd'] = MagicMock()
from seedsigner.controller import Controller, FlowBasedTestUnexpectedViewError, StopFlowBasedTest from seedsigner.controller import Controller, FlowBasedTestException, StopFlowBasedTest
from seedsigner.gui.screens.screen import RET_CODE__BACK_BUTTON, RET_CODE__POWER_BUTTON
from seedsigner.models import Settings from seedsigner.models import Settings
from seedsigner.views.view import Destination, MainMenuView, View from seedsigner.views.view import Destination, MainMenuView, View
@@ -81,6 +82,7 @@ class FlowStep:
* before_run: function that takes a View instance as an arg and modifies it before running the View. * before_run: function that takes a View instance as an arg and modifies it before running the View.
* screen_return_value: mocked Screen interaction result: raw return value as if from the Screen. * screen_return_value: mocked Screen interaction result: raw return value as if from the Screen.
* button_data_selection: mocked Screen interaction result: the View.button_data value of the desired option. * button_data_selection: mocked Screen interaction result: the View.button_data value of the desired option.
* is_redirect: expects the Destination to specify `skip_current_view=True`.
""" """
expected_view: type[View] = None expected_view: type[View] = None
before_run: Callable[[View], None] = None before_run: Callable[[View], None] = None
@@ -94,75 +96,107 @@ class FlowStep:
class FlowTestInvalidButtonDataSelectionException(FlowBasedTestException):
""" The FlowStep's button_data_selection value was not found in the View's button_data """
pass
class FlowTestUnexpectedViewException(FlowBasedTestException):
""" The Controller's current View did not match the expected View in the FlowStep sequence """
pass
class FlowTestRunScreenNotExecutedException(FlowBasedTestException):
""" The View's run_screen() method was not called but the FlowStep expected it to need user input """
pass
class FlowTest(BaseTest): class FlowTest(BaseTest):
""" Base class for any tests that do flow-based testing """ """ Base class for any tests that do flow-based testing """
def stop_test(self):
""" Raise our special Exception to stop the test """
raise StopFlowBasedTest()
def run_sequence(self, sequence: list[FlowStep], initial_destination_view_args: dict = None) -> None: def run_sequence(self, sequence: list[FlowStep], initial_destination_view_args: dict = None) -> None:
""" """
Run a pre-set sequence of Views w/manually-specified return values in order to test Run a pre-set sequence of Views w/manually-specified return values in order to test
the Controller's flow control logic and the routing from View to View. the Controller's flow control logic and the routing from View to View.
""" """
def verify_next_View_cls_and_run_view(destination: Destination, *args, **kwargs):
# Verify that the View class specified in the test sequence matches the
# View class that is being run.
if destination.View_cls != sequence[0].expected_view:
raise FlowBasedTestUnexpectedViewError(f"Expected {sequence[0].expected_view}, got {destination.View_cls}")
print(f"Confirmed {sequence[0].expected_view} is the next View")
if len(sequence) == 1:
# The last entry in the sequence just needs to confirm that we got the
# right View class; we can now stop the Controller and exit the test.
raise StopFlowBasedTest()
# Run the optional pre-run function to modify the View.
if sequence[0].before_run:
sequence[0].before_run(destination.view)
if sequence[0].is_redirect:
# The current View is going to auto-redirect without calling run_screen(),
# so we need to remove the current step from the sequence before the
# View.run() call below.
sequence.pop(0)
# Some Views reach into their Screen's variables directly (e.g.
# Screen.buttons to preserve the scroll position), so we need to mock out the
# Screen instance that is created by the View.
destination.view.screen = MagicMock()
# Run the View (we're mocking out View._run_view() so the Destination
# won't actually run the View unless we explicitly do so here).
return destination.view.run()
def next_return_value(view: View, *args, **kwargs):
# Return the return value specified in the test sequence and
# remove the completed test step from the sequence.
flow_step = sequence.pop(0)
if flow_step.button_data_selection:
# We're mocking out the View.run_screen() method, so we'll get all of the
# input args that are normally passed into the Screen.run() method,
# including the button_data kwarg.
if "button_data" in kwargs:
return kwargs.get("button_data").index(flow_step.button_data_selection)
else:
raise Exception(f"Can't specify `FlowStep.button_data_selection` if `button_data` isn't a kwarg in {view.__class__.__name__}'s run_screen()")
elif type(flow_step.screen_return_value) in [StopFlowBasedTest, Exception]:
raise flow_step.screen_return_value
return flow_step.screen_return_value
with patch("seedsigner.views.view.Destination._run_view", autospec=True) as mock_run_view: with patch("seedsigner.views.view.Destination._run_view", autospec=True) as mock_run_view:
# Mock out the View._run_view() method so we can verify the View class
# that is specified in the test sequence and then run the View.
mock_run_view.side_effect = verify_next_View_cls_and_run_view
with patch("seedsigner.views.view.View.run_screen", autospec=True) as mock_run_screen: with patch("seedsigner.views.view.View.run_screen", autospec=True) as mock_run_screen:
def run_view(destination: Destination, *args, **kwargs):
""" Replaces Destination._run_view() """
if len(sequence) == 0:
self.stop_test()
cur_flow_step = sequence[0]
# Verify that the View class specified in the test sequence matches the
# View class that is being run.
if destination.View_cls != cur_flow_step.expected_view:
raise FlowTestUnexpectedViewException(f"Expected {cur_flow_step.expected_view}, got {destination.View_cls}")
# Run the optional pre-run function to modify the View.
if cur_flow_step.before_run:
cur_flow_step.before_run(destination.view)
if cur_flow_step.is_redirect:
# The current View is going to auto-redirect without calling run_screen(),
# so we need to remove the current step from the sequence before the
# View.run() call below.
sequence.pop(0)
# Some Views reach into their Screen's variables directly (e.g.
# Screen.buttons to preserve the scroll position), so we need to mock out the
# Screen instance that is created by the View.
destination.view.screen = MagicMock()
prev_mock_run_screen_call_count = mock_run_screen.call_count
# Run the View (with our mocked run_screen) and get the next Destination that results
destination = destination.view.run()
if (cur_flow_step.button_data_selection or cur_flow_step.screen_return_value is not None) and mock_run_screen.call_count == prev_mock_run_screen_call_count:
# The FlowStep was expecting some kind of user interaction, but the View
# never called run_screen().
raise FlowTestRunScreenNotExecutedException(f"View.run_screen() was not run for {destination.View_cls.__name__}")
return destination
def run_screen(view: View, *args, **kwargs):
""" Replaces View.run_screen() """
# Return the return value specified in the test sequence and
# remove the completed test step from the sequence.
flow_step = sequence.pop(0)
if flow_step.button_data_selection:
# We're mocking out the View.run_screen() method, so we'll get all of the
# input args that are normally passed into the Screen.run() method,
# including the button_data kwarg.
if "button_data" in kwargs:
if flow_step.button_data_selection not in kwargs.get("button_data") and flow_step.button_data_selection not in [RET_CODE__BACK_BUTTON, RET_CODE__POWER_BUTTON]:
raise FlowTestInvalidButtonDataSelectionException(f"'{flow_step.button_data_selection}' not found in button_data: {kwargs.get('button_data')}")
return kwargs.get("button_data").index(flow_step.button_data_selection)
else:
raise Exception(f"Can't specify `FlowStep.button_data_selection` if `button_data` isn't a kwarg in {view.__class__.__name__}'s run_screen()")
elif type(flow_step.screen_return_value) in [StopFlowBasedTest, FlowBasedTestException]:
raise flow_step.screen_return_value
return flow_step.screen_return_value
# Mock out the Destination._run_view() method so we can verify the View class
# that is specified in the test sequence and then run the View.
mock_run_view.side_effect = run_view
# Mock out the View.run_screen() method so we can provide the # Mock out the View.run_screen() method so we can provide the
# return value that is specified in the test sequence. # return value that is specified in the test sequence.
mock_run_screen.side_effect = next_return_value mock_run_screen.side_effect = run_screen
# Start the Controller with the first View_cls specified in the test sequence # Start the Controller with the first View_cls specified in the test sequence
if sequence[0].expected_view != MainMenuView: if sequence[0].expected_view != MainMenuView:
@@ -172,5 +206,3 @@ class FlowTest(BaseTest):
# Start the Controller and run the sequence # Start the Controller and run the sequence
Controller.get_instance().start(initial_destination=initial_destination) Controller.get_instance().start(initial_destination=initial_destination)
+44 -11
View File
@@ -1,12 +1,13 @@
import pytest import pytest
# Must import test base before the Controller # Must import test base before the Controller
from base import FlowTest, FlowStep from base import FlowTest, FlowStep, FlowTestUnexpectedViewException, FlowTestInvalidButtonDataSelectionException, FlowTestRunScreenNotExecutedException
from seedsigner.controller import Controller, FlowBasedTestUnexpectedViewError from seedsigner.controller import Controller
from seedsigner.gui.screens.screen import RET_CODE__BACK_BUTTON, RET_CODE__POWER_BUTTON from seedsigner.gui.screens.screen import RET_CODE__BACK_BUTTON, RET_CODE__POWER_BUTTON
from seedsigner.models.seed import Seed from seedsigner.models.seed import Seed
from seedsigner.views.seed_views import SeedBackupView, SeedMnemonicEntryView, SeedOptionsView from seedsigner.models.settings_definition import SettingsConstants
from seedsigner.views.seed_views import SeedBackupView, SeedMnemonicEntryView, SeedOptionsView, SeedWordsWarningView
from seedsigner.views.view import MainMenuView, PowerOptionsView, UnhandledExceptionView from seedsigner.views.view import MainMenuView, PowerOptionsView, UnhandledExceptionView
from seedsigner.views.tools_views import ToolsMenuView, ToolsCalcFinalWordNumWordsView from seedsigner.views.tools_views import ToolsMenuView, ToolsCalcFinalWordNumWordsView
@@ -27,17 +28,49 @@ class TestFlowTest(FlowTest):
]) ])
def test_wrong_View_cls(self): def test_FlowTestUnexpectedViewException(self):
""" """
Ensure that the FlowTest will raise an AssertionError if the next View in the Ensure that the FlowTest will raise a FlowTestUnexpectedViewException if the next
sequence is not the expected View. View in the sequence is not the expected View.
""" """
with pytest.raises(FlowBasedTestUnexpectedViewError): with pytest.raises(FlowTestUnexpectedViewException):
self.run_sequence([ self.run_sequence([
FlowStep(MainMenuView, button_data_selection=RET_CODE__POWER_BUTTON), FlowStep(MainMenuView, button_data_selection=RET_CODE__POWER_BUTTON),
FlowStep(ToolsMenuView), # <-- Wrong target View! Should raise an AssertionError. FlowStep(ToolsMenuView), # <-- Wrong target View! Should raise an AssertionError.
]) ])
def test_FlowTestInvalidButtonDataSelectionException(self):
"""
Ensure that the FlowTest will raise a FlowTestUnexpectedViewException if the
specified button_data_selection in invalid.
"""
with pytest.raises(FlowTestInvalidButtonDataSelectionException):
self.run_sequence([
FlowStep(MainMenuView, button_data_selection="this is not a real button option!"),
])
def test_FlowTestRunScreenNotExecutedException(self):
"""
Ensure that the FlowTest will raise a FlowTestRunScreenNotExecutedException if the next
View in the sequence doesn't call its View.run_screen().
"""
# Disable dire warnings so that the SeedWordsWarningView won't execute its run_screen()
self.settings.set_value(SettingsConstants.SETTING__DIRE_WARNINGS, SettingsConstants.OPTION__DISABLED)
self.controller.storage.set_pending_seed(Seed(mnemonic=["bacon"] * 24))
self.controller.storage.finalize_pending_seed()
with pytest.raises(FlowTestRunScreenNotExecutedException):
self.run_sequence(
initial_destination_view_args=dict(seed_num=0),
sequence=[
FlowStep(SeedOptionsView, button_data_selection=SeedOptionsView.BACKUP),
FlowStep(SeedBackupView, button_data_selection=SeedBackupView.VIEW_WORDS),
FlowStep(SeedWordsWarningView, screen_return_value=0),
],
)
def test_before_run_executes(self): def test_before_run_executes(self):
""" """
@@ -69,12 +102,12 @@ class TestFlowTest(FlowTest):
# And again, but this time with a View that requires input view_args # And again, but this time with a View that requires input view_args
self.reset_controller() self.reset_controller()
self.controller = Controller.get_instance()
# Load a seed into the Controller # Load a seed into the Controller
controller = Controller.get_instance()
seed = Seed(mnemonic=["abandon "* 11 + "about"]) seed = Seed(mnemonic=["abandon "* 11 + "about"])
controller.storage.set_pending_seed(seed) self.controller.storage.set_pending_seed(seed)
controller.storage.finalize_pending_seed() self.controller.storage.finalize_pending_seed()
self.run_sequence( self.run_sequence(
initial_destination_view_args=dict(seed_num=0), initial_destination_view_args=dict(seed_num=0),