modified: activity_yaml_validator.py
modified: app.py modified: research/activity29-battleship.yaml modified: research/activity29-testship.yaml modified: research/guarded_ai.py modified: tests/functional/test_activity_flows.py modified: tests/functional/test_battleship_pre_script.py modified: tests/functional/test_guarded_ai.py modified: tests/unit/test_activity_yaml_validator.py modified: tests/unit/test_app_feedback.py modified: tests/unit/test_guarded_ai.py
This commit is contained in:
parent
d4d697db59
commit
f90df2ae57
11 changed files with 573 additions and 286 deletions
|
|
@ -379,7 +379,9 @@ class TestRealActivityFiles(unittest.TestCase):
|
|||
mock_get_client.return_value = (self.mock_client, "test-model")
|
||||
|
||||
# Load actual activity3.yaml
|
||||
activity_file = Path(__file__).parent.parent.parent / "research" / "activity3.yaml"
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent / "research" / "activity3.yaml"
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
# Should have section_5 as the terminal section
|
||||
|
|
@ -408,7 +410,9 @@ class TestRealActivityFiles(unittest.TestCase):
|
|||
mock_get_client.return_value = (self.mock_client, "test-model")
|
||||
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent / "research" / "activity17-choose-adventure.yaml"
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "research"
|
||||
/ "activity17-choose-adventure.yaml"
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
|
|
@ -450,7 +454,9 @@ class TestRealActivityFiles(unittest.TestCase):
|
|||
mock_get_client.return_value = (self.mock_client, "test-model")
|
||||
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent / "research" / "activity20-n-plus-1.yaml"
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "research"
|
||||
/ "activity20-n-plus-1.yaml"
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,9 @@ class TestBattleshipPreScript(unittest.TestCase):
|
|||
def test_battleship_yaml_has_pre_script(self):
|
||||
"""Test that battleship YAML loads and has pre_script"""
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent / "research" / "activity29-battleship.yaml"
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "research"
|
||||
/ "activity29-battleship.yaml"
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
|
|
@ -57,7 +59,9 @@ class TestBattleshipPreScript(unittest.TestCase):
|
|||
def test_battleship_pre_script_execution_simulation(self):
|
||||
"""Test simulated battleship pre_script execution"""
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent / "research" / "activity29-battleship.yaml"
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "research"
|
||||
/ "activity29-battleship.yaml"
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
|
|
@ -96,7 +100,11 @@ class TestBattleshipPreScript(unittest.TestCase):
|
|||
|
||||
def test_testship_yaml_has_pre_script(self):
|
||||
"""Test that testship YAML also has pre_script"""
|
||||
activity_file = Path(__file__).parent.parent.parent / "research" / "activity29-testship.yaml"
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "research"
|
||||
/ "activity29-testship.yaml"
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
# Should also have pre_script (same structure as battleship)
|
||||
|
|
|
|||
|
|
@ -320,7 +320,9 @@ class TestActivityYAMLChanges(unittest.TestCase):
|
|||
"""Test that activity3's new terminal section loads correctly"""
|
||||
import guarded_ai as guarded_ai
|
||||
|
||||
activity_file = Path(__file__).parent.parent.parent / "research" / "activity3.yaml"
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent / "research" / "activity3.yaml"
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
# Should have section_5 now
|
||||
|
|
@ -348,7 +350,9 @@ class TestActivityYAMLChanges(unittest.TestCase):
|
|||
import guarded_ai as guarded_ai
|
||||
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent / "research" / "activity17-choose-adventure.yaml"
|
||||
Path(__file__).parent.parent.parent
|
||||
/ "research"
|
||||
/ "activity17-choose-adventure.yaml"
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
|
|
@ -375,7 +379,9 @@ class TestActivityYAMLChanges(unittest.TestCase):
|
|||
"activity29-battleship.yaml",
|
||||
"activity29-testship.yaml",
|
||||
]:
|
||||
activity_file = Path(__file__).parent.parent.parent / "research" / battleship_file
|
||||
activity_file = (
|
||||
Path(__file__).parent.parent.parent / "research" / battleship_file
|
||||
)
|
||||
activity = guarded_ai.load_yaml_activity(str(activity_file))
|
||||
|
||||
# Find exit transitions and verify they go to step_4
|
||||
|
|
|
|||
|
|
@ -692,15 +692,23 @@ sections:
|
|||
try:
|
||||
is_valid, errors, warnings = self.validator.validate_file(temp_file)
|
||||
self.assertFalse(is_valid)
|
||||
|
||||
|
||||
# Check for specific error types
|
||||
self.assertTrue(any("feedback_prompts' must be a list" in error for error in errors))
|
||||
self.assertTrue(any("feedback_prompts' cannot be empty" in error for error in errors))
|
||||
self.assertTrue(
|
||||
any("feedback_prompts' must be a list" in error for error in errors)
|
||||
)
|
||||
self.assertTrue(
|
||||
any("feedback_prompts' cannot be empty" in error for error in errors)
|
||||
)
|
||||
self.assertTrue(any("must be a dictionary" in error for error in errors))
|
||||
self.assertTrue(any("missing required field" in error for error in errors))
|
||||
self.assertTrue(any("duplicate feedback prompt name" in error for error in errors))
|
||||
self.assertTrue(
|
||||
any("duplicate feedback prompt name" in error for error in errors)
|
||||
)
|
||||
self.assertTrue(any("name must be a string" in error for error in errors))
|
||||
self.assertTrue(any("tokens_for_ai must be a string" in error for error in errors))
|
||||
self.assertTrue(
|
||||
any("tokens_for_ai must be a string" in error for error in errors)
|
||||
)
|
||||
finally:
|
||||
os.unlink(temp_file)
|
||||
|
||||
|
|
|
|||
|
|
@ -26,29 +26,25 @@ class TestAppFeedback(unittest.TestCase):
|
|||
def setUp(self):
|
||||
"""Set up test fixtures"""
|
||||
self.sample_transition = {
|
||||
"ai_feedback": {
|
||||
"tokens_for_ai": "Additional transition instructions"
|
||||
},
|
||||
"metadata_feedback_filter": ["shot_location", "hit_result", "ship_sunk"]
|
||||
"ai_feedback": {"tokens_for_ai": "Additional transition instructions"},
|
||||
"metadata_feedback_filter": ["shot_location", "hit_result", "ship_sunk"],
|
||||
}
|
||||
|
||||
|
||||
self.sample_metadata = {
|
||||
"shot_location": "A5",
|
||||
"hit_result": "hit",
|
||||
"ship_sunk": "destroyer",
|
||||
"private_info": "should_be_filtered",
|
||||
"player_health": 100
|
||||
}
|
||||
|
||||
self.sample_new_metadata = {
|
||||
"new_shot": "B3",
|
||||
"new_result": "miss"
|
||||
"player_health": 100,
|
||||
}
|
||||
|
||||
self.sample_new_metadata = {"new_shot": "B3", "new_result": "miss"}
|
||||
|
||||
def test_provide_feedback_import(self):
|
||||
"""Test that we can import the provide_feedback function"""
|
||||
try:
|
||||
from app import provide_feedback
|
||||
|
||||
self.assertTrue(callable(provide_feedback))
|
||||
except ImportError as e:
|
||||
self.fail(f"Could not import provide_feedback: {e}")
|
||||
|
|
@ -57,11 +53,12 @@ class TestAppFeedback(unittest.TestCase):
|
|||
"""Test that we can import the provide_feedback_prompts function"""
|
||||
try:
|
||||
from app import provide_feedback_prompts
|
||||
|
||||
self.assertTrue(callable(provide_feedback_prompts))
|
||||
except ImportError as e:
|
||||
self.fail(f"Could not import provide_feedback_prompts: {e}")
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_legacy(self, mock_get_client):
|
||||
"""Test legacy provide_feedback function"""
|
||||
# Import here to avoid issues if module is not available
|
||||
|
|
@ -90,24 +87,30 @@ class TestAppFeedback(unittest.TestCase):
|
|||
|
||||
# Call function
|
||||
feedback = provide_feedback(
|
||||
transition, category, question, feedback_tokens_for_ai,
|
||||
user_response, user_language, username,
|
||||
json_metadata, json_new_metadata
|
||||
transition,
|
||||
category,
|
||||
question,
|
||||
feedback_tokens_for_ai,
|
||||
user_response,
|
||||
user_language,
|
||||
username,
|
||||
json_metadata,
|
||||
json_new_metadata,
|
||||
)
|
||||
|
||||
# Verify result
|
||||
self.assertIn("Great shot! You hit the target.", feedback)
|
||||
|
||||
|
||||
# Verify client was called
|
||||
mock_client.chat.completions.create.assert_called_once()
|
||||
call_args = mock_client.chat.completions.create.call_args[1]
|
||||
|
||||
|
||||
# Check that system message includes language and transition instructions
|
||||
system_message = call_args['messages'][0]['content']
|
||||
system_message = call_args["messages"][0]["content"]
|
||||
self.assertIn("English", system_message)
|
||||
self.assertIn("Additional transition instructions", system_message)
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_prompts_multi(self, mock_get_client):
|
||||
"""Test provide_feedback_prompts with multiple prompts"""
|
||||
try:
|
||||
|
|
@ -118,11 +121,18 @@ class TestAppFeedback(unittest.TestCase):
|
|||
# Setup mock to return different responses for each prompt
|
||||
mock_client = MagicMock()
|
||||
mock_completion_1 = MagicMock()
|
||||
mock_completion_1.choices[0].message.content = "Your shot at A5 was a hit! Enemy shot at B3 missed."
|
||||
mock_completion_1.choices[0].message.content = (
|
||||
"Your shot at A5 was a hit! Enemy shot at B3 missed."
|
||||
)
|
||||
mock_completion_2 = MagicMock()
|
||||
mock_completion_2.choices[0].message.content = "The enemy's destroyer has been sunk!"
|
||||
|
||||
mock_client.chat.completions.create.side_effect = [mock_completion_1, mock_completion_2]
|
||||
mock_completion_2.choices[0].message.content = (
|
||||
"The enemy's destroyer has been sunk!"
|
||||
)
|
||||
|
||||
mock_client.chat.completions.create.side_effect = [
|
||||
mock_completion_1,
|
||||
mock_completion_2,
|
||||
]
|
||||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
# Test data
|
||||
|
|
@ -132,12 +142,12 @@ class TestAppFeedback(unittest.TestCase):
|
|||
feedback_prompts = [
|
||||
{
|
||||
"name": "hit_miss_feedback",
|
||||
"tokens_for_ai": "Report the hit/miss results for both players this turn"
|
||||
"tokens_for_ai": "Report the hit/miss results for both players this turn",
|
||||
},
|
||||
{
|
||||
"name": "ship_sinking_feedback",
|
||||
"tokens_for_ai": "Report any ships that were sunk this turn"
|
||||
}
|
||||
"tokens_for_ai": "Report any ships that were sunk this turn",
|
||||
},
|
||||
]
|
||||
user_response = "A5"
|
||||
user_language = "English"
|
||||
|
|
@ -147,26 +157,33 @@ class TestAppFeedback(unittest.TestCase):
|
|||
|
||||
# Call function
|
||||
feedback_messages = provide_feedback_prompts(
|
||||
transition, category, question, feedback_prompts,
|
||||
user_response, user_language, username,
|
||||
json_metadata, json_new_metadata, ""
|
||||
transition,
|
||||
category,
|
||||
question,
|
||||
feedback_prompts,
|
||||
user_response,
|
||||
user_language,
|
||||
username,
|
||||
json_metadata,
|
||||
json_new_metadata,
|
||||
"",
|
||||
)
|
||||
|
||||
# Verify results
|
||||
self.assertEqual(len(feedback_messages), 2)
|
||||
|
||||
|
||||
# Check first feedback message
|
||||
self.assertEqual(feedback_messages[0]["name"], "hit_miss_feedback")
|
||||
self.assertIn("Your shot at A5 was a hit", feedback_messages[0]["content"])
|
||||
|
||||
|
||||
# Check second feedback message
|
||||
self.assertEqual(feedback_messages[1]["name"], "ship_sinking_feedback")
|
||||
self.assertIn("destroyer has been sunk", feedback_messages[1]["content"])
|
||||
|
||||
|
||||
# Verify client was called twice
|
||||
self.assertEqual(mock_client.chat.completions.create.call_count, 2)
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_with_filtered_metadata(self, mock_get_client):
|
||||
"""Test that provide_feedback works correctly with pre-filtered metadata"""
|
||||
try:
|
||||
|
|
@ -183,30 +200,37 @@ class TestAppFeedback(unittest.TestCase):
|
|||
|
||||
# Simulate app.py behavior: filter metadata before calling provide_feedback
|
||||
filtered_metadata = {
|
||||
k: v for k, v in self.sample_metadata.items()
|
||||
k: v
|
||||
for k, v in self.sample_metadata.items()
|
||||
if k in self.sample_transition["metadata_feedback_filter"]
|
||||
}
|
||||
|
||||
|
||||
provide_feedback(
|
||||
self.sample_transition, "test", "Question?", "tokens",
|
||||
"response", "English", "user",
|
||||
json.dumps(filtered_metadata), json.dumps({})
|
||||
self.sample_transition,
|
||||
"test",
|
||||
"Question?",
|
||||
"tokens",
|
||||
"response",
|
||||
"English",
|
||||
"user",
|
||||
json.dumps(filtered_metadata),
|
||||
json.dumps({}),
|
||||
)
|
||||
|
||||
# Check that user message contains only filtered metadata
|
||||
call_args = mock_client.chat.completions.create.call_args[1]
|
||||
user_message = call_args['messages'][1]['content']
|
||||
|
||||
user_message = call_args["messages"][1]["content"]
|
||||
|
||||
# Should contain filtered fields
|
||||
self.assertIn("shot_location", user_message)
|
||||
self.assertIn("hit_result", user_message)
|
||||
self.assertIn("ship_sunk", user_message)
|
||||
|
||||
|
||||
# Should NOT contain unfiltered fields (because we pre-filtered)
|
||||
self.assertNotIn("private_info", user_message)
|
||||
self.assertNotIn("player_health", user_message)
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_no_filter(self, mock_get_client):
|
||||
"""Test feedback when no metadata filter is specified"""
|
||||
try:
|
||||
|
|
@ -222,23 +246,31 @@ class TestAppFeedback(unittest.TestCase):
|
|||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
# Call function without metadata filter
|
||||
transition = {"ai_feedback": {"tokens_for_ai": "Generate feedback"}} # No metadata_feedback_filter
|
||||
|
||||
transition = {
|
||||
"ai_feedback": {"tokens_for_ai": "Generate feedback"}
|
||||
} # No metadata_feedback_filter
|
||||
|
||||
provide_feedback(
|
||||
transition, "test", "Question?", "tokens",
|
||||
"response", "English", "user",
|
||||
json.dumps(self.sample_metadata), json.dumps({})
|
||||
transition,
|
||||
"test",
|
||||
"Question?",
|
||||
"tokens",
|
||||
"response",
|
||||
"English",
|
||||
"user",
|
||||
json.dumps(self.sample_metadata),
|
||||
json.dumps({}),
|
||||
)
|
||||
|
||||
# Check that user message contains all metadata
|
||||
call_args = mock_client.chat.completions.create.call_args[1]
|
||||
user_message = call_args['messages'][1]['content']
|
||||
|
||||
user_message = call_args["messages"][1]["content"]
|
||||
|
||||
# Should contain all metadata fields when no filter is applied
|
||||
self.assertIn("private_info", user_message)
|
||||
self.assertIn("player_health", user_message)
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_error_handling(self, mock_get_client):
|
||||
"""Test error handling in feedback functions"""
|
||||
try:
|
||||
|
|
@ -253,15 +285,21 @@ class TestAppFeedback(unittest.TestCase):
|
|||
|
||||
# Call function
|
||||
feedback = provide_feedback(
|
||||
{"ai_feedback": {"tokens_for_ai": "Generate feedback"}}, "test", "Question?", "tokens",
|
||||
"response", "English", "user",
|
||||
json.dumps({}), json.dumps({})
|
||||
{"ai_feedback": {"tokens_for_ai": "Generate feedback"}},
|
||||
"test",
|
||||
"Question?",
|
||||
"tokens",
|
||||
"response",
|
||||
"English",
|
||||
"user",
|
||||
json.dumps({}),
|
||||
json.dumps({}),
|
||||
)
|
||||
|
||||
# Should handle error gracefully
|
||||
self.assertIn("Error", feedback)
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_prompts_filter_empty_and_stfu(self, mock_get_client):
|
||||
"""Test feedback_prompts with empty results and STFU tokens filtered out"""
|
||||
try:
|
||||
|
|
@ -274,12 +312,16 @@ class TestAppFeedback(unittest.TestCase):
|
|||
mock_completion_1 = MagicMock()
|
||||
mock_completion_1.choices[0].message.content = "" # Empty result
|
||||
mock_completion_2 = MagicMock()
|
||||
mock_completion_2.choices[0].message.content = "STFU" # STFU token (should be filtered)
|
||||
mock_completion_2.choices[0].message.content = (
|
||||
"STFU" # STFU token (should be filtered)
|
||||
)
|
||||
mock_completion_3 = MagicMock()
|
||||
mock_completion_3.choices[0].message.content = "Valid feedback" # Valid result
|
||||
|
||||
|
||||
mock_client.chat.completions.create.side_effect = [
|
||||
mock_completion_1, mock_completion_2, mock_completion_3
|
||||
mock_completion_1,
|
||||
mock_completion_2,
|
||||
mock_completion_3,
|
||||
]
|
||||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
|
|
@ -287,13 +329,20 @@ class TestAppFeedback(unittest.TestCase):
|
|||
feedback_prompts = [
|
||||
{"name": "empty", "tokens_for_ai": "Empty prompt"},
|
||||
{"name": "stfu", "tokens_for_ai": "STFU prompt"},
|
||||
{"name": "valid", "tokens_for_ai": "Valid prompt"}
|
||||
{"name": "valid", "tokens_for_ai": "Valid prompt"},
|
||||
]
|
||||
|
||||
feedback_messages = provide_feedback_prompts(
|
||||
{}, "test", "Question?", feedback_prompts,
|
||||
"response", "English", "user",
|
||||
json.dumps({}), json.dumps({}), ""
|
||||
{},
|
||||
"test",
|
||||
"Question?",
|
||||
feedback_prompts,
|
||||
"response",
|
||||
"English",
|
||||
"user",
|
||||
json.dumps({}),
|
||||
json.dumps({}),
|
||||
"",
|
||||
)
|
||||
|
||||
# Should only return valid feedback (empty and STFU both filtered out the same way)
|
||||
|
|
@ -301,7 +350,7 @@ class TestAppFeedback(unittest.TestCase):
|
|||
self.assertEqual(feedback_messages[0]["name"], "valid")
|
||||
self.assertEqual(feedback_messages[0]["content"], "Valid feedback")
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_prompts_stfu_partial_not_filtered(self, mock_get_client):
|
||||
"""Test that messages containing STFU as part of larger text are NOT filtered"""
|
||||
try:
|
||||
|
|
@ -312,14 +361,20 @@ class TestAppFeedback(unittest.TestCase):
|
|||
# Setup mock to return STFU as part of larger message
|
||||
mock_client = MagicMock()
|
||||
mock_completion_1 = MagicMock()
|
||||
mock_completion_1.choices[0].message.content = "STFU you rascal." # Should NOT be filtered
|
||||
mock_completion_1.choices[0].message.content = (
|
||||
"STFU you rascal." # Should NOT be filtered
|
||||
)
|
||||
mock_completion_2 = MagicMock()
|
||||
mock_completion_2.choices[0].message.content = "Go STFU yourself!" # Should NOT be filtered
|
||||
mock_completion_2.choices[0].message.content = (
|
||||
"Go STFU yourself!" # Should NOT be filtered
|
||||
)
|
||||
mock_completion_3 = MagicMock()
|
||||
mock_completion_3.choices[0].message.content = "STFU" # Should be filtered
|
||||
|
||||
|
||||
mock_client.chat.completions.create.side_effect = [
|
||||
mock_completion_1, mock_completion_2, mock_completion_3
|
||||
mock_completion_1,
|
||||
mock_completion_2,
|
||||
mock_completion_3,
|
||||
]
|
||||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
|
|
@ -327,13 +382,20 @@ class TestAppFeedback(unittest.TestCase):
|
|||
feedback_prompts = [
|
||||
{"name": "partial1", "tokens_for_ai": "Partial STFU 1"},
|
||||
{"name": "partial2", "tokens_for_ai": "Partial STFU 2"},
|
||||
{"name": "exact", "tokens_for_ai": "Exact STFU"}
|
||||
{"name": "exact", "tokens_for_ai": "Exact STFU"},
|
||||
]
|
||||
|
||||
feedback_messages = provide_feedback_prompts(
|
||||
{}, "test", "Question?", feedback_prompts,
|
||||
"response", "English", "user",
|
||||
json.dumps({}), json.dumps({}), ""
|
||||
{},
|
||||
"test",
|
||||
"Question?",
|
||||
feedback_prompts,
|
||||
"response",
|
||||
"English",
|
||||
"user",
|
||||
json.dumps({}),
|
||||
json.dumps({}),
|
||||
"",
|
||||
)
|
||||
|
||||
# Should return the two partial STFU messages, but not the exact "STFU"
|
||||
|
|
@ -343,8 +405,10 @@ class TestAppFeedback(unittest.TestCase):
|
|||
self.assertEqual(feedback_messages[1]["name"], "partial2")
|
||||
self.assertEqual(feedback_messages[1]["content"], "Go STFU yourself!")
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
def test_provide_feedback_prompts_per_prompt_metadata_filtering(self, mock_get_client):
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_prompts_per_prompt_metadata_filtering(
|
||||
self, mock_get_client
|
||||
):
|
||||
"""Test that each prompt gets its own filtered metadata"""
|
||||
try:
|
||||
from app import provide_feedback_prompts
|
||||
|
|
@ -354,70 +418,88 @@ class TestAppFeedback(unittest.TestCase):
|
|||
# Setup mock to return different responses
|
||||
mock_client = MagicMock()
|
||||
mock_completion_1 = MagicMock()
|
||||
mock_completion_1.choices[0].message.content = "Shot feedback with hit/miss data"
|
||||
mock_completion_1.choices[0].message.content = (
|
||||
"Shot feedback with hit/miss data"
|
||||
)
|
||||
mock_completion_2 = MagicMock()
|
||||
mock_completion_2.choices[0].message.content = "Ship feedback with sinking data"
|
||||
|
||||
|
||||
mock_client.chat.completions.create.side_effect = [
|
||||
mock_completion_1, mock_completion_2
|
||||
mock_completion_1,
|
||||
mock_completion_2,
|
||||
]
|
||||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
# Test data with mixed metadata
|
||||
full_metadata = {
|
||||
"user_shot": "A5",
|
||||
"user_hit_result": "hit",
|
||||
"user_hit_result": "hit",
|
||||
"ai_shot": "B3",
|
||||
"ai_hit_result": "miss",
|
||||
"user_sunk_ship_this_round": "Destroyer",
|
||||
"ai_sunk_ship_this_round": None,
|
||||
"game_over": False,
|
||||
"extra_field": "should_not_appear"
|
||||
"extra_field": "should_not_appear",
|
||||
}
|
||||
|
||||
|
||||
feedback_prompts = [
|
||||
{
|
||||
"name": "shot_report",
|
||||
"tokens_for_ai": "Report hit/miss",
|
||||
"metadata_filter": ["user_shot", "user_hit_result", "ai_shot", "ai_hit_result"]
|
||||
"metadata_filter": [
|
||||
"user_shot",
|
||||
"user_hit_result",
|
||||
"ai_shot",
|
||||
"ai_hit_result",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "ship_status",
|
||||
"name": "ship_status",
|
||||
"tokens_for_ai": "Report ship sinking",
|
||||
"metadata_filter": ["user_sunk_ship_this_round", "ai_sunk_ship_this_round"]
|
||||
}
|
||||
"metadata_filter": [
|
||||
"user_sunk_ship_this_round",
|
||||
"ai_sunk_ship_this_round",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
feedback_messages = provide_feedback_prompts(
|
||||
{}, "test", "Question?", feedback_prompts,
|
||||
"response", "English", "user",
|
||||
json.dumps(full_metadata), json.dumps({}), ""
|
||||
{},
|
||||
"test",
|
||||
"Question?",
|
||||
feedback_prompts,
|
||||
"response",
|
||||
"English",
|
||||
"user",
|
||||
json.dumps(full_metadata),
|
||||
json.dumps({}),
|
||||
"",
|
||||
)
|
||||
|
||||
# Verify both prompts got responses
|
||||
self.assertEqual(len(feedback_messages), 2)
|
||||
self.assertEqual(feedback_messages[0]["name"], "shot_report")
|
||||
self.assertEqual(feedback_messages[1]["name"], "ship_status")
|
||||
|
||||
|
||||
# Verify the first prompt only got shot-related metadata
|
||||
first_call_args = mock_client.chat.completions.create.call_args_list[0][1]
|
||||
first_user_message = first_call_args['messages'][1]['content']
|
||||
first_user_message = first_call_args["messages"][1]["content"]
|
||||
self.assertIn("user_shot", first_user_message)
|
||||
self.assertIn("user_hit_result", first_user_message)
|
||||
self.assertIn("ai_shot", first_user_message)
|
||||
self.assertIn("ai_hit_result", first_user_message)
|
||||
self.assertNotIn("user_sunk_ship_this_round", first_user_message)
|
||||
self.assertNotIn("extra_field", first_user_message)
|
||||
|
||||
# Verify the second prompt only got ship-related metadata
|
||||
|
||||
# Verify the second prompt only got ship-related metadata
|
||||
second_call_args = mock_client.chat.completions.create.call_args_list[1][1]
|
||||
second_user_message = second_call_args['messages'][1]['content']
|
||||
second_user_message = second_call_args["messages"][1]["content"]
|
||||
self.assertIn("user_sunk_ship_this_round", second_user_message)
|
||||
self.assertIn("ai_sunk_ship_this_round", second_user_message)
|
||||
self.assertNotIn("user_shot", second_user_message)
|
||||
self.assertNotIn("extra_field", second_user_message)
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_ship_status_metadata_filtering_debug(self, mock_get_client):
|
||||
"""Debug test to check if Ship Status is getting only the right metadata"""
|
||||
try:
|
||||
|
|
@ -435,59 +517,74 @@ class TestAppFeedback(unittest.TestCase):
|
|||
# Test data mimicking the actual battleship scenario
|
||||
full_metadata = {
|
||||
"user_shot": "46", # This should NOT appear in Ship Status
|
||||
"ai_shot": "49", # This should NOT appear in Ship Status
|
||||
"ai_shot": "49", # This should NOT appear in Ship Status
|
||||
"user_hit_result": "hit",
|
||||
"ai_hit_result": "miss",
|
||||
"ai_hit_result": "miss",
|
||||
"user_sunk_ship_this_round": "Destroyer", # This SHOULD appear
|
||||
"ai_sunk_ship_this_round": None, # This SHOULD appear
|
||||
"ai_sunk_ship_this_round": None, # This SHOULD appear
|
||||
"game_over": False,
|
||||
"extra_stuff": "should not appear anywhere"
|
||||
"extra_stuff": "should not appear anywhere",
|
||||
}
|
||||
|
||||
|
||||
# Exact structure from battleship YAML
|
||||
feedback_prompts = [
|
||||
{
|
||||
"name": "Shot Report",
|
||||
"tokens_for_ai": "🎯 Report ONLY the hit/miss results",
|
||||
"metadata_filter": ["user_shot", "ai_shot", "user_hit_result", "ai_hit_result"]
|
||||
"metadata_filter": [
|
||||
"user_shot",
|
||||
"ai_shot",
|
||||
"user_hit_result",
|
||||
"ai_hit_result",
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "Ship Status",
|
||||
"name": "Ship Status",
|
||||
"tokens_for_ai": "You are the Ship Destruction Oracle",
|
||||
"metadata_filter": ["user_sunk_ship_this_round", "ai_sunk_ship_this_round"]
|
||||
}
|
||||
"metadata_filter": [
|
||||
"user_sunk_ship_this_round",
|
||||
"ai_sunk_ship_this_round",
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
# Call the function
|
||||
provide_feedback_prompts(
|
||||
{}, "valid_move", "Question?", feedback_prompts,
|
||||
"46", "English", "user",
|
||||
json.dumps(full_metadata), json.dumps({}), ""
|
||||
{},
|
||||
"valid_move",
|
||||
"Question?",
|
||||
feedback_prompts,
|
||||
"46",
|
||||
"English",
|
||||
"user",
|
||||
json.dumps(full_metadata),
|
||||
json.dumps({}),
|
||||
"",
|
||||
)
|
||||
|
||||
# Check what metadata each prompt actually received
|
||||
self.assertEqual(mock_client.chat.completions.create.call_count, 2)
|
||||
|
||||
|
||||
# First call should be Shot Report
|
||||
shot_report_call = mock_client.chat.completions.create.call_args_list[0][1]
|
||||
shot_report_metadata = shot_report_call['messages'][1]['content']
|
||||
|
||||
shot_report_metadata = shot_report_call["messages"][1]["content"]
|
||||
|
||||
print("=== SHOT REPORT METADATA ===")
|
||||
print(shot_report_metadata)
|
||||
|
||||
|
||||
# Shot Report should have shot data but NOT ship destruction data
|
||||
self.assertIn("user_shot", shot_report_metadata)
|
||||
self.assertIn("46", shot_report_metadata)
|
||||
self.assertNotIn("user_sunk_ship_this_round", shot_report_metadata)
|
||||
self.assertNotIn("Destroyer", shot_report_metadata)
|
||||
|
||||
|
||||
# Second call should be Ship Status
|
||||
ship_status_call = mock_client.chat.completions.create.call_args_list[1][1]
|
||||
ship_status_metadata = ship_status_call['messages'][1]['content']
|
||||
|
||||
ship_status_metadata = ship_status_call["messages"][1]["content"]
|
||||
|
||||
print("=== SHIP STATUS METADATA ===")
|
||||
print(ship_status_metadata)
|
||||
|
||||
|
||||
# Ship Status should have ship destruction data but NOT shot data
|
||||
self.assertIn("user_sunk_ship_this_round", ship_status_metadata)
|
||||
self.assertIn("Destroyer", ship_status_metadata)
|
||||
|
|
@ -502,31 +599,36 @@ class TestAppFeedback(unittest.TestCase):
|
|||
except ImportError:
|
||||
self.skipTest("app module not available for testing")
|
||||
|
||||
with patch('app.get_openai_client_and_model') as mock_get_client:
|
||||
with patch("app.get_openai_client_and_model") as mock_get_client:
|
||||
mock_client = MagicMock()
|
||||
mock_completion = MagicMock()
|
||||
mock_completion.choices[0].message.content = "Feedback in Spanish"
|
||||
mock_client.chat.completions.create.return_value = mock_completion
|
||||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
feedback_prompts = [
|
||||
{"name": "test", "tokens_for_ai": "Base prompt"}
|
||||
]
|
||||
|
||||
feedback_prompts = [{"name": "test", "tokens_for_ai": "Base prompt"}]
|
||||
|
||||
# Test with Spanish language
|
||||
provide_feedback_prompts(
|
||||
{}, "test", "Question?", feedback_prompts,
|
||||
"response", "Spanish", "user",
|
||||
json.dumps({}), json.dumps({}), ""
|
||||
{},
|
||||
"test",
|
||||
"Question?",
|
||||
feedback_prompts,
|
||||
"response",
|
||||
"Spanish",
|
||||
"user",
|
||||
json.dumps({}),
|
||||
json.dumps({}),
|
||||
"",
|
||||
)
|
||||
|
||||
# Check that system message includes Spanish language instruction
|
||||
call_args = mock_client.chat.completions.create.call_args[1]
|
||||
system_message = call_args['messages'][0]['content']
|
||||
system_message = call_args["messages"][0]["content"]
|
||||
self.assertIn("Spanish", system_message)
|
||||
self.assertIn("Base prompt", system_message)
|
||||
|
||||
@patch('app.get_openai_client_and_model')
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_provide_feedback_transition_tokens(self, mock_get_client):
|
||||
"""Test that transition ai_feedback tokens are included"""
|
||||
try:
|
||||
|
|
@ -541,27 +643,96 @@ class TestAppFeedback(unittest.TestCase):
|
|||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
transition = {
|
||||
"ai_feedback": {
|
||||
"tokens_for_ai": "Be more dramatic in your feedback"
|
||||
}
|
||||
"ai_feedback": {"tokens_for_ai": "Be more dramatic in your feedback"}
|
||||
}
|
||||
|
||||
feedback_prompts = [
|
||||
{"name": "test", "tokens_for_ai": "Base prompt"}
|
||||
]
|
||||
|
||||
|
||||
feedback_prompts = [{"name": "test", "tokens_for_ai": "Base prompt"}]
|
||||
|
||||
provide_feedback_prompts(
|
||||
transition, "test", "Question?", feedback_prompts,
|
||||
"response", "English", "user",
|
||||
json.dumps({}), json.dumps({}), ""
|
||||
transition,
|
||||
"test",
|
||||
"Question?",
|
||||
feedback_prompts,
|
||||
"response",
|
||||
"English",
|
||||
"user",
|
||||
json.dumps({}),
|
||||
json.dumps({}),
|
||||
"",
|
||||
)
|
||||
|
||||
# Check that system message includes both base and transition tokens
|
||||
call_args = mock_client.chat.completions.create.call_args[1]
|
||||
system_message = call_args['messages'][0]['content']
|
||||
system_message = call_args["messages"][0]["content"]
|
||||
self.assertIn("Base prompt", system_message)
|
||||
self.assertIn("Be more dramatic in your feedback", system_message)
|
||||
|
||||
@patch("app.get_openai_client_and_model")
|
||||
def test_user_response_filtering_with_metadata_filter(self, mock_get_client):
|
||||
"""Test that user_response is filtered correctly using metadata_filter approach"""
|
||||
try:
|
||||
from app import provide_feedback_prompts
|
||||
except ImportError:
|
||||
self.skipTest("app module not available for testing")
|
||||
|
||||
# Setup mock
|
||||
mock_client = MagicMock()
|
||||
mock_completion = MagicMock()
|
||||
mock_completion.choices[0].message.content = "Response for prompt"
|
||||
mock_client.chat.completions.create.return_value = mock_completion
|
||||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
# Test feedback prompts - one that includes user_response, one that doesn't
|
||||
feedback_prompts = [
|
||||
{
|
||||
"name": "Shot Report",
|
||||
"tokens_for_ai": "Report shot positions",
|
||||
"metadata_filter": [
|
||||
"user_shot",
|
||||
"user_response",
|
||||
], # Includes user_response
|
||||
},
|
||||
{
|
||||
"name": "Ship Status",
|
||||
"tokens_for_ai": "Report ship status",
|
||||
"metadata_filter": ["ship_status"], # Does NOT include user_response
|
||||
},
|
||||
]
|
||||
|
||||
metadata = {"user_shot": "35", "ship_status": "intact"}
|
||||
|
||||
user_response = "I choose position 35"
|
||||
|
||||
provide_feedback_prompts(
|
||||
{},
|
||||
"valid_move",
|
||||
"Choose position?",
|
||||
feedback_prompts,
|
||||
user_response,
|
||||
"English",
|
||||
"user",
|
||||
json.dumps(metadata),
|
||||
json.dumps({}),
|
||||
"",
|
||||
)
|
||||
|
||||
# Should have 2 calls
|
||||
self.assertEqual(mock_client.chat.completions.create.call_count, 2)
|
||||
|
||||
# First call (Shot Report) should have user_response
|
||||
first_call = mock_client.chat.completions.create.call_args_list[0][1]
|
||||
first_user_message = first_call["messages"][1]["content"]
|
||||
self.assertIn(
|
||||
"I choose position 35", first_user_message
|
||||
) # user_response should be present
|
||||
|
||||
# Second call (Ship Status) should NOT have user_response
|
||||
second_call = mock_client.chat.completions.create.call_args_list[1][1]
|
||||
second_user_message = second_call["messages"][1]["content"]
|
||||
self.assertEqual(
|
||||
second_user_message.count("I choose position 35"), 0
|
||||
) # user_response should be empty/filtered
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
unittest.main(verbosity=2)
|
||||
|
|
|
|||
|
|
@ -41,15 +41,20 @@ class TestGuardedAI(unittest.TestCase):
|
|||
"user_hit_result": "hit",
|
||||
"ai_hit_result": "miss",
|
||||
}
|
||||
|
||||
|
||||
self.sample_transition = {
|
||||
"ai_feedback": {
|
||||
"tokens_for_ai": "Additional transition-specific instructions"
|
||||
},
|
||||
"metadata_feedback_filter": ["user_shot", "ai_shot", "user_hit_result", "ai_hit_result"]
|
||||
"metadata_feedback_filter": [
|
||||
"user_shot",
|
||||
"ai_shot",
|
||||
"user_hit_result",
|
||||
"ai_hit_result",
|
||||
],
|
||||
}
|
||||
|
||||
@patch('guarded_ai.get_openai_client_and_model')
|
||||
@patch("guarded_ai.get_openai_client_and_model")
|
||||
def test_categorize_response(self, mock_get_client):
|
||||
"""Test response categorization"""
|
||||
# Setup mock
|
||||
|
|
@ -69,20 +74,20 @@ class TestGuardedAI(unittest.TestCase):
|
|||
|
||||
# Verify result
|
||||
self.assertEqual(category, "correct_answer")
|
||||
|
||||
|
||||
# Verify client was called correctly
|
||||
mock_client.chat.completions.create.assert_called_once()
|
||||
call_args = mock_client.chat.completions.create.call_args[1]
|
||||
self.assertEqual(call_args['model'], 'test-model')
|
||||
self.assertEqual(call_args['max_tokens'], 5)
|
||||
self.assertEqual(call_args['temperature'], 0)
|
||||
|
||||
# Check message content
|
||||
messages = call_args['messages']
|
||||
self.assertEqual(len(messages), 2)
|
||||
self.assertIn("correct_answer, wrong_answer", messages[0]['content'])
|
||||
self.assertEqual(call_args["model"], "test-model")
|
||||
self.assertEqual(call_args["max_tokens"], 5)
|
||||
self.assertEqual(call_args["temperature"], 0)
|
||||
|
||||
@patch('guarded_ai.get_openai_client_and_model')
|
||||
# Check message content
|
||||
messages = call_args["messages"]
|
||||
self.assertEqual(len(messages), 2)
|
||||
self.assertIn("correct_answer, wrong_answer", messages[0]["content"])
|
||||
|
||||
@patch("guarded_ai.get_openai_client_and_model")
|
||||
def test_generate_ai_feedback(self, mock_get_client):
|
||||
"""Test AI feedback generation"""
|
||||
# Setup mock
|
||||
|
|
@ -99,23 +104,25 @@ class TestGuardedAI(unittest.TestCase):
|
|||
tokens_for_ai = "Provide encouraging feedback"
|
||||
metadata = {"score": 100}
|
||||
|
||||
feedback = generate_ai_feedback(category, question, user_response, tokens_for_ai, metadata)
|
||||
feedback = generate_ai_feedback(
|
||||
category, question, user_response, tokens_for_ai, metadata
|
||||
)
|
||||
|
||||
# Verify result
|
||||
self.assertEqual(feedback, "Great job on the math!")
|
||||
|
||||
|
||||
# Verify client was called correctly
|
||||
mock_client.chat.completions.create.assert_called_once()
|
||||
call_args = mock_client.chat.completions.create.call_args[1]
|
||||
self.assertEqual(call_args['model'], 'test-model')
|
||||
self.assertEqual(call_args['max_tokens'], 250)
|
||||
self.assertEqual(call_args['temperature'], 0.7)
|
||||
self.assertEqual(call_args["model"], "test-model")
|
||||
self.assertEqual(call_args["max_tokens"], 250)
|
||||
self.assertEqual(call_args["temperature"], 0.7)
|
||||
|
||||
@patch('guarded_ai.generate_ai_feedback')
|
||||
@patch("guarded_ai.generate_ai_feedback")
|
||||
def test_provide_feedback_legacy(self, mock_generate_feedback):
|
||||
"""Test legacy single feedback system"""
|
||||
mock_generate_feedback.return_value = "Good work! Try again."
|
||||
|
||||
|
||||
# Test data
|
||||
transition = self.sample_transition
|
||||
category = "partial_understanding"
|
||||
|
|
@ -127,42 +134,50 @@ class TestGuardedAI(unittest.TestCase):
|
|||
|
||||
# Call function
|
||||
feedback = provide_feedback(
|
||||
transition, category, question, user_response,
|
||||
user_language, tokens_for_ai, metadata
|
||||
transition,
|
||||
category,
|
||||
question,
|
||||
user_response,
|
||||
user_language,
|
||||
tokens_for_ai,
|
||||
metadata,
|
||||
)
|
||||
|
||||
# Verify feedback was generated
|
||||
self.assertIn("AI Feedback:", feedback)
|
||||
self.assertIn("Good work! Try again.", feedback)
|
||||
|
||||
|
||||
# Verify generate_ai_feedback was called with filtered metadata
|
||||
mock_generate_feedback.assert_called_once()
|
||||
call_args = mock_generate_feedback.call_args[0]
|
||||
self.assertEqual(call_args[0], category) # category
|
||||
self.assertEqual(call_args[1], question) # question
|
||||
self.assertEqual(call_args[2], user_response) # user_response
|
||||
|
||||
|
||||
# Check tokens_for_ai includes language and transition instructions
|
||||
tokens_arg = call_args[3]
|
||||
self.assertIn("English", tokens_arg)
|
||||
self.assertIn("Additional transition-specific instructions", tokens_arg)
|
||||
|
||||
|
||||
# Check metadata was filtered
|
||||
filtered_metadata = call_args[4]
|
||||
expected_filtered = {k: v for k, v in self.sample_metadata.items()
|
||||
if k in transition["metadata_feedback_filter"]}
|
||||
expected_filtered = {
|
||||
k: v
|
||||
for k, v in self.sample_metadata.items()
|
||||
if k in transition["metadata_feedback_filter"]
|
||||
}
|
||||
# Since our test metadata doesn't have the filtered keys, it should be empty or contain only matching keys
|
||||
# But the function should have passed what it received
|
||||
|
||||
@patch('guarded_ai.generate_ai_feedback')
|
||||
@patch("guarded_ai.generate_ai_feedback")
|
||||
def test_provide_feedback_prompts(self, mock_generate_feedback):
|
||||
"""Test new multi-prompt feedback system"""
|
||||
# Setup mock to return different feedback for each prompt
|
||||
mock_generate_feedback.side_effect = [
|
||||
"Hit at A5, miss at B3",
|
||||
"No ships were sunk this round"
|
||||
"No ships were sunk this round",
|
||||
]
|
||||
|
||||
|
||||
# Test data
|
||||
transition = self.sample_transition
|
||||
category = "valid_move"
|
||||
|
|
@ -170,12 +185,12 @@ class TestGuardedAI(unittest.TestCase):
|
|||
feedback_prompts = [
|
||||
{
|
||||
"name": "hit_miss",
|
||||
"tokens_for_ai": "Report the hit/miss results for both players"
|
||||
"tokens_for_ai": "Report the hit/miss results for both players",
|
||||
},
|
||||
{
|
||||
"name": "ship_sinking",
|
||||
"tokens_for_ai": "Report any ships that were sunk"
|
||||
}
|
||||
"name": "ship_sinking",
|
||||
"tokens_for_ai": "Report any ships that were sunk",
|
||||
},
|
||||
]
|
||||
user_response = "A5"
|
||||
user_language = "English"
|
||||
|
|
@ -183,47 +198,61 @@ class TestGuardedAI(unittest.TestCase):
|
|||
|
||||
# Call function
|
||||
feedback_messages = provide_feedback_prompts(
|
||||
transition, category, question, feedback_prompts,
|
||||
user_response, user_language, metadata, ""
|
||||
transition,
|
||||
category,
|
||||
question,
|
||||
feedback_prompts,
|
||||
user_response,
|
||||
user_language,
|
||||
metadata,
|
||||
"",
|
||||
)
|
||||
|
||||
# Verify we got the expected number of feedback messages
|
||||
self.assertEqual(len(feedback_messages), 2)
|
||||
|
||||
|
||||
# Verify message structure
|
||||
self.assertEqual(feedback_messages[0]["name"], "hit_miss")
|
||||
self.assertEqual(feedback_messages[0]["content"], "Hit at A5, miss at B3")
|
||||
self.assertEqual(feedback_messages[1]["name"], "ship_sinking")
|
||||
self.assertEqual(feedback_messages[1]["content"], "No ships were sunk this round")
|
||||
|
||||
self.assertEqual(
|
||||
feedback_messages[1]["content"], "No ships were sunk this round"
|
||||
)
|
||||
|
||||
# Verify generate_ai_feedback was called twice
|
||||
self.assertEqual(mock_generate_feedback.call_count, 2)
|
||||
|
||||
@patch('guarded_ai.generate_ai_feedback')
|
||||
@patch("guarded_ai.generate_ai_feedback")
|
||||
def test_provide_feedback_prompts_empty_responses(self, mock_generate_feedback):
|
||||
"""Test that empty feedback responses are filtered out"""
|
||||
# Setup mock to return empty/whitespace responses
|
||||
mock_generate_feedback.side_effect = [
|
||||
"", # Empty response
|
||||
" ", # Whitespace only
|
||||
"Valid feedback" # Valid response
|
||||
"Valid feedback", # Valid response
|
||||
]
|
||||
|
||||
|
||||
transition = {}
|
||||
category = "test"
|
||||
question = "Test?"
|
||||
feedback_prompts = [
|
||||
{"name": "empty", "tokens_for_ai": "Empty prompt"},
|
||||
{"name": "whitespace", "tokens_for_ai": "Whitespace prompt"},
|
||||
{"name": "valid", "tokens_for_ai": "Valid prompt"}
|
||||
{"name": "whitespace", "tokens_for_ai": "Whitespace prompt"},
|
||||
{"name": "valid", "tokens_for_ai": "Valid prompt"},
|
||||
]
|
||||
user_response = "Test response"
|
||||
user_language = "English"
|
||||
metadata = {}
|
||||
|
||||
feedback_messages = provide_feedback_prompts(
|
||||
transition, category, question, feedback_prompts,
|
||||
user_response, user_language, metadata, ""
|
||||
transition,
|
||||
category,
|
||||
question,
|
||||
feedback_prompts,
|
||||
user_response,
|
||||
user_language,
|
||||
metadata,
|
||||
"",
|
||||
)
|
||||
|
||||
# Should only return the valid feedback message
|
||||
|
|
@ -241,44 +270,53 @@ class TestGuardedAI(unittest.TestCase):
|
|||
tokens_for_ai = "Base tokens"
|
||||
metadata = {}
|
||||
|
||||
with patch('guarded_ai.generate_ai_feedback') as mock_generate:
|
||||
with patch("guarded_ai.generate_ai_feedback") as mock_generate:
|
||||
mock_generate.return_value = "" # Should not be called
|
||||
|
||||
|
||||
feedback = provide_feedback(
|
||||
transition, category, question, user_response,
|
||||
user_language, tokens_for_ai, metadata
|
||||
transition,
|
||||
category,
|
||||
question,
|
||||
user_response,
|
||||
user_language,
|
||||
tokens_for_ai,
|
||||
metadata,
|
||||
)
|
||||
|
||||
# Should NOT call generate_ai_feedback when no ai_feedback in transition
|
||||
mock_generate.assert_not_called()
|
||||
self.assertEqual(feedback, "")
|
||||
|
||||
@patch.dict('os.environ', {'MODEL_ENDPOINT_0': 'http://test.com', 'MODEL_API_KEY_0': 'test-key'})
|
||||
@patch.dict(
|
||||
"os.environ",
|
||||
{"MODEL_ENDPOINT_0": "http://test.com", "MODEL_API_KEY_0": "test-key"},
|
||||
)
|
||||
def test_initialize_model_map(self):
|
||||
"""Test model map initialization from environment variables"""
|
||||
with patch('guarded_ai.get_client_for_endpoint') as mock_get_client:
|
||||
with patch("guarded_ai.get_client_for_endpoint") as mock_get_client:
|
||||
mock_client = MagicMock()
|
||||
mock_get_client.return_value = mock_client
|
||||
|
||||
|
||||
# Clear and reinitialize
|
||||
import guarded_ai
|
||||
|
||||
guarded_ai.MODEL_CLIENT_MAP = {}
|
||||
initialize_model_map()
|
||||
|
||||
|
||||
# Verify client was created and stored
|
||||
mock_get_client.assert_called_with('http://test.com', 'test-key')
|
||||
self.assertIn('endpoint_0', guarded_ai.MODEL_CLIENT_MAP)
|
||||
self.assertEqual(guarded_ai.MODEL_CLIENT_MAP['endpoint_0'][0], mock_client)
|
||||
mock_get_client.assert_called_with("http://test.com", "test-key")
|
||||
self.assertIn("endpoint_0", guarded_ai.MODEL_CLIENT_MAP)
|
||||
self.assertEqual(guarded_ai.MODEL_CLIENT_MAP["endpoint_0"][0], mock_client)
|
||||
|
||||
def test_get_openai_client_and_model_default(self):
|
||||
"""Test getting OpenAI client with default model"""
|
||||
with patch('guarded_ai.MODEL_CLIENT_MAP', {}):
|
||||
with patch('guarded_ai.get_client_for_endpoint') as mock_get_client:
|
||||
with patch("guarded_ai.MODEL_CLIENT_MAP", {}):
|
||||
with patch("guarded_ai.get_client_for_endpoint") as mock_get_client:
|
||||
mock_client = MagicMock()
|
||||
mock_get_client.return_value = mock_client
|
||||
|
||||
|
||||
client, model = get_openai_client_and_model()
|
||||
|
||||
|
||||
# Should return default model name
|
||||
self.assertEqual(model, "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic")
|
||||
self.assertEqual(client, mock_client)
|
||||
|
|
@ -286,18 +324,16 @@ class TestGuardedAI(unittest.TestCase):
|
|||
def test_get_openai_client_and_model_from_map(self):
|
||||
"""Test getting OpenAI client from model map"""
|
||||
mock_client = MagicMock()
|
||||
test_map = {
|
||||
'endpoint_0': (mock_client, 'http://test.com')
|
||||
}
|
||||
|
||||
with patch('guarded_ai.MODEL_CLIENT_MAP', test_map):
|
||||
test_map = {"endpoint_0": (mock_client, "http://test.com")}
|
||||
|
||||
with patch("guarded_ai.MODEL_CLIENT_MAP", test_map):
|
||||
client, model = get_openai_client_and_model("test-model")
|
||||
|
||||
|
||||
# Should return client from map
|
||||
self.assertEqual(client, mock_client)
|
||||
self.assertEqual(model, "test-model")
|
||||
|
||||
@patch('guarded_ai.get_openai_client_and_model')
|
||||
@patch("guarded_ai.get_openai_client_and_model")
|
||||
def test_categorize_response_error_handling(self, mock_get_client):
|
||||
"""Test error handling in categorize_response"""
|
||||
# Setup mock to raise exception
|
||||
|
|
@ -306,11 +342,11 @@ class TestGuardedAI(unittest.TestCase):
|
|||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
category = categorize_response("Test?", "Answer", ["bucket1"], "tokens")
|
||||
|
||||
|
||||
# Should return error string
|
||||
self.assertIn("Error:", category)
|
||||
|
||||
@patch('guarded_ai.get_openai_client_and_model')
|
||||
@patch("guarded_ai.get_openai_client_and_model")
|
||||
def test_generate_ai_feedback_error_handling(self, mock_get_client):
|
||||
"""Test error handling in generate_ai_feedback"""
|
||||
# Setup mock to raise exception
|
||||
|
|
@ -319,10 +355,10 @@ class TestGuardedAI(unittest.TestCase):
|
|||
mock_get_client.return_value = (mock_client, "test-model")
|
||||
|
||||
feedback = generate_ai_feedback("cat", "Q?", "A", "tokens", {})
|
||||
|
||||
|
||||
# Should return error string
|
||||
self.assertIn("Error:", feedback)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
unittest.main(verbosity=2)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue