engineering/opencompletion.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

modified: activity_yaml_validator.py

Browse source 
modified:   app.py
	modified:   research/activity29-battleship.yaml
	modified:   research/activity29-testship.yaml
	modified:   research/guarded_ai.py
	modified:   tests/functional/test_activity_flows.py
	modified:   tests/functional/test_battleship_pre_script.py
	modified:   tests/functional/test_guarded_ai.py
	modified:   tests/unit/test_activity_yaml_validator.py
	modified:   tests/unit/test_app_feedback.py
	modified:   tests/unit/test_guarded_ai.py
This commit is contained in:
Russell Ballestrini 2025-08-11 12:39:42 -04:00
parent d4d697db59
commit f90df2ae57
11 changed files with 573 additions and 286 deletions
Download patch file Download diff file Expand all files Collapse all files

10
activity_yaml_validator.py
View file

@ -244,7 +244,9 @@ class ActivityYAMLValidator:
# Validate feedback_prompts (new multi-prompt system)
if "feedback_prompts" in step:
self._validate_feedback_prompts(step["feedback_prompts"], section_id, step_id)
self._validate_feedback_prompts(
step["feedback_prompts"], section_id, step_id
)
# Validate buckets and transitions
if "buckets" in step:
@ -255,7 +257,9 @@ class ActivityYAMLValidator:
step["transitions"], step.get("buckets", []), section_id, step_id
)
def _validate_feedback_prompts(self, feedback_prompts: List[Dict[str, Any]], section_id: str, step_id: str):
def _validate_feedback_prompts(
self, feedback_prompts: List[Dict[str, Any]], section_id: str, step_id: str
):
"""Validate feedback_prompts structure"""
if not isinstance(feedback_prompts, list):
self.errors.append(
@ -308,7 +312,7 @@ class ActivityYAMLValidator:
elif "STFU" in prompt["tokens_for_ai"]:
# This is valid - STFU token is used to suppress empty feedback messages
pass
# Validate metadata_filter (optional)
if "metadata_filter" in prompt:
if not isinstance(prompt["metadata_filter"], list):

86
app.py
View file

@ -2155,7 +2155,7 @@ def handle_activity_response(room_name, user_response, username):
# Handle feedback systems
feedback_messages = []
if "feedback_prompts" in step:
# New multi-prompt system - pass full metadata, let each prompt filter
multi_feedback_messages = provide_feedback_prompts(
@ -2168,7 +2168,7 @@ def handle_activity_response(room_name, user_response, username):
username,
json.dumps(activity_state.dict_metadata), # Pass full metadata
json.dumps(new_metadata),
feedback_tokens_for_ai # Pass legacy tokens to be combined
feedback_tokens_for_ai, # Pass legacy tokens to be combined
)
feedback_messages.extend(multi_feedback_messages)
elif feedback_tokens_for_ai:
@ -2181,7 +2181,7 @@ def handle_activity_response(room_name, user_response, username):
for k, v in activity_state.dict_metadata.items()
if k in filter_keys
}
feedback = provide_feedback(
transition,
category,
@ -2194,17 +2194,16 @@ def handle_activity_response(room_name, user_response, username):
json.dumps(new_metadata),
)
if feedback and feedback.strip():
feedback_messages.append({
"name": "Feedback",
"content": feedback
})
feedback_messages.append(
{"name": "Feedback", "content": feedback}
)
# Store and emit all feedback messages
for feedback_msg in feedback_messages:
new_message = Message(
username=f"System ({feedback_msg['name'].title()})",
content=feedback_msg['content'],
room_id=room.id
content=feedback_msg["content"],
room_id=room.id,
)
db.session.add(new_message)
db.session.commit()
@ -2214,7 +2213,7 @@ def handle_activity_response(room_name, user_response, username):
{
"id": new_message.id,
"username": f"System ({feedback_msg['name'].title()})",
"content": feedback_msg['content'],
"content": feedback_msg["content"],
},
room=room_name,
)
@ -2626,53 +2625,78 @@ def provide_feedback_prompts(
):
"""Generate feedback from multiple prompts"""
feedback_messages = []
# Parse full metadata once for filtering
full_metadata = json.loads(json_metadata)
# Add user_response to metadata for filtering purposes
full_metadata["user_response"] = user_response
for prompt in feedback_prompts:
prompt_name = prompt.get("name", "unnamed")
tokens_for_ai = prompt.get("tokens_for_ai", "")
# Apply per-prompt metadata filtering if specified
prompt_metadata = full_metadata
if "metadata_filter" in prompt:
filter_keys = prompt["metadata_filter"]
prompt_metadata = {k: v for k, v in full_metadata.items() if k in filter_keys}
print(f"DEBUG: Prompt '{prompt_name}' filter_keys: {filter_keys}")
print(f"DEBUG: Prompt '{prompt_name}' filtered metadata: {prompt_metadata}")
prompt_metadata = {
k: v for k, v in full_metadata.items() if k in filter_keys
}
# Special debug for Ship Status
if prompt_name == "Ship Status":
print(f"DEBUG SHIP STATUS - filter_keys: {filter_keys}")
print(f"DEBUG SHIP STATUS - filtered metadata: {prompt_metadata}")
print(
f"DEBUG SHIP STATUS - user_sunk_ship_this_round = '{prompt_metadata.get('user_sunk_ship_this_round')}'"
)
print(
f"DEBUG SHIP STATUS - ai_sunk_ship_this_round = '{prompt_metadata.get('ai_sunk_ship_this_round')}'"
)
else:
print(f"DEBUG: Prompt '{prompt_name}' has NO metadata_filter, using full metadata")
print(f"DEBUG: Prompt '{prompt_name}' full metadata: {prompt_metadata}")
if prompt_name == "Ship Status":
print(
f"DEBUG SHIP STATUS - NO metadata_filter, full metadata: {prompt_metadata}"
)
# Combine legacy tokens with prompt-specific tokens
if legacy_tokens_for_ai:
tokens_for_ai = legacy_tokens_for_ai + " " + tokens_for_ai
# Add language instruction
tokens_for_ai += f" You must provide the feedback in the user's language: {user_language}."
tokens_for_ai += (
f" You must provide the feedback in the user's language: {user_language}."
)
# Add transition-specific AI feedback if present
if "ai_feedback" in transition:
tokens_for_ai += f" {transition['ai_feedback'].get('tokens_for_ai', '')}"
# Determine user_response for this prompt based on metadata filtering
filtered_user_response = user_response
if (
"metadata_filter" in prompt
and "user_response" not in prompt["metadata_filter"]
):
filtered_user_response = "" # Remove user response if not in filter
ai_feedback = generate_ai_feedback(
category,
question,
user_response,
filtered_user_response,
tokens_for_ai,
username,
json.dumps(prompt_metadata), # Use filtered metadata for this prompt
json_new_metadata,
)
# Only add feedback if it has content and isn't exactly the STFU token
if ai_feedback and ai_feedback.strip() and ai_feedback.strip() != "STFU":
feedback_messages.append({
"name": prompt_name,
"content": ai_feedback.strip()
})
feedback_messages.append(
{"name": prompt_name, "content": ai_feedback.strip()}
)
return feedback_messages

16
research/activity29-battleship.yaml
View file

@ -205,6 +205,7 @@ sections:
- ai_shot
- user_hit_result
- ai_hit_result
- user_response
- name: "Ship Status"
tokens_for_ai: |
@ -214,12 +215,17 @@ sections:
- user_sunk_ship_this_round: If this contains a ship name like "Cruiser", it means THE USER destroyed an ENEMY ship
- ai_sunk_ship_this_round: If this contains a ship name like "Destroyer", it means THE ENEMY destroyed a USER ship
Your responses:
- If user_sunk_ship_this_round has a ship name: "💥 You have destroyed the enemy's [ship name]! It sinks beneath the waves!"
- If ai_sunk_ship_this_round has a ship name: "🔥 The enemy has destroyed your [ship name]! It has been claimed by the sea!"
- If both have ship names: combine both messages above
- If both are null/empty: "STFU"
Examples of when to respond:
- If ai_sunk_ship_this_round = "Submarine": Generate submarine destruction story
- If ai_sunk_ship_this_round = "Carrier": Generate carrier destruction story
- If user_sunk_ship_this_round = "Destroyer": Generate destroyer victory story
- If both = "None": Respond with "STFU"
Your responses:
- If user_sunk_ship_this_round equals "Carrier", "Battleship", "Cruiser", "Submarine", or "Destroyer": "💥 You have destroyed the enemy's [ship name]! Write 3 dramatic sentences describing how this specific type of warship meets its end - does it explode? Break apart? Burn? Implode? Make it cinematic!"
- If ai_sunk_ship_this_round equals "Carrier", "Battleship", "Cruiser", "Submarine", or "Destroyer": "🔥 The enemy has destroyed your [ship name]! Write 3 dramatic sentences describing how this specific type of warship is destroyed - the fire, water, explosions, or structural failure. Make it epic!"
- If both equal ship names: combine both messages above
- If both equal "None" or null: "STFU"
Do NOT confuse who destroyed what. user_sunk_ship_this_round = USER victory. ai_sunk_ship_this_round = USER loss.
metadata_filter:
- user_sunk_ship_this_round

16
research/activity29-testship.yaml
View file

@ -183,6 +183,7 @@ sections:
- ai_shot
- user_hit_result
- ai_hit_result
- user_response
- name: "Ship Status"
tokens_for_ai: |
@ -192,12 +193,17 @@ sections:
- user_sunk_ship_this_round: If this contains a ship name like "Cruiser", it means THE USER destroyed an ENEMY ship
- ai_sunk_ship_this_round: If this contains a ship name like "Destroyer", it means THE ENEMY destroyed a USER ship
Your responses:
- If user_sunk_ship_this_round has a ship name: "💥 You have destroyed the enemy's [ship name]! It sinks beneath the waves!"
- If ai_sunk_ship_this_round has a ship name: "🔥 The enemy has destroyed your [ship name]! It has been claimed by the sea!"
- If both have ship names: combine both messages above
- If both are null/empty: "STFU"
Examples of when to respond:
- If ai_sunk_ship_this_round = "Submarine": Generate submarine destruction story
- If ai_sunk_ship_this_round = "Carrier": Generate carrier destruction story
- If user_sunk_ship_this_round = "Destroyer": Generate destroyer victory story
- If both = "None": Respond with "STFU"
Your responses:
- If user_sunk_ship_this_round equals "Carrier", "Battleship", "Cruiser", "Submarine", or "Destroyer": "💥 You have destroyed the enemy's [ship name]! Write 3 dramatic sentences describing how this specific type of warship meets its end - does it explode? Break apart? Burn? Implode? Make it cinematic!"
- If ai_sunk_ship_this_round equals "Carrier", "Battleship", "Cruiser", "Submarine", or "Destroyer": "🔥 The enemy has destroyed your [ship name]! Write 3 dramatic sentences describing how this specific type of warship is destroyed - the fire, water, explosions, or structural failure. Make it epic!"
- If both equal ship names: combine both messages above
- If both equal "None" or null: "STFU"
Do NOT confuse who destroyed what. user_sunk_ship_this_round = USER victory. ai_sunk_ship_this_round = USER loss.
metadata_filter:
- user_sunk_ship_this_round

56
research/guarded_ai.py
View file

@ -163,39 +163,52 @@ def provide_feedback_prompts(
):
"""Generate feedback from multiple prompts"""
feedback_messages = []
# Add user_response to metadata for filtering purposes
full_metadata = metadata.copy()
full_metadata["user_response"] = user_response
for prompt in feedback_prompts:
prompt_name = prompt.get("name", "unnamed")
tokens_for_ai = prompt.get("tokens_for_ai", "")
# Apply per-prompt metadata filtering if specified
prompt_metadata = metadata
prompt_metadata = full_metadata
if "metadata_filter" in prompt:
filter_keys = prompt["metadata_filter"]
prompt_metadata = {k: v for k, v in metadata.items() if k in filter_keys}
prompt_metadata = {
k: v for k, v in full_metadata.items() if k in filter_keys
}
# Combine legacy tokens with prompt-specific tokens
if legacy_tokens_for_ai:
tokens_for_ai = legacy_tokens_for_ai + " " + tokens_for_ai
# Add language instruction
tokens_for_ai += f" Provide the feedback in {user_language}."
# Add transition-specific AI feedback if present
if "ai_feedback" in transition:
tokens_for_ai += f" {transition['ai_feedback'].get('tokens_for_ai', '')}"
# Determine user_response for this prompt based on metadata filtering
filtered_user_response = user_response
if (
"metadata_filter" in prompt
and "user_response" not in prompt["metadata_filter"]
):
filtered_user_response = "" # Remove user response if not in filter
ai_feedback = generate_ai_feedback(
category, question, user_response, tokens_for_ai, prompt_metadata
category, question, filtered_user_response, tokens_for_ai, prompt_metadata
)
# Only add feedback if it has content and isn't exactly the STFU token
if ai_feedback and ai_feedback.strip() and ai_feedback.strip() != "STFU":
feedback_messages.append({
"name": prompt_name,
"content": ai_feedback.strip()
})
feedback_messages.append(
{"name": prompt_name, "content": ai_feedback.strip()}
)
return feedback_messages
@ -463,7 +476,7 @@ def simulate_activity(yaml_file_path):
# Provide feedback based on the category
feedback_messages = []
if "feedback_prompts" in step:
# New multi-prompt system - legacy tokens get combined with each prompt
multi_feedback_messages = provide_feedback_prompts(
@ -474,7 +487,9 @@ def simulate_activity(yaml_file_path):
user_response,
user_language,
metadata,
step.get("feedback_tokens_for_ai", "") # Pass legacy tokens to be combined
step.get(
"feedback_tokens_for_ai", ""
), # Pass legacy tokens to be combined
)
feedback_messages.extend(multi_feedback_messages)
elif step.get("feedback_tokens_for_ai"):
@ -489,11 +504,8 @@ def simulate_activity(yaml_file_path):
metadata,
)
if feedback and feedback.strip():
feedback_messages.append({
"name": "Feedback",
"content": feedback
})
feedback_messages.append({"name": "Feedback", "content": feedback})
# Display all feedback messages
for feedback_msg in feedback_messages:
print(f"\n{feedback_msg['name']}: {feedback_msg['content']}")

12
tests/functional/test_activity_flows.py
View file

@ -379,7 +379,9 @@ class TestRealActivityFiles(unittest.TestCase):
mock_get_client.return_value = (self.mock_client, "test-model")
# Load actual activity3.yaml
activity_file = Path(__file__).parent.parent.parent / "research" / "activity3.yaml"
activity_file = (
Path(__file__).parent.parent.parent / "research" / "activity3.yaml"
)
activity = guarded_ai.load_yaml_activity(str(activity_file))
# Should have section_5 as the terminal section
@ -408,7 +410,9 @@ class TestRealActivityFiles(unittest.TestCase):
mock_get_client.return_value = (self.mock_client, "test-model")
activity_file = (
Path(__file__).parent.parent.parent / "research" / "activity17-choose-adventure.yaml"
Path(__file__).parent.parent.parent
/ "research"
/ "activity17-choose-adventure.yaml"
)
activity = guarded_ai.load_yaml_activity(str(activity_file))
@ -450,7 +454,9 @@ class TestRealActivityFiles(unittest.TestCase):
mock_get_client.return_value = (self.mock_client, "test-model")
activity_file = (
Path(__file__).parent.parent.parent / "research" / "activity20-n-plus-1.yaml"
Path(__file__).parent.parent.parent
/ "research"
/ "activity20-n-plus-1.yaml"
)
activity = guarded_ai.load_yaml_activity(str(activity_file))

14
tests/functional/test_battleship_pre_script.py
View file

@ -28,7 +28,9 @@ class TestBattleshipPreScript(unittest.TestCase):
def test_battleship_yaml_has_pre_script(self):
"""Test that battleship YAML loads and has pre_script"""
activity_file = (
Path(__file__).parent.parent.parent / "research" / "activity29-battleship.yaml"
Path(__file__).parent.parent.parent
/ "research"
/ "activity29-battleship.yaml"
)
activity = guarded_ai.load_yaml_activity(str(activity_file))
@ -57,7 +59,9 @@ class TestBattleshipPreScript(unittest.TestCase):
def test_battleship_pre_script_execution_simulation(self):
"""Test simulated battleship pre_script execution"""
activity_file = (
Path(__file__).parent.parent.parent / "research" / "activity29-battleship.yaml"
Path(__file__).parent.parent.parent
/ "research"
/ "activity29-battleship.yaml"
)
activity = guarded_ai.load_yaml_activity(str(activity_file))
@ -96,7 +100,11 @@ class TestBattleshipPreScript(unittest.TestCase):
def test_testship_yaml_has_pre_script(self):
"""Test that testship YAML also has pre_script"""
activity_file = Path(__file__).parent.parent.parent / "research" / "activity29-testship.yaml"
activity_file = (
Path(__file__).parent.parent.parent
/ "research"
/ "activity29-testship.yaml"
)
activity = guarded_ai.load_yaml_activity(str(activity_file))
# Should also have pre_script (same structure as battleship)

12
tests/functional/test_guarded_ai.py
View file

@ -320,7 +320,9 @@ class TestActivityYAMLChanges(unittest.TestCase):
"""Test that activity3's new terminal section loads correctly"""
import guarded_ai as guarded_ai
activity_file = Path(__file__).parent.parent.parent / "research" / "activity3.yaml"
activity_file = (
Path(__file__).parent.parent.parent / "research" / "activity3.yaml"
)
activity = guarded_ai.load_yaml_activity(str(activity_file))
# Should have section_5 now
@ -348,7 +350,9 @@ class TestActivityYAMLChanges(unittest.TestCase):
import guarded_ai as guarded_ai
activity_file = (
Path(__file__).parent.parent.parent / "research" / "activity17-choose-adventure.yaml"
Path(__file__).parent.parent.parent
/ "research"
/ "activity17-choose-adventure.yaml"
)
activity = guarded_ai.load_yaml_activity(str(activity_file))
@ -375,7 +379,9 @@ class TestActivityYAMLChanges(unittest.TestCase):
"activity29-battleship.yaml",
"activity29-testship.yaml",
]:
activity_file = Path(__file__).parent.parent.parent / "research" / battleship_file
activity_file = (
Path(__file__).parent.parent.parent / "research" / battleship_file
)
activity = guarded_ai.load_yaml_activity(str(activity_file))
# Find exit transitions and verify they go to step_4

18
tests/unit/test_activity_yaml_validator.py
View file

@ -692,15 +692,23 @@ sections:
try:
is_valid, errors, warnings = self.validator.validate_file(temp_file)
self.assertFalse(is_valid)
# Check for specific error types
self.assertTrue(any("feedback_prompts' must be a list" in error for error in errors))
self.assertTrue(any("feedback_prompts' cannot be empty" in error for error in errors))
self.assertTrue(
any("feedback_prompts' must be a list" in error for error in errors)
)
self.assertTrue(
any("feedback_prompts' cannot be empty" in error for error in errors)
)
self.assertTrue(any("must be a dictionary" in error for error in errors))
self.assertTrue(any("missing required field" in error for error in errors))
self.assertTrue(any("duplicate feedback prompt name" in error for error in errors))
self.assertTrue(
any("duplicate feedback prompt name" in error for error in errors)
)
self.assertTrue(any("name must be a string" in error for error in errors))
self.assertTrue(any("tokens_for_ai must be a string" in error for error in errors))
self.assertTrue(
any("tokens_for_ai must be a string" in error for error in errors)
)
finally:
os.unlink(temp_file)

439
tests/unit/test_app_feedback.py
View file

@ -26,29 +26,25 @@ class TestAppFeedback(unittest.TestCase):
def setUp(self):
"""Set up test fixtures"""
self.sample_transition = {
"ai_feedback": {
"tokens_for_ai": "Additional transition instructions"
},
"metadata_feedback_filter": ["shot_location", "hit_result", "ship_sunk"]
"ai_feedback": {"tokens_for_ai": "Additional transition instructions"},
"metadata_feedback_filter": ["shot_location", "hit_result", "ship_sunk"],
}
self.sample_metadata = {
"shot_location": "A5",
"hit_result": "hit",
"ship_sunk": "destroyer",
"private_info": "should_be_filtered",
"player_health": 100
}
self.sample_new_metadata = {
"new_shot": "B3",
"new_result": "miss"
"player_health": 100,
}
self.sample_new_metadata = {"new_shot": "B3", "new_result": "miss"}
def test_provide_feedback_import(self):
"""Test that we can import the provide_feedback function"""
try:
from app import provide_feedback
self.assertTrue(callable(provide_feedback))
except ImportError as e:
self.fail(f"Could not import provide_feedback: {e}")
@ -57,11 +53,12 @@ class TestAppFeedback(unittest.TestCase):
"""Test that we can import the provide_feedback_prompts function"""
try:
from app import provide_feedback_prompts
self.assertTrue(callable(provide_feedback_prompts))
except ImportError as e:
self.fail(f"Could not import provide_feedback_prompts: {e}")
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_provide_feedback_legacy(self, mock_get_client):
"""Test legacy provide_feedback function"""
# Import here to avoid issues if module is not available
@ -90,24 +87,30 @@ class TestAppFeedback(unittest.TestCase):
# Call function
feedback = provide_feedback(
transition, category, question, feedback_tokens_for_ai,
user_response, user_language, username,
json_metadata, json_new_metadata
transition,
category,
question,
feedback_tokens_for_ai,
user_response,
user_language,
username,
json_metadata,
json_new_metadata,
)
# Verify result
self.assertIn("Great shot! You hit the target.", feedback)
# Verify client was called
mock_client.chat.completions.create.assert_called_once()
call_args = mock_client.chat.completions.create.call_args[1]
# Check that system message includes language and transition instructions
system_message = call_args['messages'][0]['content']
system_message = call_args["messages"][0]["content"]
self.assertIn("English", system_message)
self.assertIn("Additional transition instructions", system_message)
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_provide_feedback_prompts_multi(self, mock_get_client):
"""Test provide_feedback_prompts with multiple prompts"""
try:
@ -118,11 +121,18 @@ class TestAppFeedback(unittest.TestCase):
# Setup mock to return different responses for each prompt
mock_client = MagicMock()
mock_completion_1 = MagicMock()
mock_completion_1.choices[0].message.content = "Your shot at A5 was a hit! Enemy shot at B3 missed."
mock_completion_1.choices[0].message.content = (
"Your shot at A5 was a hit! Enemy shot at B3 missed."
)
mock_completion_2 = MagicMock()
mock_completion_2.choices[0].message.content = "The enemy's destroyer has been sunk!"
mock_client.chat.completions.create.side_effect = [mock_completion_1, mock_completion_2]
mock_completion_2.choices[0].message.content = (
"The enemy's destroyer has been sunk!"
)
mock_client.chat.completions.create.side_effect = [
mock_completion_1,
mock_completion_2,
]
mock_get_client.return_value = (mock_client, "test-model")
# Test data
@ -132,12 +142,12 @@ class TestAppFeedback(unittest.TestCase):
feedback_prompts = [
{
"name": "hit_miss_feedback",
"tokens_for_ai": "Report the hit/miss results for both players this turn"
"tokens_for_ai": "Report the hit/miss results for both players this turn",
},
{
"name": "ship_sinking_feedback",
"tokens_for_ai": "Report any ships that were sunk this turn"
}
"tokens_for_ai": "Report any ships that were sunk this turn",
},
]
user_response = "A5"
user_language = "English"
@ -147,26 +157,33 @@ class TestAppFeedback(unittest.TestCase):
# Call function
feedback_messages = provide_feedback_prompts(
transition, category, question, feedback_prompts,
user_response, user_language, username,
json_metadata, json_new_metadata, ""
transition,
category,
question,
feedback_prompts,
user_response,
user_language,
username,
json_metadata,
json_new_metadata,
"",
)
# Verify results
self.assertEqual(len(feedback_messages), 2)
# Check first feedback message
self.assertEqual(feedback_messages[0]["name"], "hit_miss_feedback")
self.assertIn("Your shot at A5 was a hit", feedback_messages[0]["content"])
# Check second feedback message
self.assertEqual(feedback_messages[1]["name"], "ship_sinking_feedback")
self.assertIn("destroyer has been sunk", feedback_messages[1]["content"])
# Verify client was called twice
self.assertEqual(mock_client.chat.completions.create.call_count, 2)
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_provide_feedback_with_filtered_metadata(self, mock_get_client):
"""Test that provide_feedback works correctly with pre-filtered metadata"""
try:
@ -183,30 +200,37 @@ class TestAppFeedback(unittest.TestCase):
# Simulate app.py behavior: filter metadata before calling provide_feedback
filtered_metadata = {
k: v for k, v in self.sample_metadata.items()
k: v
for k, v in self.sample_metadata.items()
if k in self.sample_transition["metadata_feedback_filter"]
}
provide_feedback(
self.sample_transition, "test", "Question?", "tokens",
"response", "English", "user",
json.dumps(filtered_metadata), json.dumps({})
self.sample_transition,
"test",
"Question?",
"tokens",
"response",
"English",
"user",
json.dumps(filtered_metadata),
json.dumps({}),
)
# Check that user message contains only filtered metadata
call_args = mock_client.chat.completions.create.call_args[1]
user_message = call_args['messages'][1]['content']
user_message = call_args["messages"][1]["content"]
# Should contain filtered fields
self.assertIn("shot_location", user_message)
self.assertIn("hit_result", user_message)
self.assertIn("ship_sunk", user_message)
# Should NOT contain unfiltered fields (because we pre-filtered)
self.assertNotIn("private_info", user_message)
self.assertNotIn("player_health", user_message)
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_provide_feedback_no_filter(self, mock_get_client):
"""Test feedback when no metadata filter is specified"""
try:
@ -222,23 +246,31 @@ class TestAppFeedback(unittest.TestCase):
mock_get_client.return_value = (mock_client, "test-model")
# Call function without metadata filter
transition = {"ai_feedback": {"tokens_for_ai": "Generate feedback"}} # No metadata_feedback_filter
transition = {
"ai_feedback": {"tokens_for_ai": "Generate feedback"}
} # No metadata_feedback_filter
provide_feedback(
transition, "test", "Question?", "tokens",
"response", "English", "user",
json.dumps(self.sample_metadata), json.dumps({})
transition,
"test",
"Question?",
"tokens",
"response",
"English",
"user",
json.dumps(self.sample_metadata),
json.dumps({}),
)
# Check that user message contains all metadata
call_args = mock_client.chat.completions.create.call_args[1]
user_message = call_args['messages'][1]['content']
user_message = call_args["messages"][1]["content"]
# Should contain all metadata fields when no filter is applied
self.assertIn("private_info", user_message)
self.assertIn("player_health", user_message)
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_provide_feedback_error_handling(self, mock_get_client):
"""Test error handling in feedback functions"""
try:
@ -253,15 +285,21 @@ class TestAppFeedback(unittest.TestCase):
# Call function
feedback = provide_feedback(
{"ai_feedback": {"tokens_for_ai": "Generate feedback"}}, "test", "Question?", "tokens",
"response", "English", "user",
json.dumps({}), json.dumps({})
{"ai_feedback": {"tokens_for_ai": "Generate feedback"}},
"test",
"Question?",
"tokens",
"response",
"English",
"user",
json.dumps({}),
json.dumps({}),
)
# Should handle error gracefully
self.assertIn("Error", feedback)
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_provide_feedback_prompts_filter_empty_and_stfu(self, mock_get_client):
"""Test feedback_prompts with empty results and STFU tokens filtered out"""
try:
@ -274,12 +312,16 @@ class TestAppFeedback(unittest.TestCase):
mock_completion_1 = MagicMock()
mock_completion_1.choices[0].message.content = "" # Empty result
mock_completion_2 = MagicMock()
mock_completion_2.choices[0].message.content = "STFU" # STFU token (should be filtered)
mock_completion_2.choices[0].message.content = (
"STFU" # STFU token (should be filtered)
)
mock_completion_3 = MagicMock()
mock_completion_3.choices[0].message.content = "Valid feedback" # Valid result
mock_client.chat.completions.create.side_effect = [
mock_completion_1, mock_completion_2, mock_completion_3
mock_completion_1,
mock_completion_2,
mock_completion_3,
]
mock_get_client.return_value = (mock_client, "test-model")
@ -287,13 +329,20 @@ class TestAppFeedback(unittest.TestCase):
feedback_prompts = [
{"name": "empty", "tokens_for_ai": "Empty prompt"},
{"name": "stfu", "tokens_for_ai": "STFU prompt"},
{"name": "valid", "tokens_for_ai": "Valid prompt"}
{"name": "valid", "tokens_for_ai": "Valid prompt"},
]
feedback_messages = provide_feedback_prompts(
{}, "test", "Question?", feedback_prompts,
"response", "English", "user",
json.dumps({}), json.dumps({}), ""
{},
"test",
"Question?",
feedback_prompts,
"response",
"English",
"user",
json.dumps({}),
json.dumps({}),
"",
)
# Should only return valid feedback (empty and STFU both filtered out the same way)
@ -301,7 +350,7 @@ class TestAppFeedback(unittest.TestCase):
self.assertEqual(feedback_messages[0]["name"], "valid")
self.assertEqual(feedback_messages[0]["content"], "Valid feedback")
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_provide_feedback_prompts_stfu_partial_not_filtered(self, mock_get_client):
"""Test that messages containing STFU as part of larger text are NOT filtered"""
try:
@ -312,14 +361,20 @@ class TestAppFeedback(unittest.TestCase):
# Setup mock to return STFU as part of larger message
mock_client = MagicMock()
mock_completion_1 = MagicMock()
mock_completion_1.choices[0].message.content = "STFU you rascal." # Should NOT be filtered
mock_completion_1.choices[0].message.content = (
"STFU you rascal." # Should NOT be filtered
)
mock_completion_2 = MagicMock()
mock_completion_2.choices[0].message.content = "Go STFU yourself!" # Should NOT be filtered
mock_completion_2.choices[0].message.content = (
"Go STFU yourself!" # Should NOT be filtered
)
mock_completion_3 = MagicMock()
mock_completion_3.choices[0].message.content = "STFU" # Should be filtered
mock_client.chat.completions.create.side_effect = [
mock_completion_1, mock_completion_2, mock_completion_3
mock_completion_1,
mock_completion_2,
mock_completion_3,
]
mock_get_client.return_value = (mock_client, "test-model")
@ -327,13 +382,20 @@ class TestAppFeedback(unittest.TestCase):
feedback_prompts = [
{"name": "partial1", "tokens_for_ai": "Partial STFU 1"},
{"name": "partial2", "tokens_for_ai": "Partial STFU 2"},
{"name": "exact", "tokens_for_ai": "Exact STFU"}
{"name": "exact", "tokens_for_ai": "Exact STFU"},
]
feedback_messages = provide_feedback_prompts(
{}, "test", "Question?", feedback_prompts,
"response", "English", "user",
json.dumps({}), json.dumps({}), ""
{},
"test",
"Question?",
feedback_prompts,
"response",
"English",
"user",
json.dumps({}),
json.dumps({}),
"",
)
# Should return the two partial STFU messages, but not the exact "STFU"
@ -343,8 +405,10 @@ class TestAppFeedback(unittest.TestCase):
self.assertEqual(feedback_messages[1]["name"], "partial2")
self.assertEqual(feedback_messages[1]["content"], "Go STFU yourself!")
@patch('app.get_openai_client_and_model')
def test_provide_feedback_prompts_per_prompt_metadata_filtering(self, mock_get_client):
@patch("app.get_openai_client_and_model")
def test_provide_feedback_prompts_per_prompt_metadata_filtering(
self, mock_get_client
):
"""Test that each prompt gets its own filtered metadata"""
try:
from app import provide_feedback_prompts
@ -354,70 +418,88 @@ class TestAppFeedback(unittest.TestCase):
# Setup mock to return different responses
mock_client = MagicMock()
mock_completion_1 = MagicMock()
mock_completion_1.choices[0].message.content = "Shot feedback with hit/miss data"
mock_completion_1.choices[0].message.content = (
"Shot feedback with hit/miss data"
)
mock_completion_2 = MagicMock()
mock_completion_2.choices[0].message.content = "Ship feedback with sinking data"
mock_client.chat.completions.create.side_effect = [
mock_completion_1, mock_completion_2
mock_completion_1,
mock_completion_2,
]
mock_get_client.return_value = (mock_client, "test-model")
# Test data with mixed metadata
full_metadata = {
"user_shot": "A5",
"user_hit_result": "hit",
"user_hit_result": "hit",
"ai_shot": "B3",
"ai_hit_result": "miss",
"user_sunk_ship_this_round": "Destroyer",
"ai_sunk_ship_this_round": None,
"game_over": False,
"extra_field": "should_not_appear"
"extra_field": "should_not_appear",
}
feedback_prompts = [
{
"name": "shot_report",
"tokens_for_ai": "Report hit/miss",
"metadata_filter": ["user_shot", "user_hit_result", "ai_shot", "ai_hit_result"]
"metadata_filter": [
"user_shot",
"user_hit_result",
"ai_shot",
"ai_hit_result",
],
},
{
"name": "ship_status",
"name": "ship_status",
"tokens_for_ai": "Report ship sinking",
"metadata_filter": ["user_sunk_ship_this_round", "ai_sunk_ship_this_round"]
}
"metadata_filter": [
"user_sunk_ship_this_round",
"ai_sunk_ship_this_round",
],
},
]
feedback_messages = provide_feedback_prompts(
{}, "test", "Question?", feedback_prompts,
"response", "English", "user",
json.dumps(full_metadata), json.dumps({}), ""
{},
"test",
"Question?",
feedback_prompts,
"response",
"English",
"user",
json.dumps(full_metadata),
json.dumps({}),
"",
)
# Verify both prompts got responses
self.assertEqual(len(feedback_messages), 2)
self.assertEqual(feedback_messages[0]["name"], "shot_report")
self.assertEqual(feedback_messages[1]["name"], "ship_status")
# Verify the first prompt only got shot-related metadata
first_call_args = mock_client.chat.completions.create.call_args_list[0][1]
first_user_message = first_call_args['messages'][1]['content']
first_user_message = first_call_args["messages"][1]["content"]
self.assertIn("user_shot", first_user_message)
self.assertIn("user_hit_result", first_user_message)
self.assertIn("ai_shot", first_user_message)
self.assertIn("ai_hit_result", first_user_message)
self.assertNotIn("user_sunk_ship_this_round", first_user_message)
self.assertNotIn("extra_field", first_user_message)
# Verify the second prompt only got ship-related metadata
# Verify the second prompt only got ship-related metadata
second_call_args = mock_client.chat.completions.create.call_args_list[1][1]
second_user_message = second_call_args['messages'][1]['content']
second_user_message = second_call_args["messages"][1]["content"]
self.assertIn("user_sunk_ship_this_round", second_user_message)
self.assertIn("ai_sunk_ship_this_round", second_user_message)
self.assertNotIn("user_shot", second_user_message)
self.assertNotIn("extra_field", second_user_message)
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_ship_status_metadata_filtering_debug(self, mock_get_client):
"""Debug test to check if Ship Status is getting only the right metadata"""
try:
@ -435,59 +517,74 @@ class TestAppFeedback(unittest.TestCase):
# Test data mimicking the actual battleship scenario
full_metadata = {
"user_shot": "46", # This should NOT appear in Ship Status
"ai_shot": "49", # This should NOT appear in Ship Status
"ai_shot": "49", # This should NOT appear in Ship Status
"user_hit_result": "hit",
"ai_hit_result": "miss",
"ai_hit_result": "miss",
"user_sunk_ship_this_round": "Destroyer", # This SHOULD appear
"ai_sunk_ship_this_round": None, # This SHOULD appear
"ai_sunk_ship_this_round": None, # This SHOULD appear
"game_over": False,
"extra_stuff": "should not appear anywhere"
"extra_stuff": "should not appear anywhere",
}
# Exact structure from battleship YAML
feedback_prompts = [
{
"name": "Shot Report",
"tokens_for_ai": "🎯 Report ONLY the hit/miss results",
"metadata_filter": ["user_shot", "ai_shot", "user_hit_result", "ai_hit_result"]
"metadata_filter": [
"user_shot",
"ai_shot",
"user_hit_result",
"ai_hit_result",
],
},
{
"name": "Ship Status",
"name": "Ship Status",
"tokens_for_ai": "You are the Ship Destruction Oracle",
"metadata_filter": ["user_sunk_ship_this_round", "ai_sunk_ship_this_round"]
}
"metadata_filter": [
"user_sunk_ship_this_round",
"ai_sunk_ship_this_round",
],
},
]
# Call the function
provide_feedback_prompts(
{}, "valid_move", "Question?", feedback_prompts,
"46", "English", "user",
json.dumps(full_metadata), json.dumps({}), ""
{},
"valid_move",
"Question?",
feedback_prompts,
"46",
"English",
"user",
json.dumps(full_metadata),
json.dumps({}),
"",
)
# Check what metadata each prompt actually received
self.assertEqual(mock_client.chat.completions.create.call_count, 2)
# First call should be Shot Report
shot_report_call = mock_client.chat.completions.create.call_args_list[0][1]
shot_report_metadata = shot_report_call['messages'][1]['content']
shot_report_metadata = shot_report_call["messages"][1]["content"]
print("=== SHOT REPORT METADATA ===")
print(shot_report_metadata)
# Shot Report should have shot data but NOT ship destruction data
self.assertIn("user_shot", shot_report_metadata)
self.assertIn("46", shot_report_metadata)
self.assertNotIn("user_sunk_ship_this_round", shot_report_metadata)
self.assertNotIn("Destroyer", shot_report_metadata)
# Second call should be Ship Status
ship_status_call = mock_client.chat.completions.create.call_args_list[1][1]
ship_status_metadata = ship_status_call['messages'][1]['content']
ship_status_metadata = ship_status_call["messages"][1]["content"]
print("=== SHIP STATUS METADATA ===")
print(ship_status_metadata)
# Ship Status should have ship destruction data but NOT shot data
self.assertIn("user_sunk_ship_this_round", ship_status_metadata)
self.assertIn("Destroyer", ship_status_metadata)
@ -502,31 +599,36 @@ class TestAppFeedback(unittest.TestCase):
except ImportError:
self.skipTest("app module not available for testing")
with patch('app.get_openai_client_and_model') as mock_get_client:
with patch("app.get_openai_client_and_model") as mock_get_client:
mock_client = MagicMock()
mock_completion = MagicMock()
mock_completion.choices[0].message.content = "Feedback in Spanish"
mock_client.chat.completions.create.return_value = mock_completion
mock_get_client.return_value = (mock_client, "test-model")
feedback_prompts = [
{"name": "test", "tokens_for_ai": "Base prompt"}
]
feedback_prompts = [{"name": "test", "tokens_for_ai": "Base prompt"}]
# Test with Spanish language
provide_feedback_prompts(
{}, "test", "Question?", feedback_prompts,
"response", "Spanish", "user",
json.dumps({}), json.dumps({}), ""
{},
"test",
"Question?",
feedback_prompts,
"response",
"Spanish",
"user",
json.dumps({}),
json.dumps({}),
"",
)
# Check that system message includes Spanish language instruction
call_args = mock_client.chat.completions.create.call_args[1]
system_message = call_args['messages'][0]['content']
system_message = call_args["messages"][0]["content"]
self.assertIn("Spanish", system_message)
self.assertIn("Base prompt", system_message)
@patch('app.get_openai_client_and_model')
@patch("app.get_openai_client_and_model")
def test_provide_feedback_transition_tokens(self, mock_get_client):
"""Test that transition ai_feedback tokens are included"""
try:
@ -541,27 +643,96 @@ class TestAppFeedback(unittest.TestCase):
mock_get_client.return_value = (mock_client, "test-model")
transition = {
"ai_feedback": {
"tokens_for_ai": "Be more dramatic in your feedback"
}
"ai_feedback": {"tokens_for_ai": "Be more dramatic in your feedback"}
}
feedback_prompts = [
{"name": "test", "tokens_for_ai": "Base prompt"}
]
feedback_prompts = [{"name": "test", "tokens_for_ai": "Base prompt"}]
provide_feedback_prompts(
transition, "test", "Question?", feedback_prompts,
"response", "English", "user",
json.dumps({}), json.dumps({}), ""
transition,
"test",
"Question?",
feedback_prompts,
"response",
"English",
"user",
json.dumps({}),
json.dumps({}),
"",
)
# Check that system message includes both base and transition tokens
call_args = mock_client.chat.completions.create.call_args[1]
system_message = call_args['messages'][0]['content']
system_message = call_args["messages"][0]["content"]
self.assertIn("Base prompt", system_message)
self.assertIn("Be more dramatic in your feedback", system_message)
@patch("app.get_openai_client_and_model")
def test_user_response_filtering_with_metadata_filter(self, mock_get_client):
"""Test that user_response is filtered correctly using metadata_filter approach"""
try:
from app import provide_feedback_prompts
except ImportError:
self.skipTest("app module not available for testing")
# Setup mock
mock_client = MagicMock()
mock_completion = MagicMock()
mock_completion.choices[0].message.content = "Response for prompt"
mock_client.chat.completions.create.return_value = mock_completion
mock_get_client.return_value = (mock_client, "test-model")
# Test feedback prompts - one that includes user_response, one that doesn't
feedback_prompts = [
{
"name": "Shot Report",
"tokens_for_ai": "Report shot positions",
"metadata_filter": [
"user_shot",
"user_response",
], # Includes user_response
},
{
"name": "Ship Status",
"tokens_for_ai": "Report ship status",
"metadata_filter": ["ship_status"], # Does NOT include user_response
},
]
metadata = {"user_shot": "35", "ship_status": "intact"}
user_response = "I choose position 35"
provide_feedback_prompts(
{},
"valid_move",
"Choose position?",
feedback_prompts,
user_response,
"English",
"user",
json.dumps(metadata),
json.dumps({}),
"",
)
# Should have 2 calls
self.assertEqual(mock_client.chat.completions.create.call_count, 2)
# First call (Shot Report) should have user_response
first_call = mock_client.chat.completions.create.call_args_list[0][1]
first_user_message = first_call["messages"][1]["content"]
self.assertIn(
"I choose position 35", first_user_message
) # user_response should be present
# Second call (Ship Status) should NOT have user_response
second_call = mock_client.chat.completions.create.call_args_list[1][1]
second_user_message = second_call["messages"][1]["content"]
self.assertEqual(
second_user_message.count("I choose position 35"), 0
) # user_response should be empty/filtered
if __name__ == "__main__":
unittest.main(verbosity=2)
unittest.main(verbosity=2)

180
tests/unit/test_guarded_ai.py
View file

@ -41,15 +41,20 @@ class TestGuardedAI(unittest.TestCase):
"user_hit_result": "hit",
"ai_hit_result": "miss",
}
self.sample_transition = {
"ai_feedback": {
"tokens_for_ai": "Additional transition-specific instructions"
},
"metadata_feedback_filter": ["user_shot", "ai_shot", "user_hit_result", "ai_hit_result"]
"metadata_feedback_filter": [
"user_shot",
"ai_shot",
"user_hit_result",
"ai_hit_result",
],
}
@patch('guarded_ai.get_openai_client_and_model')
@patch("guarded_ai.get_openai_client_and_model")
def test_categorize_response(self, mock_get_client):
"""Test response categorization"""
# Setup mock
@ -69,20 +74,20 @@ class TestGuardedAI(unittest.TestCase):
# Verify result
self.assertEqual(category, "correct_answer")
# Verify client was called correctly
mock_client.chat.completions.create.assert_called_once()
call_args = mock_client.chat.completions.create.call_args[1]
self.assertEqual(call_args['model'], 'test-model')
self.assertEqual(call_args['max_tokens'], 5)
self.assertEqual(call_args['temperature'], 0)
# Check message content
messages = call_args['messages']
self.assertEqual(len(messages), 2)
self.assertIn("correct_answer, wrong_answer", messages[0]['content'])
self.assertEqual(call_args["model"], "test-model")
self.assertEqual(call_args["max_tokens"], 5)
self.assertEqual(call_args["temperature"], 0)
@patch('guarded_ai.get_openai_client_and_model')
# Check message content
messages = call_args["messages"]
self.assertEqual(len(messages), 2)
self.assertIn("correct_answer, wrong_answer", messages[0]["content"])
@patch("guarded_ai.get_openai_client_and_model")
def test_generate_ai_feedback(self, mock_get_client):
"""Test AI feedback generation"""
# Setup mock
@ -99,23 +104,25 @@ class TestGuardedAI(unittest.TestCase):
tokens_for_ai = "Provide encouraging feedback"
metadata = {"score": 100}
feedback = generate_ai_feedback(category, question, user_response, tokens_for_ai, metadata)
feedback = generate_ai_feedback(
category, question, user_response, tokens_for_ai, metadata
)
# Verify result
self.assertEqual(feedback, "Great job on the math!")
# Verify client was called correctly
mock_client.chat.completions.create.assert_called_once()
call_args = mock_client.chat.completions.create.call_args[1]
self.assertEqual(call_args['model'], 'test-model')
self.assertEqual(call_args['max_tokens'], 250)
self.assertEqual(call_args['temperature'], 0.7)
self.assertEqual(call_args["model"], "test-model")
self.assertEqual(call_args["max_tokens"], 250)
self.assertEqual(call_args["temperature"], 0.7)
@patch('guarded_ai.generate_ai_feedback')
@patch("guarded_ai.generate_ai_feedback")
def test_provide_feedback_legacy(self, mock_generate_feedback):
"""Test legacy single feedback system"""
mock_generate_feedback.return_value = "Good work! Try again."
# Test data
transition = self.sample_transition
category = "partial_understanding"
@ -127,42 +134,50 @@ class TestGuardedAI(unittest.TestCase):
# Call function
feedback = provide_feedback(
transition, category, question, user_response,
user_language, tokens_for_ai, metadata
transition,
category,
question,
user_response,
user_language,
tokens_for_ai,
metadata,
)
# Verify feedback was generated
self.assertIn("AI Feedback:", feedback)
self.assertIn("Good work! Try again.", feedback)
# Verify generate_ai_feedback was called with filtered metadata
mock_generate_feedback.assert_called_once()
call_args = mock_generate_feedback.call_args[0]
self.assertEqual(call_args[0], category) # category
self.assertEqual(call_args[1], question) # question
self.assertEqual(call_args[2], user_response) # user_response
# Check tokens_for_ai includes language and transition instructions
tokens_arg = call_args[3]
self.assertIn("English", tokens_arg)
self.assertIn("Additional transition-specific instructions", tokens_arg)
# Check metadata was filtered
filtered_metadata = call_args[4]
expected_filtered = {k: v for k, v in self.sample_metadata.items()
if k in transition["metadata_feedback_filter"]}
expected_filtered = {
k: v
for k, v in self.sample_metadata.items()
if k in transition["metadata_feedback_filter"]
}
# Since our test metadata doesn't have the filtered keys, it should be empty or contain only matching keys
# But the function should have passed what it received
@patch('guarded_ai.generate_ai_feedback')
@patch("guarded_ai.generate_ai_feedback")
def test_provide_feedback_prompts(self, mock_generate_feedback):
"""Test new multi-prompt feedback system"""
# Setup mock to return different feedback for each prompt
mock_generate_feedback.side_effect = [
"Hit at A5, miss at B3",
"No ships were sunk this round"
"No ships were sunk this round",
]
# Test data
transition = self.sample_transition
category = "valid_move"
@ -170,12 +185,12 @@ class TestGuardedAI(unittest.TestCase):
feedback_prompts = [
{
"name": "hit_miss",
"tokens_for_ai": "Report the hit/miss results for both players"
"tokens_for_ai": "Report the hit/miss results for both players",
},
{
"name": "ship_sinking",
"tokens_for_ai": "Report any ships that were sunk"
}
"name": "ship_sinking",
"tokens_for_ai": "Report any ships that were sunk",
},
]
user_response = "A5"
user_language = "English"
@ -183,47 +198,61 @@ class TestGuardedAI(unittest.TestCase):
# Call function
feedback_messages = provide_feedback_prompts(
transition, category, question, feedback_prompts,
user_response, user_language, metadata, ""
transition,
category,
question,
feedback_prompts,
user_response,
user_language,
metadata,
"",
)
# Verify we got the expected number of feedback messages
self.assertEqual(len(feedback_messages), 2)
# Verify message structure
self.assertEqual(feedback_messages[0]["name"], "hit_miss")
self.assertEqual(feedback_messages[0]["content"], "Hit at A5, miss at B3")
self.assertEqual(feedback_messages[1]["name"], "ship_sinking")
self.assertEqual(feedback_messages[1]["content"], "No ships were sunk this round")
self.assertEqual(
feedback_messages[1]["content"], "No ships were sunk this round"
)
# Verify generate_ai_feedback was called twice
self.assertEqual(mock_generate_feedback.call_count, 2)
@patch('guarded_ai.generate_ai_feedback')
@patch("guarded_ai.generate_ai_feedback")
def test_provide_feedback_prompts_empty_responses(self, mock_generate_feedback):
"""Test that empty feedback responses are filtered out"""
# Setup mock to return empty/whitespace responses
mock_generate_feedback.side_effect = [
"", # Empty response
" ", # Whitespace only
"Valid feedback" # Valid response
"Valid feedback", # Valid response
]
transition = {}
category = "test"
question = "Test?"
feedback_prompts = [
{"name": "empty", "tokens_for_ai": "Empty prompt"},
{"name": "whitespace", "tokens_for_ai": "Whitespace prompt"},
{"name": "valid", "tokens_for_ai": "Valid prompt"}
{"name": "whitespace", "tokens_for_ai": "Whitespace prompt"},
{"name": "valid", "tokens_for_ai": "Valid prompt"},
]
user_response = "Test response"
user_language = "English"
metadata = {}
feedback_messages = provide_feedback_prompts(
transition, category, question, feedback_prompts,
user_response, user_language, metadata, ""
transition,
category,
question,
feedback_prompts,
user_response,
user_language,
metadata,
"",
)
# Should only return the valid feedback message
@ -241,44 +270,53 @@ class TestGuardedAI(unittest.TestCase):
tokens_for_ai = "Base tokens"
metadata = {}
with patch('guarded_ai.generate_ai_feedback') as mock_generate:
with patch("guarded_ai.generate_ai_feedback") as mock_generate:
mock_generate.return_value = "" # Should not be called
feedback = provide_feedback(
transition, category, question, user_response,
user_language, tokens_for_ai, metadata
transition,
category,
question,
user_response,
user_language,
tokens_for_ai,
metadata,
)
# Should NOT call generate_ai_feedback when no ai_feedback in transition
mock_generate.assert_not_called()
self.assertEqual(feedback, "")
@patch.dict('os.environ', {'MODEL_ENDPOINT_0': 'http://test.com', 'MODEL_API_KEY_0': 'test-key'})
@patch.dict(
"os.environ",
{"MODEL_ENDPOINT_0": "http://test.com", "MODEL_API_KEY_0": "test-key"},
)
def test_initialize_model_map(self):
"""Test model map initialization from environment variables"""
with patch('guarded_ai.get_client_for_endpoint') as mock_get_client:
with patch("guarded_ai.get_client_for_endpoint") as mock_get_client:
mock_client = MagicMock()
mock_get_client.return_value = mock_client
# Clear and reinitialize
import guarded_ai
guarded_ai.MODEL_CLIENT_MAP = {}
initialize_model_map()
# Verify client was created and stored
mock_get_client.assert_called_with('http://test.com', 'test-key')
self.assertIn('endpoint_0', guarded_ai.MODEL_CLIENT_MAP)
self.assertEqual(guarded_ai.MODEL_CLIENT_MAP['endpoint_0'][0], mock_client)
mock_get_client.assert_called_with("http://test.com", "test-key")
self.assertIn("endpoint_0", guarded_ai.MODEL_CLIENT_MAP)
self.assertEqual(guarded_ai.MODEL_CLIENT_MAP["endpoint_0"][0], mock_client)
def test_get_openai_client_and_model_default(self):
"""Test getting OpenAI client with default model"""
with patch('guarded_ai.MODEL_CLIENT_MAP', {}):
with patch('guarded_ai.get_client_for_endpoint') as mock_get_client:
with patch("guarded_ai.MODEL_CLIENT_MAP", {}):
with patch("guarded_ai.get_client_for_endpoint") as mock_get_client:
mock_client = MagicMock()
mock_get_client.return_value = mock_client
client, model = get_openai_client_and_model()
# Should return default model name
self.assertEqual(model, "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic")
self.assertEqual(client, mock_client)
@ -286,18 +324,16 @@ class TestGuardedAI(unittest.TestCase):
def test_get_openai_client_and_model_from_map(self):
"""Test getting OpenAI client from model map"""
mock_client = MagicMock()
test_map = {
'endpoint_0': (mock_client, 'http://test.com')
}
with patch('guarded_ai.MODEL_CLIENT_MAP', test_map):
test_map = {"endpoint_0": (mock_client, "http://test.com")}
with patch("guarded_ai.MODEL_CLIENT_MAP", test_map):
client, model = get_openai_client_and_model("test-model")
# Should return client from map
self.assertEqual(client, mock_client)
self.assertEqual(model, "test-model")
@patch('guarded_ai.get_openai_client_and_model')
@patch("guarded_ai.get_openai_client_and_model")
def test_categorize_response_error_handling(self, mock_get_client):
"""Test error handling in categorize_response"""
# Setup mock to raise exception
@ -306,11 +342,11 @@ class TestGuardedAI(unittest.TestCase):
mock_get_client.return_value = (mock_client, "test-model")
category = categorize_response("Test?", "Answer", ["bucket1"], "tokens")
# Should return error string
self.assertIn("Error:", category)
@patch('guarded_ai.get_openai_client_and_model')
@patch("guarded_ai.get_openai_client_and_model")
def test_generate_ai_feedback_error_handling(self, mock_get_client):
"""Test error handling in generate_ai_feedback"""
# Setup mock to raise exception
@ -319,10 +355,10 @@ class TestGuardedAI(unittest.TestCase):
mock_get_client.return_value = (mock_client, "test-model")
feedback = generate_ai_feedback("cat", "Q?", "A", "tokens", {})
# Should return error string
self.assertIn("Error:", feedback)
if __name__ == "__main__":
unittest.main(verbosity=2)
unittest.main(verbosity=2)