diff --git a/activity_yaml_validator.py b/activity_yaml_validator.py index c604c52..3fd84c7 100644 --- a/activity_yaml_validator.py +++ b/activity_yaml_validator.py @@ -244,7 +244,9 @@ class ActivityYAMLValidator: # Validate feedback_prompts (new multi-prompt system) if "feedback_prompts" in step: - self._validate_feedback_prompts(step["feedback_prompts"], section_id, step_id) + self._validate_feedback_prompts( + step["feedback_prompts"], section_id, step_id + ) # Validate buckets and transitions if "buckets" in step: @@ -255,7 +257,9 @@ class ActivityYAMLValidator: step["transitions"], step.get("buckets", []), section_id, step_id ) - def _validate_feedback_prompts(self, feedback_prompts: List[Dict[str, Any]], section_id: str, step_id: str): + def _validate_feedback_prompts( + self, feedback_prompts: List[Dict[str, Any]], section_id: str, step_id: str + ): """Validate feedback_prompts structure""" if not isinstance(feedback_prompts, list): self.errors.append( @@ -308,7 +312,7 @@ class ActivityYAMLValidator: elif "STFU" in prompt["tokens_for_ai"]: # This is valid - STFU token is used to suppress empty feedback messages pass - + # Validate metadata_filter (optional) if "metadata_filter" in prompt: if not isinstance(prompt["metadata_filter"], list): diff --git a/app.py b/app.py index ea71569..9228bc6 100644 --- a/app.py +++ b/app.py @@ -2155,7 +2155,7 @@ def handle_activity_response(room_name, user_response, username): # Handle feedback systems feedback_messages = [] - + if "feedback_prompts" in step: # New multi-prompt system - pass full metadata, let each prompt filter multi_feedback_messages = provide_feedback_prompts( @@ -2168,7 +2168,7 @@ def handle_activity_response(room_name, user_response, username): username, json.dumps(activity_state.dict_metadata), # Pass full metadata json.dumps(new_metadata), - feedback_tokens_for_ai # Pass legacy tokens to be combined + feedback_tokens_for_ai, # Pass legacy tokens to be combined ) feedback_messages.extend(multi_feedback_messages) elif feedback_tokens_for_ai: @@ -2181,7 +2181,7 @@ def handle_activity_response(room_name, user_response, username): for k, v in activity_state.dict_metadata.items() if k in filter_keys } - + feedback = provide_feedback( transition, category, @@ -2194,17 +2194,16 @@ def handle_activity_response(room_name, user_response, username): json.dumps(new_metadata), ) if feedback and feedback.strip(): - feedback_messages.append({ - "name": "Feedback", - "content": feedback - }) - + feedback_messages.append( + {"name": "Feedback", "content": feedback} + ) + # Store and emit all feedback messages for feedback_msg in feedback_messages: new_message = Message( username=f"System ({feedback_msg['name'].title()})", - content=feedback_msg['content'], - room_id=room.id + content=feedback_msg["content"], + room_id=room.id, ) db.session.add(new_message) db.session.commit() @@ -2214,7 +2213,7 @@ def handle_activity_response(room_name, user_response, username): { "id": new_message.id, "username": f"System ({feedback_msg['name'].title()})", - "content": feedback_msg['content'], + "content": feedback_msg["content"], }, room=room_name, ) @@ -2626,53 +2625,78 @@ def provide_feedback_prompts( ): """Generate feedback from multiple prompts""" feedback_messages = [] - + # Parse full metadata once for filtering full_metadata = json.loads(json_metadata) - + + # Add user_response to metadata for filtering purposes + full_metadata["user_response"] = user_response + for prompt in feedback_prompts: prompt_name = prompt.get("name", "unnamed") tokens_for_ai = prompt.get("tokens_for_ai", "") - + # Apply per-prompt metadata filtering if specified prompt_metadata = full_metadata if "metadata_filter" in prompt: filter_keys = prompt["metadata_filter"] - prompt_metadata = {k: v for k, v in full_metadata.items() if k in filter_keys} - print(f"DEBUG: Prompt '{prompt_name}' filter_keys: {filter_keys}") - print(f"DEBUG: Prompt '{prompt_name}' filtered metadata: {prompt_metadata}") + prompt_metadata = { + k: v for k, v in full_metadata.items() if k in filter_keys + } + + # Special debug for Ship Status + if prompt_name == "Ship Status": + print(f"DEBUG SHIP STATUS - filter_keys: {filter_keys}") + print(f"DEBUG SHIP STATUS - filtered metadata: {prompt_metadata}") + print( + f"DEBUG SHIP STATUS - user_sunk_ship_this_round = '{prompt_metadata.get('user_sunk_ship_this_round')}'" + ) + print( + f"DEBUG SHIP STATUS - ai_sunk_ship_this_round = '{prompt_metadata.get('ai_sunk_ship_this_round')}'" + ) else: - print(f"DEBUG: Prompt '{prompt_name}' has NO metadata_filter, using full metadata") - print(f"DEBUG: Prompt '{prompt_name}' full metadata: {prompt_metadata}") - + if prompt_name == "Ship Status": + print( + f"DEBUG SHIP STATUS - NO metadata_filter, full metadata: {prompt_metadata}" + ) + # Combine legacy tokens with prompt-specific tokens if legacy_tokens_for_ai: tokens_for_ai = legacy_tokens_for_ai + " " + tokens_for_ai - + # Add language instruction - tokens_for_ai += f" You must provide the feedback in the user's language: {user_language}." - + tokens_for_ai += ( + f" You must provide the feedback in the user's language: {user_language}." + ) + # Add transition-specific AI feedback if present if "ai_feedback" in transition: tokens_for_ai += f" {transition['ai_feedback'].get('tokens_for_ai', '')}" - + + # Determine user_response for this prompt based on metadata filtering + filtered_user_response = user_response + if ( + "metadata_filter" in prompt + and "user_response" not in prompt["metadata_filter"] + ): + filtered_user_response = "" # Remove user response if not in filter + ai_feedback = generate_ai_feedback( category, question, - user_response, + filtered_user_response, tokens_for_ai, username, json.dumps(prompt_metadata), # Use filtered metadata for this prompt json_new_metadata, ) - + # Only add feedback if it has content and isn't exactly the STFU token if ai_feedback and ai_feedback.strip() and ai_feedback.strip() != "STFU": - feedback_messages.append({ - "name": prompt_name, - "content": ai_feedback.strip() - }) - + feedback_messages.append( + {"name": prompt_name, "content": ai_feedback.strip()} + ) + return feedback_messages diff --git a/research/activity29-battleship.yaml b/research/activity29-battleship.yaml index cd42f99..83168e3 100644 --- a/research/activity29-battleship.yaml +++ b/research/activity29-battleship.yaml @@ -205,6 +205,7 @@ sections: - ai_shot - user_hit_result - ai_hit_result + - user_response - name: "Ship Status" tokens_for_ai: | @@ -214,12 +215,17 @@ sections: - user_sunk_ship_this_round: If this contains a ship name like "Cruiser", it means THE USER destroyed an ENEMY ship - ai_sunk_ship_this_round: If this contains a ship name like "Destroyer", it means THE ENEMY destroyed a USER ship - Your responses: - - If user_sunk_ship_this_round has a ship name: "💥 You have destroyed the enemy's [ship name]! It sinks beneath the waves!" - - If ai_sunk_ship_this_round has a ship name: "🔥 The enemy has destroyed your [ship name]! It has been claimed by the sea!" - - If both have ship names: combine both messages above - - If both are null/empty: "STFU" + Examples of when to respond: + - If ai_sunk_ship_this_round = "Submarine": Generate submarine destruction story + - If ai_sunk_ship_this_round = "Carrier": Generate carrier destruction story + - If user_sunk_ship_this_round = "Destroyer": Generate destroyer victory story + - If both = "None": Respond with "STFU" + Your responses: + - If user_sunk_ship_this_round equals "Carrier", "Battleship", "Cruiser", "Submarine", or "Destroyer": "💥 You have destroyed the enemy's [ship name]! Write 3 dramatic sentences describing how this specific type of warship meets its end - does it explode? Break apart? Burn? Implode? Make it cinematic!" + - If ai_sunk_ship_this_round equals "Carrier", "Battleship", "Cruiser", "Submarine", or "Destroyer": "🔥 The enemy has destroyed your [ship name]! Write 3 dramatic sentences describing how this specific type of warship is destroyed - the fire, water, explosions, or structural failure. Make it epic!" + - If both equal ship names: combine both messages above + - If both equal "None" or null: "STFU" Do NOT confuse who destroyed what. user_sunk_ship_this_round = USER victory. ai_sunk_ship_this_round = USER loss. metadata_filter: - user_sunk_ship_this_round diff --git a/research/activity29-testship.yaml b/research/activity29-testship.yaml index bf2556a..c560703 100644 --- a/research/activity29-testship.yaml +++ b/research/activity29-testship.yaml @@ -183,6 +183,7 @@ sections: - ai_shot - user_hit_result - ai_hit_result + - user_response - name: "Ship Status" tokens_for_ai: | @@ -192,12 +193,17 @@ sections: - user_sunk_ship_this_round: If this contains a ship name like "Cruiser", it means THE USER destroyed an ENEMY ship - ai_sunk_ship_this_round: If this contains a ship name like "Destroyer", it means THE ENEMY destroyed a USER ship - Your responses: - - If user_sunk_ship_this_round has a ship name: "💥 You have destroyed the enemy's [ship name]! It sinks beneath the waves!" - - If ai_sunk_ship_this_round has a ship name: "🔥 The enemy has destroyed your [ship name]! It has been claimed by the sea!" - - If both have ship names: combine both messages above - - If both are null/empty: "STFU" + Examples of when to respond: + - If ai_sunk_ship_this_round = "Submarine": Generate submarine destruction story + - If ai_sunk_ship_this_round = "Carrier": Generate carrier destruction story + - If user_sunk_ship_this_round = "Destroyer": Generate destroyer victory story + - If both = "None": Respond with "STFU" + Your responses: + - If user_sunk_ship_this_round equals "Carrier", "Battleship", "Cruiser", "Submarine", or "Destroyer": "💥 You have destroyed the enemy's [ship name]! Write 3 dramatic sentences describing how this specific type of warship meets its end - does it explode? Break apart? Burn? Implode? Make it cinematic!" + - If ai_sunk_ship_this_round equals "Carrier", "Battleship", "Cruiser", "Submarine", or "Destroyer": "🔥 The enemy has destroyed your [ship name]! Write 3 dramatic sentences describing how this specific type of warship is destroyed - the fire, water, explosions, or structural failure. Make it epic!" + - If both equal ship names: combine both messages above + - If both equal "None" or null: "STFU" Do NOT confuse who destroyed what. user_sunk_ship_this_round = USER victory. ai_sunk_ship_this_round = USER loss. metadata_filter: - user_sunk_ship_this_round diff --git a/research/guarded_ai.py b/research/guarded_ai.py index 1914b86..4a3054b 100644 --- a/research/guarded_ai.py +++ b/research/guarded_ai.py @@ -163,39 +163,52 @@ def provide_feedback_prompts( ): """Generate feedback from multiple prompts""" feedback_messages = [] - + + # Add user_response to metadata for filtering purposes + full_metadata = metadata.copy() + full_metadata["user_response"] = user_response + for prompt in feedback_prompts: prompt_name = prompt.get("name", "unnamed") tokens_for_ai = prompt.get("tokens_for_ai", "") - + # Apply per-prompt metadata filtering if specified - prompt_metadata = metadata + prompt_metadata = full_metadata if "metadata_filter" in prompt: filter_keys = prompt["metadata_filter"] - prompt_metadata = {k: v for k, v in metadata.items() if k in filter_keys} - + prompt_metadata = { + k: v for k, v in full_metadata.items() if k in filter_keys + } + # Combine legacy tokens with prompt-specific tokens if legacy_tokens_for_ai: tokens_for_ai = legacy_tokens_for_ai + " " + tokens_for_ai - + # Add language instruction tokens_for_ai += f" Provide the feedback in {user_language}." - + # Add transition-specific AI feedback if present if "ai_feedback" in transition: tokens_for_ai += f" {transition['ai_feedback'].get('tokens_for_ai', '')}" - + + # Determine user_response for this prompt based on metadata filtering + filtered_user_response = user_response + if ( + "metadata_filter" in prompt + and "user_response" not in prompt["metadata_filter"] + ): + filtered_user_response = "" # Remove user response if not in filter + ai_feedback = generate_ai_feedback( - category, question, user_response, tokens_for_ai, prompt_metadata + category, question, filtered_user_response, tokens_for_ai, prompt_metadata ) - + # Only add feedback if it has content and isn't exactly the STFU token if ai_feedback and ai_feedback.strip() and ai_feedback.strip() != "STFU": - feedback_messages.append({ - "name": prompt_name, - "content": ai_feedback.strip() - }) - + feedback_messages.append( + {"name": prompt_name, "content": ai_feedback.strip()} + ) + return feedback_messages @@ -463,7 +476,7 @@ def simulate_activity(yaml_file_path): # Provide feedback based on the category feedback_messages = [] - + if "feedback_prompts" in step: # New multi-prompt system - legacy tokens get combined with each prompt multi_feedback_messages = provide_feedback_prompts( @@ -474,7 +487,9 @@ def simulate_activity(yaml_file_path): user_response, user_language, metadata, - step.get("feedback_tokens_for_ai", "") # Pass legacy tokens to be combined + step.get( + "feedback_tokens_for_ai", "" + ), # Pass legacy tokens to be combined ) feedback_messages.extend(multi_feedback_messages) elif step.get("feedback_tokens_for_ai"): @@ -489,11 +504,8 @@ def simulate_activity(yaml_file_path): metadata, ) if feedback and feedback.strip(): - feedback_messages.append({ - "name": "Feedback", - "content": feedback - }) - + feedback_messages.append({"name": "Feedback", "content": feedback}) + # Display all feedback messages for feedback_msg in feedback_messages: print(f"\n{feedback_msg['name']}: {feedback_msg['content']}") diff --git a/tests/functional/test_activity_flows.py b/tests/functional/test_activity_flows.py index e5f7780..bab1722 100644 --- a/tests/functional/test_activity_flows.py +++ b/tests/functional/test_activity_flows.py @@ -379,7 +379,9 @@ class TestRealActivityFiles(unittest.TestCase): mock_get_client.return_value = (self.mock_client, "test-model") # Load actual activity3.yaml - activity_file = Path(__file__).parent.parent.parent / "research" / "activity3.yaml" + activity_file = ( + Path(__file__).parent.parent.parent / "research" / "activity3.yaml" + ) activity = guarded_ai.load_yaml_activity(str(activity_file)) # Should have section_5 as the terminal section @@ -408,7 +410,9 @@ class TestRealActivityFiles(unittest.TestCase): mock_get_client.return_value = (self.mock_client, "test-model") activity_file = ( - Path(__file__).parent.parent.parent / "research" / "activity17-choose-adventure.yaml" + Path(__file__).parent.parent.parent + / "research" + / "activity17-choose-adventure.yaml" ) activity = guarded_ai.load_yaml_activity(str(activity_file)) @@ -450,7 +454,9 @@ class TestRealActivityFiles(unittest.TestCase): mock_get_client.return_value = (self.mock_client, "test-model") activity_file = ( - Path(__file__).parent.parent.parent / "research" / "activity20-n-plus-1.yaml" + Path(__file__).parent.parent.parent + / "research" + / "activity20-n-plus-1.yaml" ) activity = guarded_ai.load_yaml_activity(str(activity_file)) diff --git a/tests/functional/test_battleship_pre_script.py b/tests/functional/test_battleship_pre_script.py index db9226a..33a8097 100644 --- a/tests/functional/test_battleship_pre_script.py +++ b/tests/functional/test_battleship_pre_script.py @@ -28,7 +28,9 @@ class TestBattleshipPreScript(unittest.TestCase): def test_battleship_yaml_has_pre_script(self): """Test that battleship YAML loads and has pre_script""" activity_file = ( - Path(__file__).parent.parent.parent / "research" / "activity29-battleship.yaml" + Path(__file__).parent.parent.parent + / "research" + / "activity29-battleship.yaml" ) activity = guarded_ai.load_yaml_activity(str(activity_file)) @@ -57,7 +59,9 @@ class TestBattleshipPreScript(unittest.TestCase): def test_battleship_pre_script_execution_simulation(self): """Test simulated battleship pre_script execution""" activity_file = ( - Path(__file__).parent.parent.parent / "research" / "activity29-battleship.yaml" + Path(__file__).parent.parent.parent + / "research" + / "activity29-battleship.yaml" ) activity = guarded_ai.load_yaml_activity(str(activity_file)) @@ -96,7 +100,11 @@ class TestBattleshipPreScript(unittest.TestCase): def test_testship_yaml_has_pre_script(self): """Test that testship YAML also has pre_script""" - activity_file = Path(__file__).parent.parent.parent / "research" / "activity29-testship.yaml" + activity_file = ( + Path(__file__).parent.parent.parent + / "research" + / "activity29-testship.yaml" + ) activity = guarded_ai.load_yaml_activity(str(activity_file)) # Should also have pre_script (same structure as battleship) diff --git a/tests/functional/test_guarded_ai.py b/tests/functional/test_guarded_ai.py index 69f6adf..fc03328 100644 --- a/tests/functional/test_guarded_ai.py +++ b/tests/functional/test_guarded_ai.py @@ -320,7 +320,9 @@ class TestActivityYAMLChanges(unittest.TestCase): """Test that activity3's new terminal section loads correctly""" import guarded_ai as guarded_ai - activity_file = Path(__file__).parent.parent.parent / "research" / "activity3.yaml" + activity_file = ( + Path(__file__).parent.parent.parent / "research" / "activity3.yaml" + ) activity = guarded_ai.load_yaml_activity(str(activity_file)) # Should have section_5 now @@ -348,7 +350,9 @@ class TestActivityYAMLChanges(unittest.TestCase): import guarded_ai as guarded_ai activity_file = ( - Path(__file__).parent.parent.parent / "research" / "activity17-choose-adventure.yaml" + Path(__file__).parent.parent.parent + / "research" + / "activity17-choose-adventure.yaml" ) activity = guarded_ai.load_yaml_activity(str(activity_file)) @@ -375,7 +379,9 @@ class TestActivityYAMLChanges(unittest.TestCase): "activity29-battleship.yaml", "activity29-testship.yaml", ]: - activity_file = Path(__file__).parent.parent.parent / "research" / battleship_file + activity_file = ( + Path(__file__).parent.parent.parent / "research" / battleship_file + ) activity = guarded_ai.load_yaml_activity(str(activity_file)) # Find exit transitions and verify they go to step_4 diff --git a/tests/unit/test_activity_yaml_validator.py b/tests/unit/test_activity_yaml_validator.py index 8c87fe9..390a775 100644 --- a/tests/unit/test_activity_yaml_validator.py +++ b/tests/unit/test_activity_yaml_validator.py @@ -692,15 +692,23 @@ sections: try: is_valid, errors, warnings = self.validator.validate_file(temp_file) self.assertFalse(is_valid) - + # Check for specific error types - self.assertTrue(any("feedback_prompts' must be a list" in error for error in errors)) - self.assertTrue(any("feedback_prompts' cannot be empty" in error for error in errors)) + self.assertTrue( + any("feedback_prompts' must be a list" in error for error in errors) + ) + self.assertTrue( + any("feedback_prompts' cannot be empty" in error for error in errors) + ) self.assertTrue(any("must be a dictionary" in error for error in errors)) self.assertTrue(any("missing required field" in error for error in errors)) - self.assertTrue(any("duplicate feedback prompt name" in error for error in errors)) + self.assertTrue( + any("duplicate feedback prompt name" in error for error in errors) + ) self.assertTrue(any("name must be a string" in error for error in errors)) - self.assertTrue(any("tokens_for_ai must be a string" in error for error in errors)) + self.assertTrue( + any("tokens_for_ai must be a string" in error for error in errors) + ) finally: os.unlink(temp_file) diff --git a/tests/unit/test_app_feedback.py b/tests/unit/test_app_feedback.py index 61d80d9..a60773f 100644 --- a/tests/unit/test_app_feedback.py +++ b/tests/unit/test_app_feedback.py @@ -26,29 +26,25 @@ class TestAppFeedback(unittest.TestCase): def setUp(self): """Set up test fixtures""" self.sample_transition = { - "ai_feedback": { - "tokens_for_ai": "Additional transition instructions" - }, - "metadata_feedback_filter": ["shot_location", "hit_result", "ship_sunk"] + "ai_feedback": {"tokens_for_ai": "Additional transition instructions"}, + "metadata_feedback_filter": ["shot_location", "hit_result", "ship_sunk"], } - + self.sample_metadata = { "shot_location": "A5", "hit_result": "hit", "ship_sunk": "destroyer", "private_info": "should_be_filtered", - "player_health": 100 - } - - self.sample_new_metadata = { - "new_shot": "B3", - "new_result": "miss" + "player_health": 100, } + self.sample_new_metadata = {"new_shot": "B3", "new_result": "miss"} + def test_provide_feedback_import(self): """Test that we can import the provide_feedback function""" try: from app import provide_feedback + self.assertTrue(callable(provide_feedback)) except ImportError as e: self.fail(f"Could not import provide_feedback: {e}") @@ -57,11 +53,12 @@ class TestAppFeedback(unittest.TestCase): """Test that we can import the provide_feedback_prompts function""" try: from app import provide_feedback_prompts + self.assertTrue(callable(provide_feedback_prompts)) except ImportError as e: self.fail(f"Could not import provide_feedback_prompts: {e}") - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_provide_feedback_legacy(self, mock_get_client): """Test legacy provide_feedback function""" # Import here to avoid issues if module is not available @@ -90,24 +87,30 @@ class TestAppFeedback(unittest.TestCase): # Call function feedback = provide_feedback( - transition, category, question, feedback_tokens_for_ai, - user_response, user_language, username, - json_metadata, json_new_metadata + transition, + category, + question, + feedback_tokens_for_ai, + user_response, + user_language, + username, + json_metadata, + json_new_metadata, ) # Verify result self.assertIn("Great shot! You hit the target.", feedback) - + # Verify client was called mock_client.chat.completions.create.assert_called_once() call_args = mock_client.chat.completions.create.call_args[1] - + # Check that system message includes language and transition instructions - system_message = call_args['messages'][0]['content'] + system_message = call_args["messages"][0]["content"] self.assertIn("English", system_message) self.assertIn("Additional transition instructions", system_message) - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_provide_feedback_prompts_multi(self, mock_get_client): """Test provide_feedback_prompts with multiple prompts""" try: @@ -118,11 +121,18 @@ class TestAppFeedback(unittest.TestCase): # Setup mock to return different responses for each prompt mock_client = MagicMock() mock_completion_1 = MagicMock() - mock_completion_1.choices[0].message.content = "Your shot at A5 was a hit! Enemy shot at B3 missed." + mock_completion_1.choices[0].message.content = ( + "Your shot at A5 was a hit! Enemy shot at B3 missed." + ) mock_completion_2 = MagicMock() - mock_completion_2.choices[0].message.content = "The enemy's destroyer has been sunk!" - - mock_client.chat.completions.create.side_effect = [mock_completion_1, mock_completion_2] + mock_completion_2.choices[0].message.content = ( + "The enemy's destroyer has been sunk!" + ) + + mock_client.chat.completions.create.side_effect = [ + mock_completion_1, + mock_completion_2, + ] mock_get_client.return_value = (mock_client, "test-model") # Test data @@ -132,12 +142,12 @@ class TestAppFeedback(unittest.TestCase): feedback_prompts = [ { "name": "hit_miss_feedback", - "tokens_for_ai": "Report the hit/miss results for both players this turn" + "tokens_for_ai": "Report the hit/miss results for both players this turn", }, { "name": "ship_sinking_feedback", - "tokens_for_ai": "Report any ships that were sunk this turn" - } + "tokens_for_ai": "Report any ships that were sunk this turn", + }, ] user_response = "A5" user_language = "English" @@ -147,26 +157,33 @@ class TestAppFeedback(unittest.TestCase): # Call function feedback_messages = provide_feedback_prompts( - transition, category, question, feedback_prompts, - user_response, user_language, username, - json_metadata, json_new_metadata, "" + transition, + category, + question, + feedback_prompts, + user_response, + user_language, + username, + json_metadata, + json_new_metadata, + "", ) # Verify results self.assertEqual(len(feedback_messages), 2) - + # Check first feedback message self.assertEqual(feedback_messages[0]["name"], "hit_miss_feedback") self.assertIn("Your shot at A5 was a hit", feedback_messages[0]["content"]) - + # Check second feedback message self.assertEqual(feedback_messages[1]["name"], "ship_sinking_feedback") self.assertIn("destroyer has been sunk", feedback_messages[1]["content"]) - + # Verify client was called twice self.assertEqual(mock_client.chat.completions.create.call_count, 2) - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_provide_feedback_with_filtered_metadata(self, mock_get_client): """Test that provide_feedback works correctly with pre-filtered metadata""" try: @@ -183,30 +200,37 @@ class TestAppFeedback(unittest.TestCase): # Simulate app.py behavior: filter metadata before calling provide_feedback filtered_metadata = { - k: v for k, v in self.sample_metadata.items() + k: v + for k, v in self.sample_metadata.items() if k in self.sample_transition["metadata_feedback_filter"] } - + provide_feedback( - self.sample_transition, "test", "Question?", "tokens", - "response", "English", "user", - json.dumps(filtered_metadata), json.dumps({}) + self.sample_transition, + "test", + "Question?", + "tokens", + "response", + "English", + "user", + json.dumps(filtered_metadata), + json.dumps({}), ) # Check that user message contains only filtered metadata call_args = mock_client.chat.completions.create.call_args[1] - user_message = call_args['messages'][1]['content'] - + user_message = call_args["messages"][1]["content"] + # Should contain filtered fields self.assertIn("shot_location", user_message) self.assertIn("hit_result", user_message) self.assertIn("ship_sunk", user_message) - + # Should NOT contain unfiltered fields (because we pre-filtered) self.assertNotIn("private_info", user_message) self.assertNotIn("player_health", user_message) - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_provide_feedback_no_filter(self, mock_get_client): """Test feedback when no metadata filter is specified""" try: @@ -222,23 +246,31 @@ class TestAppFeedback(unittest.TestCase): mock_get_client.return_value = (mock_client, "test-model") # Call function without metadata filter - transition = {"ai_feedback": {"tokens_for_ai": "Generate feedback"}} # No metadata_feedback_filter - + transition = { + "ai_feedback": {"tokens_for_ai": "Generate feedback"} + } # No metadata_feedback_filter + provide_feedback( - transition, "test", "Question?", "tokens", - "response", "English", "user", - json.dumps(self.sample_metadata), json.dumps({}) + transition, + "test", + "Question?", + "tokens", + "response", + "English", + "user", + json.dumps(self.sample_metadata), + json.dumps({}), ) # Check that user message contains all metadata call_args = mock_client.chat.completions.create.call_args[1] - user_message = call_args['messages'][1]['content'] - + user_message = call_args["messages"][1]["content"] + # Should contain all metadata fields when no filter is applied self.assertIn("private_info", user_message) self.assertIn("player_health", user_message) - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_provide_feedback_error_handling(self, mock_get_client): """Test error handling in feedback functions""" try: @@ -253,15 +285,21 @@ class TestAppFeedback(unittest.TestCase): # Call function feedback = provide_feedback( - {"ai_feedback": {"tokens_for_ai": "Generate feedback"}}, "test", "Question?", "tokens", - "response", "English", "user", - json.dumps({}), json.dumps({}) + {"ai_feedback": {"tokens_for_ai": "Generate feedback"}}, + "test", + "Question?", + "tokens", + "response", + "English", + "user", + json.dumps({}), + json.dumps({}), ) # Should handle error gracefully self.assertIn("Error", feedback) - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_provide_feedback_prompts_filter_empty_and_stfu(self, mock_get_client): """Test feedback_prompts with empty results and STFU tokens filtered out""" try: @@ -274,12 +312,16 @@ class TestAppFeedback(unittest.TestCase): mock_completion_1 = MagicMock() mock_completion_1.choices[0].message.content = "" # Empty result mock_completion_2 = MagicMock() - mock_completion_2.choices[0].message.content = "STFU" # STFU token (should be filtered) + mock_completion_2.choices[0].message.content = ( + "STFU" # STFU token (should be filtered) + ) mock_completion_3 = MagicMock() mock_completion_3.choices[0].message.content = "Valid feedback" # Valid result - + mock_client.chat.completions.create.side_effect = [ - mock_completion_1, mock_completion_2, mock_completion_3 + mock_completion_1, + mock_completion_2, + mock_completion_3, ] mock_get_client.return_value = (mock_client, "test-model") @@ -287,13 +329,20 @@ class TestAppFeedback(unittest.TestCase): feedback_prompts = [ {"name": "empty", "tokens_for_ai": "Empty prompt"}, {"name": "stfu", "tokens_for_ai": "STFU prompt"}, - {"name": "valid", "tokens_for_ai": "Valid prompt"} + {"name": "valid", "tokens_for_ai": "Valid prompt"}, ] feedback_messages = provide_feedback_prompts( - {}, "test", "Question?", feedback_prompts, - "response", "English", "user", - json.dumps({}), json.dumps({}), "" + {}, + "test", + "Question?", + feedback_prompts, + "response", + "English", + "user", + json.dumps({}), + json.dumps({}), + "", ) # Should only return valid feedback (empty and STFU both filtered out the same way) @@ -301,7 +350,7 @@ class TestAppFeedback(unittest.TestCase): self.assertEqual(feedback_messages[0]["name"], "valid") self.assertEqual(feedback_messages[0]["content"], "Valid feedback") - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_provide_feedback_prompts_stfu_partial_not_filtered(self, mock_get_client): """Test that messages containing STFU as part of larger text are NOT filtered""" try: @@ -312,14 +361,20 @@ class TestAppFeedback(unittest.TestCase): # Setup mock to return STFU as part of larger message mock_client = MagicMock() mock_completion_1 = MagicMock() - mock_completion_1.choices[0].message.content = "STFU you rascal." # Should NOT be filtered + mock_completion_1.choices[0].message.content = ( + "STFU you rascal." # Should NOT be filtered + ) mock_completion_2 = MagicMock() - mock_completion_2.choices[0].message.content = "Go STFU yourself!" # Should NOT be filtered + mock_completion_2.choices[0].message.content = ( + "Go STFU yourself!" # Should NOT be filtered + ) mock_completion_3 = MagicMock() mock_completion_3.choices[0].message.content = "STFU" # Should be filtered - + mock_client.chat.completions.create.side_effect = [ - mock_completion_1, mock_completion_2, mock_completion_3 + mock_completion_1, + mock_completion_2, + mock_completion_3, ] mock_get_client.return_value = (mock_client, "test-model") @@ -327,13 +382,20 @@ class TestAppFeedback(unittest.TestCase): feedback_prompts = [ {"name": "partial1", "tokens_for_ai": "Partial STFU 1"}, {"name": "partial2", "tokens_for_ai": "Partial STFU 2"}, - {"name": "exact", "tokens_for_ai": "Exact STFU"} + {"name": "exact", "tokens_for_ai": "Exact STFU"}, ] feedback_messages = provide_feedback_prompts( - {}, "test", "Question?", feedback_prompts, - "response", "English", "user", - json.dumps({}), json.dumps({}), "" + {}, + "test", + "Question?", + feedback_prompts, + "response", + "English", + "user", + json.dumps({}), + json.dumps({}), + "", ) # Should return the two partial STFU messages, but not the exact "STFU" @@ -343,8 +405,10 @@ class TestAppFeedback(unittest.TestCase): self.assertEqual(feedback_messages[1]["name"], "partial2") self.assertEqual(feedback_messages[1]["content"], "Go STFU yourself!") - @patch('app.get_openai_client_and_model') - def test_provide_feedback_prompts_per_prompt_metadata_filtering(self, mock_get_client): + @patch("app.get_openai_client_and_model") + def test_provide_feedback_prompts_per_prompt_metadata_filtering( + self, mock_get_client + ): """Test that each prompt gets its own filtered metadata""" try: from app import provide_feedback_prompts @@ -354,70 +418,88 @@ class TestAppFeedback(unittest.TestCase): # Setup mock to return different responses mock_client = MagicMock() mock_completion_1 = MagicMock() - mock_completion_1.choices[0].message.content = "Shot feedback with hit/miss data" + mock_completion_1.choices[0].message.content = ( + "Shot feedback with hit/miss data" + ) mock_completion_2 = MagicMock() mock_completion_2.choices[0].message.content = "Ship feedback with sinking data" - + mock_client.chat.completions.create.side_effect = [ - mock_completion_1, mock_completion_2 + mock_completion_1, + mock_completion_2, ] mock_get_client.return_value = (mock_client, "test-model") # Test data with mixed metadata full_metadata = { "user_shot": "A5", - "user_hit_result": "hit", + "user_hit_result": "hit", "ai_shot": "B3", "ai_hit_result": "miss", "user_sunk_ship_this_round": "Destroyer", "ai_sunk_ship_this_round": None, "game_over": False, - "extra_field": "should_not_appear" + "extra_field": "should_not_appear", } - + feedback_prompts = [ { "name": "shot_report", "tokens_for_ai": "Report hit/miss", - "metadata_filter": ["user_shot", "user_hit_result", "ai_shot", "ai_hit_result"] + "metadata_filter": [ + "user_shot", + "user_hit_result", + "ai_shot", + "ai_hit_result", + ], }, { - "name": "ship_status", + "name": "ship_status", "tokens_for_ai": "Report ship sinking", - "metadata_filter": ["user_sunk_ship_this_round", "ai_sunk_ship_this_round"] - } + "metadata_filter": [ + "user_sunk_ship_this_round", + "ai_sunk_ship_this_round", + ], + }, ] feedback_messages = provide_feedback_prompts( - {}, "test", "Question?", feedback_prompts, - "response", "English", "user", - json.dumps(full_metadata), json.dumps({}), "" + {}, + "test", + "Question?", + feedback_prompts, + "response", + "English", + "user", + json.dumps(full_metadata), + json.dumps({}), + "", ) # Verify both prompts got responses self.assertEqual(len(feedback_messages), 2) self.assertEqual(feedback_messages[0]["name"], "shot_report") self.assertEqual(feedback_messages[1]["name"], "ship_status") - + # Verify the first prompt only got shot-related metadata first_call_args = mock_client.chat.completions.create.call_args_list[0][1] - first_user_message = first_call_args['messages'][1]['content'] + first_user_message = first_call_args["messages"][1]["content"] self.assertIn("user_shot", first_user_message) self.assertIn("user_hit_result", first_user_message) self.assertIn("ai_shot", first_user_message) self.assertIn("ai_hit_result", first_user_message) self.assertNotIn("user_sunk_ship_this_round", first_user_message) self.assertNotIn("extra_field", first_user_message) - - # Verify the second prompt only got ship-related metadata + + # Verify the second prompt only got ship-related metadata second_call_args = mock_client.chat.completions.create.call_args_list[1][1] - second_user_message = second_call_args['messages'][1]['content'] + second_user_message = second_call_args["messages"][1]["content"] self.assertIn("user_sunk_ship_this_round", second_user_message) self.assertIn("ai_sunk_ship_this_round", second_user_message) self.assertNotIn("user_shot", second_user_message) self.assertNotIn("extra_field", second_user_message) - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_ship_status_metadata_filtering_debug(self, mock_get_client): """Debug test to check if Ship Status is getting only the right metadata""" try: @@ -435,59 +517,74 @@ class TestAppFeedback(unittest.TestCase): # Test data mimicking the actual battleship scenario full_metadata = { "user_shot": "46", # This should NOT appear in Ship Status - "ai_shot": "49", # This should NOT appear in Ship Status + "ai_shot": "49", # This should NOT appear in Ship Status "user_hit_result": "hit", - "ai_hit_result": "miss", + "ai_hit_result": "miss", "user_sunk_ship_this_round": "Destroyer", # This SHOULD appear - "ai_sunk_ship_this_round": None, # This SHOULD appear + "ai_sunk_ship_this_round": None, # This SHOULD appear "game_over": False, - "extra_stuff": "should not appear anywhere" + "extra_stuff": "should not appear anywhere", } - + # Exact structure from battleship YAML feedback_prompts = [ { "name": "Shot Report", "tokens_for_ai": "🎯 Report ONLY the hit/miss results", - "metadata_filter": ["user_shot", "ai_shot", "user_hit_result", "ai_hit_result"] + "metadata_filter": [ + "user_shot", + "ai_shot", + "user_hit_result", + "ai_hit_result", + ], }, { - "name": "Ship Status", + "name": "Ship Status", "tokens_for_ai": "You are the Ship Destruction Oracle", - "metadata_filter": ["user_sunk_ship_this_round", "ai_sunk_ship_this_round"] - } + "metadata_filter": [ + "user_sunk_ship_this_round", + "ai_sunk_ship_this_round", + ], + }, ] # Call the function provide_feedback_prompts( - {}, "valid_move", "Question?", feedback_prompts, - "46", "English", "user", - json.dumps(full_metadata), json.dumps({}), "" + {}, + "valid_move", + "Question?", + feedback_prompts, + "46", + "English", + "user", + json.dumps(full_metadata), + json.dumps({}), + "", ) # Check what metadata each prompt actually received self.assertEqual(mock_client.chat.completions.create.call_count, 2) - + # First call should be Shot Report shot_report_call = mock_client.chat.completions.create.call_args_list[0][1] - shot_report_metadata = shot_report_call['messages'][1]['content'] - + shot_report_metadata = shot_report_call["messages"][1]["content"] + print("=== SHOT REPORT METADATA ===") print(shot_report_metadata) - + # Shot Report should have shot data but NOT ship destruction data self.assertIn("user_shot", shot_report_metadata) self.assertIn("46", shot_report_metadata) self.assertNotIn("user_sunk_ship_this_round", shot_report_metadata) self.assertNotIn("Destroyer", shot_report_metadata) - + # Second call should be Ship Status ship_status_call = mock_client.chat.completions.create.call_args_list[1][1] - ship_status_metadata = ship_status_call['messages'][1]['content'] - + ship_status_metadata = ship_status_call["messages"][1]["content"] + print("=== SHIP STATUS METADATA ===") print(ship_status_metadata) - + # Ship Status should have ship destruction data but NOT shot data self.assertIn("user_sunk_ship_this_round", ship_status_metadata) self.assertIn("Destroyer", ship_status_metadata) @@ -502,31 +599,36 @@ class TestAppFeedback(unittest.TestCase): except ImportError: self.skipTest("app module not available for testing") - with patch('app.get_openai_client_and_model') as mock_get_client: + with patch("app.get_openai_client_and_model") as mock_get_client: mock_client = MagicMock() mock_completion = MagicMock() mock_completion.choices[0].message.content = "Feedback in Spanish" mock_client.chat.completions.create.return_value = mock_completion mock_get_client.return_value = (mock_client, "test-model") - feedback_prompts = [ - {"name": "test", "tokens_for_ai": "Base prompt"} - ] - + feedback_prompts = [{"name": "test", "tokens_for_ai": "Base prompt"}] + # Test with Spanish language provide_feedback_prompts( - {}, "test", "Question?", feedback_prompts, - "response", "Spanish", "user", - json.dumps({}), json.dumps({}), "" + {}, + "test", + "Question?", + feedback_prompts, + "response", + "Spanish", + "user", + json.dumps({}), + json.dumps({}), + "", ) # Check that system message includes Spanish language instruction call_args = mock_client.chat.completions.create.call_args[1] - system_message = call_args['messages'][0]['content'] + system_message = call_args["messages"][0]["content"] self.assertIn("Spanish", system_message) self.assertIn("Base prompt", system_message) - @patch('app.get_openai_client_and_model') + @patch("app.get_openai_client_and_model") def test_provide_feedback_transition_tokens(self, mock_get_client): """Test that transition ai_feedback tokens are included""" try: @@ -541,27 +643,96 @@ class TestAppFeedback(unittest.TestCase): mock_get_client.return_value = (mock_client, "test-model") transition = { - "ai_feedback": { - "tokens_for_ai": "Be more dramatic in your feedback" - } + "ai_feedback": {"tokens_for_ai": "Be more dramatic in your feedback"} } - - feedback_prompts = [ - {"name": "test", "tokens_for_ai": "Base prompt"} - ] - + + feedback_prompts = [{"name": "test", "tokens_for_ai": "Base prompt"}] + provide_feedback_prompts( - transition, "test", "Question?", feedback_prompts, - "response", "English", "user", - json.dumps({}), json.dumps({}), "" + transition, + "test", + "Question?", + feedback_prompts, + "response", + "English", + "user", + json.dumps({}), + json.dumps({}), + "", ) # Check that system message includes both base and transition tokens call_args = mock_client.chat.completions.create.call_args[1] - system_message = call_args['messages'][0]['content'] + system_message = call_args["messages"][0]["content"] self.assertIn("Base prompt", system_message) self.assertIn("Be more dramatic in your feedback", system_message) + @patch("app.get_openai_client_and_model") + def test_user_response_filtering_with_metadata_filter(self, mock_get_client): + """Test that user_response is filtered correctly using metadata_filter approach""" + try: + from app import provide_feedback_prompts + except ImportError: + self.skipTest("app module not available for testing") + + # Setup mock + mock_client = MagicMock() + mock_completion = MagicMock() + mock_completion.choices[0].message.content = "Response for prompt" + mock_client.chat.completions.create.return_value = mock_completion + mock_get_client.return_value = (mock_client, "test-model") + + # Test feedback prompts - one that includes user_response, one that doesn't + feedback_prompts = [ + { + "name": "Shot Report", + "tokens_for_ai": "Report shot positions", + "metadata_filter": [ + "user_shot", + "user_response", + ], # Includes user_response + }, + { + "name": "Ship Status", + "tokens_for_ai": "Report ship status", + "metadata_filter": ["ship_status"], # Does NOT include user_response + }, + ] + + metadata = {"user_shot": "35", "ship_status": "intact"} + + user_response = "I choose position 35" + + provide_feedback_prompts( + {}, + "valid_move", + "Choose position?", + feedback_prompts, + user_response, + "English", + "user", + json.dumps(metadata), + json.dumps({}), + "", + ) + + # Should have 2 calls + self.assertEqual(mock_client.chat.completions.create.call_count, 2) + + # First call (Shot Report) should have user_response + first_call = mock_client.chat.completions.create.call_args_list[0][1] + first_user_message = first_call["messages"][1]["content"] + self.assertIn( + "I choose position 35", first_user_message + ) # user_response should be present + + # Second call (Ship Status) should NOT have user_response + second_call = mock_client.chat.completions.create.call_args_list[1][1] + second_user_message = second_call["messages"][1]["content"] + self.assertEqual( + second_user_message.count("I choose position 35"), 0 + ) # user_response should be empty/filtered + if __name__ == "__main__": - unittest.main(verbosity=2) \ No newline at end of file + unittest.main(verbosity=2) diff --git a/tests/unit/test_guarded_ai.py b/tests/unit/test_guarded_ai.py index 0712e85..f02eef1 100644 --- a/tests/unit/test_guarded_ai.py +++ b/tests/unit/test_guarded_ai.py @@ -41,15 +41,20 @@ class TestGuardedAI(unittest.TestCase): "user_hit_result": "hit", "ai_hit_result": "miss", } - + self.sample_transition = { "ai_feedback": { "tokens_for_ai": "Additional transition-specific instructions" }, - "metadata_feedback_filter": ["user_shot", "ai_shot", "user_hit_result", "ai_hit_result"] + "metadata_feedback_filter": [ + "user_shot", + "ai_shot", + "user_hit_result", + "ai_hit_result", + ], } - @patch('guarded_ai.get_openai_client_and_model') + @patch("guarded_ai.get_openai_client_and_model") def test_categorize_response(self, mock_get_client): """Test response categorization""" # Setup mock @@ -69,20 +74,20 @@ class TestGuardedAI(unittest.TestCase): # Verify result self.assertEqual(category, "correct_answer") - + # Verify client was called correctly mock_client.chat.completions.create.assert_called_once() call_args = mock_client.chat.completions.create.call_args[1] - self.assertEqual(call_args['model'], 'test-model') - self.assertEqual(call_args['max_tokens'], 5) - self.assertEqual(call_args['temperature'], 0) - - # Check message content - messages = call_args['messages'] - self.assertEqual(len(messages), 2) - self.assertIn("correct_answer, wrong_answer", messages[0]['content']) + self.assertEqual(call_args["model"], "test-model") + self.assertEqual(call_args["max_tokens"], 5) + self.assertEqual(call_args["temperature"], 0) - @patch('guarded_ai.get_openai_client_and_model') + # Check message content + messages = call_args["messages"] + self.assertEqual(len(messages), 2) + self.assertIn("correct_answer, wrong_answer", messages[0]["content"]) + + @patch("guarded_ai.get_openai_client_and_model") def test_generate_ai_feedback(self, mock_get_client): """Test AI feedback generation""" # Setup mock @@ -99,23 +104,25 @@ class TestGuardedAI(unittest.TestCase): tokens_for_ai = "Provide encouraging feedback" metadata = {"score": 100} - feedback = generate_ai_feedback(category, question, user_response, tokens_for_ai, metadata) + feedback = generate_ai_feedback( + category, question, user_response, tokens_for_ai, metadata + ) # Verify result self.assertEqual(feedback, "Great job on the math!") - + # Verify client was called correctly mock_client.chat.completions.create.assert_called_once() call_args = mock_client.chat.completions.create.call_args[1] - self.assertEqual(call_args['model'], 'test-model') - self.assertEqual(call_args['max_tokens'], 250) - self.assertEqual(call_args['temperature'], 0.7) + self.assertEqual(call_args["model"], "test-model") + self.assertEqual(call_args["max_tokens"], 250) + self.assertEqual(call_args["temperature"], 0.7) - @patch('guarded_ai.generate_ai_feedback') + @patch("guarded_ai.generate_ai_feedback") def test_provide_feedback_legacy(self, mock_generate_feedback): """Test legacy single feedback system""" mock_generate_feedback.return_value = "Good work! Try again." - + # Test data transition = self.sample_transition category = "partial_understanding" @@ -127,42 +134,50 @@ class TestGuardedAI(unittest.TestCase): # Call function feedback = provide_feedback( - transition, category, question, user_response, - user_language, tokens_for_ai, metadata + transition, + category, + question, + user_response, + user_language, + tokens_for_ai, + metadata, ) # Verify feedback was generated self.assertIn("AI Feedback:", feedback) self.assertIn("Good work! Try again.", feedback) - + # Verify generate_ai_feedback was called with filtered metadata mock_generate_feedback.assert_called_once() call_args = mock_generate_feedback.call_args[0] self.assertEqual(call_args[0], category) # category self.assertEqual(call_args[1], question) # question self.assertEqual(call_args[2], user_response) # user_response - + # Check tokens_for_ai includes language and transition instructions tokens_arg = call_args[3] self.assertIn("English", tokens_arg) self.assertIn("Additional transition-specific instructions", tokens_arg) - + # Check metadata was filtered filtered_metadata = call_args[4] - expected_filtered = {k: v for k, v in self.sample_metadata.items() - if k in transition["metadata_feedback_filter"]} + expected_filtered = { + k: v + for k, v in self.sample_metadata.items() + if k in transition["metadata_feedback_filter"] + } # Since our test metadata doesn't have the filtered keys, it should be empty or contain only matching keys # But the function should have passed what it received - @patch('guarded_ai.generate_ai_feedback') + @patch("guarded_ai.generate_ai_feedback") def test_provide_feedback_prompts(self, mock_generate_feedback): """Test new multi-prompt feedback system""" # Setup mock to return different feedback for each prompt mock_generate_feedback.side_effect = [ "Hit at A5, miss at B3", - "No ships were sunk this round" + "No ships were sunk this round", ] - + # Test data transition = self.sample_transition category = "valid_move" @@ -170,12 +185,12 @@ class TestGuardedAI(unittest.TestCase): feedback_prompts = [ { "name": "hit_miss", - "tokens_for_ai": "Report the hit/miss results for both players" + "tokens_for_ai": "Report the hit/miss results for both players", }, { - "name": "ship_sinking", - "tokens_for_ai": "Report any ships that were sunk" - } + "name": "ship_sinking", + "tokens_for_ai": "Report any ships that were sunk", + }, ] user_response = "A5" user_language = "English" @@ -183,47 +198,61 @@ class TestGuardedAI(unittest.TestCase): # Call function feedback_messages = provide_feedback_prompts( - transition, category, question, feedback_prompts, - user_response, user_language, metadata, "" + transition, + category, + question, + feedback_prompts, + user_response, + user_language, + metadata, + "", ) # Verify we got the expected number of feedback messages self.assertEqual(len(feedback_messages), 2) - + # Verify message structure self.assertEqual(feedback_messages[0]["name"], "hit_miss") self.assertEqual(feedback_messages[0]["content"], "Hit at A5, miss at B3") self.assertEqual(feedback_messages[1]["name"], "ship_sinking") - self.assertEqual(feedback_messages[1]["content"], "No ships were sunk this round") - + self.assertEqual( + feedback_messages[1]["content"], "No ships were sunk this round" + ) + # Verify generate_ai_feedback was called twice self.assertEqual(mock_generate_feedback.call_count, 2) - @patch('guarded_ai.generate_ai_feedback') + @patch("guarded_ai.generate_ai_feedback") def test_provide_feedback_prompts_empty_responses(self, mock_generate_feedback): """Test that empty feedback responses are filtered out""" # Setup mock to return empty/whitespace responses mock_generate_feedback.side_effect = [ "", # Empty response " ", # Whitespace only - "Valid feedback" # Valid response + "Valid feedback", # Valid response ] - + transition = {} category = "test" question = "Test?" feedback_prompts = [ {"name": "empty", "tokens_for_ai": "Empty prompt"}, - {"name": "whitespace", "tokens_for_ai": "Whitespace prompt"}, - {"name": "valid", "tokens_for_ai": "Valid prompt"} + {"name": "whitespace", "tokens_for_ai": "Whitespace prompt"}, + {"name": "valid", "tokens_for_ai": "Valid prompt"}, ] user_response = "Test response" user_language = "English" metadata = {} feedback_messages = provide_feedback_prompts( - transition, category, question, feedback_prompts, - user_response, user_language, metadata, "" + transition, + category, + question, + feedback_prompts, + user_response, + user_language, + metadata, + "", ) # Should only return the valid feedback message @@ -241,44 +270,53 @@ class TestGuardedAI(unittest.TestCase): tokens_for_ai = "Base tokens" metadata = {} - with patch('guarded_ai.generate_ai_feedback') as mock_generate: + with patch("guarded_ai.generate_ai_feedback") as mock_generate: mock_generate.return_value = "" # Should not be called - + feedback = provide_feedback( - transition, category, question, user_response, - user_language, tokens_for_ai, metadata + transition, + category, + question, + user_response, + user_language, + tokens_for_ai, + metadata, ) # Should NOT call generate_ai_feedback when no ai_feedback in transition mock_generate.assert_not_called() self.assertEqual(feedback, "") - @patch.dict('os.environ', {'MODEL_ENDPOINT_0': 'http://test.com', 'MODEL_API_KEY_0': 'test-key'}) + @patch.dict( + "os.environ", + {"MODEL_ENDPOINT_0": "http://test.com", "MODEL_API_KEY_0": "test-key"}, + ) def test_initialize_model_map(self): """Test model map initialization from environment variables""" - with patch('guarded_ai.get_client_for_endpoint') as mock_get_client: + with patch("guarded_ai.get_client_for_endpoint") as mock_get_client: mock_client = MagicMock() mock_get_client.return_value = mock_client - + # Clear and reinitialize import guarded_ai + guarded_ai.MODEL_CLIENT_MAP = {} initialize_model_map() - + # Verify client was created and stored - mock_get_client.assert_called_with('http://test.com', 'test-key') - self.assertIn('endpoint_0', guarded_ai.MODEL_CLIENT_MAP) - self.assertEqual(guarded_ai.MODEL_CLIENT_MAP['endpoint_0'][0], mock_client) + mock_get_client.assert_called_with("http://test.com", "test-key") + self.assertIn("endpoint_0", guarded_ai.MODEL_CLIENT_MAP) + self.assertEqual(guarded_ai.MODEL_CLIENT_MAP["endpoint_0"][0], mock_client) def test_get_openai_client_and_model_default(self): """Test getting OpenAI client with default model""" - with patch('guarded_ai.MODEL_CLIENT_MAP', {}): - with patch('guarded_ai.get_client_for_endpoint') as mock_get_client: + with patch("guarded_ai.MODEL_CLIENT_MAP", {}): + with patch("guarded_ai.get_client_for_endpoint") as mock_get_client: mock_client = MagicMock() mock_get_client.return_value = mock_client - + client, model = get_openai_client_and_model() - + # Should return default model name self.assertEqual(model, "adamo1139/Hermes-3-Llama-3.1-8B-FP8-Dynamic") self.assertEqual(client, mock_client) @@ -286,18 +324,16 @@ class TestGuardedAI(unittest.TestCase): def test_get_openai_client_and_model_from_map(self): """Test getting OpenAI client from model map""" mock_client = MagicMock() - test_map = { - 'endpoint_0': (mock_client, 'http://test.com') - } - - with patch('guarded_ai.MODEL_CLIENT_MAP', test_map): + test_map = {"endpoint_0": (mock_client, "http://test.com")} + + with patch("guarded_ai.MODEL_CLIENT_MAP", test_map): client, model = get_openai_client_and_model("test-model") - + # Should return client from map self.assertEqual(client, mock_client) self.assertEqual(model, "test-model") - @patch('guarded_ai.get_openai_client_and_model') + @patch("guarded_ai.get_openai_client_and_model") def test_categorize_response_error_handling(self, mock_get_client): """Test error handling in categorize_response""" # Setup mock to raise exception @@ -306,11 +342,11 @@ class TestGuardedAI(unittest.TestCase): mock_get_client.return_value = (mock_client, "test-model") category = categorize_response("Test?", "Answer", ["bucket1"], "tokens") - + # Should return error string self.assertIn("Error:", category) - @patch('guarded_ai.get_openai_client_and_model') + @patch("guarded_ai.get_openai_client_and_model") def test_generate_ai_feedback_error_handling(self, mock_get_client): """Test error handling in generate_ai_feedback""" # Setup mock to raise exception @@ -319,10 +355,10 @@ class TestGuardedAI(unittest.TestCase): mock_get_client.return_value = (mock_client, "test-model") feedback = generate_ai_feedback("cat", "Q?", "A", "tokens", {}) - + # Should return error string self.assertIn("Error:", feedback) if __name__ == "__main__": - unittest.main(verbosity=2) \ No newline at end of file + unittest.main(verbosity=2)