From 3b542285f67feae87f054a029787ff1f88e6dc8d Mon Sep 17 00:00:00 2001 From: Russell Ballestrini Date: Thu, 8 Aug 2024 08:59:20 -0400 Subject: [PATCH] counts_as_attempt implemented and we caught up guarded_ai to have metadata modified: app.py modified: research/guarded_ai.py --- app.py | 99 +++++++++++++++++++++--------------------- research/guarded_ai.py | 95 +++++++++++++++++++++++++--------------- 2 files changed, 110 insertions(+), 84 deletions(-) diff --git a/app.py b/app.py index 1182057..2d71b5e 100644 --- a/app.py +++ b/app.py @@ -444,7 +444,8 @@ def handle_message(data): chat_gpt, data["username"], room.name, - model_name="gpt-4o", + # model_name="gpt-4o", + model_name="gpt-4o-2024-08-06", ) if "gpt-mini" in data["message"]: @@ -1983,6 +1984,24 @@ def handle_activity_response(room_name, user_response, username): step["tokens_for_ai"], ) + transition = step["transitions"].get(category, None) + + if transition is None: + # Emit an error message and return early + socketio.emit( + "message", + { + "id": None, + "username": "System", + "content": f"Error: Unrecognized category '{category}'. Please try again.", + }, + room=room_name, + ) + return + + next_section_and_step = transition.get("next_section_and_step", None) + counts_as_attempt = transition.get("counts_as_attempt", True) + # Emit the category to the frontend socketio.emit( "message", @@ -1995,12 +2014,10 @@ def handle_activity_response(room_name, user_response, username): ) # Check metadata conditions for the current step - if "metadata_conditions" in step["transitions"][category]: + if "metadata_conditions" in transition: conditions_met = all( activity_state.dict_metadata.get(key) == value - for key, value in step["transitions"][category][ - "metadata_conditions" - ].items() + for key, value in transition["metadata_conditions"].items() ) if not conditions_met: # Emit a message indicating the conditions are not met @@ -2041,27 +2058,23 @@ def handle_activity_response(room_name, user_response, username): new_metadata = {} # Update metadata based on user actions - if "metadata_add" in step["transitions"][category]: - for key, value in step["transitions"][category][ - "metadata_add" - ].items(): + if "metadata_add" in transition: + for key, value in transition["metadata_add"].items(): if value == "the-users-response": value = user_response new_metadata[key] = value activity_state.add_metadata(key, value) - if "metadata_remove" in step["transitions"][category]: - for key in step["transitions"][category]["metadata_remove"]: + if "metadata_remove" in transition: + for key in transition["metadata_remove"]: activity_state.remove_metadata(key) # Handle metadata_random - if "metadata_random" in step["transitions"][category]: + if "metadata_random" in transition: random_key = random.choice( - list(step["transitions"][category]["metadata_random"].keys()) + list(transition["metadata_random"].keys()) ) - random_value = step["transitions"][category]["metadata_random"][ - random_key - ] + random_value = transition["metadata_random"][random_key] new_metadata[random_key] = random_value activity_state.add_metadata(random_key, random_value) @@ -2070,12 +2083,11 @@ def handle_activity_response(room_name, user_response, username): db.session.commit() # Provide feedback based on the category - feedback, next_section_and_step = provide_feedback( - activity_content, - section["section_id"], - step["step_id"], + feedback = provide_feedback( + transition, category, step["question"], + step["tokens_for_ai"], user_response, username, activity_state.json_metadata, @@ -2101,10 +2113,8 @@ def handle_activity_response(room_name, user_response, username): ) # Emit the transition content blocks if they exist - if "content_blocks" in step["transitions"][category]: - transition_content = "\n\n".join( - step["transitions"][category]["content_blocks"] - ) + if "content_blocks" in transition: + transition_content = "\n\n".join(transition["content_blocks"]) new_message = Message( username="System", content=transition_content, room_id=room.id ) @@ -2166,9 +2176,10 @@ def handle_activity_response(room_name, user_response, username): ) else: # the user response is any bucket other than correct. - activity_state.attempts += 1 - db.session.add(activity_state) - db.session.commit() + if counts_as_attempt: + activity_state.attempts += 1 + db.session.add(activity_state) + db.session.commit() # Emit the question again question_content = f"Question: {step['question']}" @@ -2401,42 +2412,30 @@ def generate_ai_feedback( def provide_feedback( - yaml_content, - section_id, - step_id, + transition, category, question, user_response, + tokens_for_ai, username, json_metadata, json_new_metadata, ): - section = next( - (s for s in yaml_content["sections"] if s["section_id"] == section_id), None - ) - if not section: - return "Section not found.", None - - step = next((s for s in section["steps"] if s["step_id"] == step_id), None) - if not step: - return "Step not found.", None - - transition = step["transitions"].get(category, None) - if not transition: - return "Category not found.", None - feedback = "" if "ai_feedback" in transition: - tokens_for_ai = ( - step["tokens_for_ai"] + " " + transition["ai_feedback"]["tokens_for_ai"] - ) + tokens_for_ai += " " + transition["ai_feedback"]["tokens_for_ai"] ai_feedback = generate_ai_feedback( - category, question, user_response, tokens_for_ai, username, json_metadata, json_new_metadata, + category, + question, + user_response, + tokens_for_ai, + username, + json_metadata, + json_new_metadata, ) feedback += f"\n\nAI Feedback: {ai_feedback}" - next_section_and_step = transition.get("next_section_and_step", None) - return feedback, next_section_and_step + return feedback if __name__ == "__main__": diff --git a/research/guarded_ai.py b/research/guarded_ai.py index 1a4a0f3..9294975 100644 --- a/research/guarded_ai.py +++ b/research/guarded_ai.py @@ -1,5 +1,7 @@ import argparse import yaml +import json +import random from openai import OpenAI client = OpenAI() @@ -64,35 +66,16 @@ def generate_ai_feedback(category, question, user_response, tokens_for_ai): # Provide feedback based on the category -def provide_feedback( - yaml_content, section_id, step_id, category, question, user_response -): - section = next( - (s for s in yaml_content["sections"] if s["section_id"] == section_id), None - ) - if not section: - return "Section not found." - - step = next((s for s in section["steps"] if s["step_id"] == step_id), None) - if not step: - return "Step not found." - - transition = step["transitions"].get(category, None) - if not transition: - return "Category not found." - +def provide_feedback(transition, category, question, user_response, tokens_for_ai): feedback = "" if "ai_feedback" in transition: - tokens_for_ai = ( - step["tokens_for_ai"] + " " + transition["ai_feedback"]["tokens_for_ai"] - ) + tokens_for_ai += " " + transition["ai_feedback"]["tokens_for_ai"] ai_feedback = generate_ai_feedback( category, question, user_response, tokens_for_ai ) feedback += f"\n\nAI Feedback: {ai_feedback}" - next_section_and_step = transition.get("next_section_and_step", None) - return feedback, next_section_and_step + return feedback def get_next_section_and_step(activity_content, current_section_id, current_step_id): @@ -118,7 +101,6 @@ def get_next_section_and_step(activity_content, current_section_id, current_step return None, None -# Simulate the activity def simulate_activity(yaml_file_path): yaml_content = load_yaml_activity(yaml_file_path) max_attempts = yaml_content.get("default_max_attempts_per_step", 3) @@ -126,8 +108,12 @@ def simulate_activity(yaml_file_path): current_section_id = yaml_content["sections"][0]["section_id"] current_step_id = yaml_content["sections"][0]["steps"][0]["step_id"] + metadata = {} + while current_section_id and current_step_id: - print(f"\n\nCurrent section: {current_section_id}, Current step: {current_step_id}\n\n") + print( + f"\n\nCurrent section: {current_section_id}, Current step: {current_step_id}\n\n" + ) section = next( ( s @@ -171,16 +157,47 @@ def simulate_activity(yaml_file_path): ) print(f"\nCategory: {category}") - feedback, next_section_and_step = provide_feedback( - yaml_content, - section["section_id"], - step["step_id"], - category, - question, - user_response, + transition = step["transitions"].get(category, None) + if not transition: + print("\nError: No valid transition found. Please try again.") + continue + + # Check metadata conditions + if "metadata_conditions" in transition: + conditions_met = all( + metadata.get(key) == value + for key, value in transition["metadata_conditions"].items() + ) + if not conditions_met: + print("\nYou do not meet the required conditions to proceed.") + print(f"Current Metadata: {json.dumps(metadata, indent=2)}") + continue + + feedback = provide_feedback( + transition, category, question, user_response, step["tokens_for_ai"] ) print(f"\nFeedback: {feedback}") + # Update metadata based on user actions + if "metadata_add" in transition: + for key, value in transition["metadata_add"].items(): + if value == "the-users-response": + value = user_response + metadata[key] = value + + if "metadata_remove" in transition: + for key in transition["metadata_remove"]: + if key in metadata: + del metadata[key] + + # Handle metadata_random + if "metadata_random" in transition: + random_key = random.choice(list(transition["metadata_random"].keys())) + random_value = transition["metadata_random"][random_key] + metadata[random_key] = random_value + + print(f"\nMetadata: {json.dumps(metadata, indent=2)}") + if category not in [ "off_topic", "asking_clarifying_questions", @@ -188,11 +205,16 @@ def simulate_activity(yaml_file_path): ]: break - attempts += 1 + # Access counts_as_attempt directly from the transition + counts_as_attempt = transition.get("counts_as_attempt", True) + if counts_as_attempt: + attempts += 1 if attempts == max_attempts: print("\nMaximum attempts reached. Moving to the next step.") + # Access next_section_and_step directly from the transition + next_section_and_step = transition.get("next_section_and_step", None) if next_section_and_step: current_section_id, current_step_id = next_section_and_step.split(":") else: @@ -200,9 +222,14 @@ def simulate_activity(yaml_file_path): yaml_content, current_section_id, current_step_id ) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Simulate an activity.") - parser.add_argument("yaml_file_path", type=str, help="Path to the activity YAML file", default="activity0.yaml") + parser.add_argument( + "yaml_file_path", + type=str, + help="Path to the activity YAML file", + default="activity0.yaml", + ) args = parser.parse_args() simulate_activity(args.yaml_file_path) - # simulate_activity("activity13-choose-adventure.yaml")