counts_as_attempt implemented and we caught up guarded_ai to have
metadata modified: app.py modified: research/guarded_ai.py
This commit is contained in:
parent
b0fa97ffc3
commit
3b542285f6
2 changed files with 110 additions and 84 deletions
99
app.py
99
app.py
|
|
@ -444,7 +444,8 @@ def handle_message(data):
|
|||
chat_gpt,
|
||||
data["username"],
|
||||
room.name,
|
||||
model_name="gpt-4o",
|
||||
# model_name="gpt-4o",
|
||||
model_name="gpt-4o-2024-08-06",
|
||||
)
|
||||
|
||||
if "gpt-mini" in data["message"]:
|
||||
|
|
@ -1983,6 +1984,24 @@ def handle_activity_response(room_name, user_response, username):
|
|||
step["tokens_for_ai"],
|
||||
)
|
||||
|
||||
transition = step["transitions"].get(category, None)
|
||||
|
||||
if transition is None:
|
||||
# Emit an error message and return early
|
||||
socketio.emit(
|
||||
"message",
|
||||
{
|
||||
"id": None,
|
||||
"username": "System",
|
||||
"content": f"Error: Unrecognized category '{category}'. Please try again.",
|
||||
},
|
||||
room=room_name,
|
||||
)
|
||||
return
|
||||
|
||||
next_section_and_step = transition.get("next_section_and_step", None)
|
||||
counts_as_attempt = transition.get("counts_as_attempt", True)
|
||||
|
||||
# Emit the category to the frontend
|
||||
socketio.emit(
|
||||
"message",
|
||||
|
|
@ -1995,12 +2014,10 @@ def handle_activity_response(room_name, user_response, username):
|
|||
)
|
||||
|
||||
# Check metadata conditions for the current step
|
||||
if "metadata_conditions" in step["transitions"][category]:
|
||||
if "metadata_conditions" in transition:
|
||||
conditions_met = all(
|
||||
activity_state.dict_metadata.get(key) == value
|
||||
for key, value in step["transitions"][category][
|
||||
"metadata_conditions"
|
||||
].items()
|
||||
for key, value in transition["metadata_conditions"].items()
|
||||
)
|
||||
if not conditions_met:
|
||||
# Emit a message indicating the conditions are not met
|
||||
|
|
@ -2041,27 +2058,23 @@ def handle_activity_response(room_name, user_response, username):
|
|||
new_metadata = {}
|
||||
|
||||
# Update metadata based on user actions
|
||||
if "metadata_add" in step["transitions"][category]:
|
||||
for key, value in step["transitions"][category][
|
||||
"metadata_add"
|
||||
].items():
|
||||
if "metadata_add" in transition:
|
||||
for key, value in transition["metadata_add"].items():
|
||||
if value == "the-users-response":
|
||||
value = user_response
|
||||
new_metadata[key] = value
|
||||
activity_state.add_metadata(key, value)
|
||||
|
||||
if "metadata_remove" in step["transitions"][category]:
|
||||
for key in step["transitions"][category]["metadata_remove"]:
|
||||
if "metadata_remove" in transition:
|
||||
for key in transition["metadata_remove"]:
|
||||
activity_state.remove_metadata(key)
|
||||
|
||||
# Handle metadata_random
|
||||
if "metadata_random" in step["transitions"][category]:
|
||||
if "metadata_random" in transition:
|
||||
random_key = random.choice(
|
||||
list(step["transitions"][category]["metadata_random"].keys())
|
||||
list(transition["metadata_random"].keys())
|
||||
)
|
||||
random_value = step["transitions"][category]["metadata_random"][
|
||||
random_key
|
||||
]
|
||||
random_value = transition["metadata_random"][random_key]
|
||||
new_metadata[random_key] = random_value
|
||||
activity_state.add_metadata(random_key, random_value)
|
||||
|
||||
|
|
@ -2070,12 +2083,11 @@ def handle_activity_response(room_name, user_response, username):
|
|||
db.session.commit()
|
||||
|
||||
# Provide feedback based on the category
|
||||
feedback, next_section_and_step = provide_feedback(
|
||||
activity_content,
|
||||
section["section_id"],
|
||||
step["step_id"],
|
||||
feedback = provide_feedback(
|
||||
transition,
|
||||
category,
|
||||
step["question"],
|
||||
step["tokens_for_ai"],
|
||||
user_response,
|
||||
username,
|
||||
activity_state.json_metadata,
|
||||
|
|
@ -2101,10 +2113,8 @@ def handle_activity_response(room_name, user_response, username):
|
|||
)
|
||||
|
||||
# Emit the transition content blocks if they exist
|
||||
if "content_blocks" in step["transitions"][category]:
|
||||
transition_content = "\n\n".join(
|
||||
step["transitions"][category]["content_blocks"]
|
||||
)
|
||||
if "content_blocks" in transition:
|
||||
transition_content = "\n\n".join(transition["content_blocks"])
|
||||
new_message = Message(
|
||||
username="System", content=transition_content, room_id=room.id
|
||||
)
|
||||
|
|
@ -2166,9 +2176,10 @@ def handle_activity_response(room_name, user_response, username):
|
|||
)
|
||||
else:
|
||||
# the user response is any bucket other than correct.
|
||||
activity_state.attempts += 1
|
||||
db.session.add(activity_state)
|
||||
db.session.commit()
|
||||
if counts_as_attempt:
|
||||
activity_state.attempts += 1
|
||||
db.session.add(activity_state)
|
||||
db.session.commit()
|
||||
|
||||
# Emit the question again
|
||||
question_content = f"Question: {step['question']}"
|
||||
|
|
@ -2401,42 +2412,30 @@ def generate_ai_feedback(
|
|||
|
||||
|
||||
def provide_feedback(
|
||||
yaml_content,
|
||||
section_id,
|
||||
step_id,
|
||||
transition,
|
||||
category,
|
||||
question,
|
||||
user_response,
|
||||
tokens_for_ai,
|
||||
username,
|
||||
json_metadata,
|
||||
json_new_metadata,
|
||||
):
|
||||
section = next(
|
||||
(s for s in yaml_content["sections"] if s["section_id"] == section_id), None
|
||||
)
|
||||
if not section:
|
||||
return "Section not found.", None
|
||||
|
||||
step = next((s for s in section["steps"] if s["step_id"] == step_id), None)
|
||||
if not step:
|
||||
return "Step not found.", None
|
||||
|
||||
transition = step["transitions"].get(category, None)
|
||||
if not transition:
|
||||
return "Category not found.", None
|
||||
|
||||
feedback = ""
|
||||
if "ai_feedback" in transition:
|
||||
tokens_for_ai = (
|
||||
step["tokens_for_ai"] + " " + transition["ai_feedback"]["tokens_for_ai"]
|
||||
)
|
||||
tokens_for_ai += " " + transition["ai_feedback"]["tokens_for_ai"]
|
||||
ai_feedback = generate_ai_feedback(
|
||||
category, question, user_response, tokens_for_ai, username, json_metadata, json_new_metadata,
|
||||
category,
|
||||
question,
|
||||
user_response,
|
||||
tokens_for_ai,
|
||||
username,
|
||||
json_metadata,
|
||||
json_new_metadata,
|
||||
)
|
||||
feedback += f"\n\nAI Feedback: {ai_feedback}"
|
||||
|
||||
next_section_and_step = transition.get("next_section_and_step", None)
|
||||
return feedback, next_section_and_step
|
||||
return feedback
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
import argparse
|
||||
import yaml
|
||||
import json
|
||||
import random
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI()
|
||||
|
|
@ -64,35 +66,16 @@ def generate_ai_feedback(category, question, user_response, tokens_for_ai):
|
|||
|
||||
|
||||
# Provide feedback based on the category
|
||||
def provide_feedback(
|
||||
yaml_content, section_id, step_id, category, question, user_response
|
||||
):
|
||||
section = next(
|
||||
(s for s in yaml_content["sections"] if s["section_id"] == section_id), None
|
||||
)
|
||||
if not section:
|
||||
return "Section not found."
|
||||
|
||||
step = next((s for s in section["steps"] if s["step_id"] == step_id), None)
|
||||
if not step:
|
||||
return "Step not found."
|
||||
|
||||
transition = step["transitions"].get(category, None)
|
||||
if not transition:
|
||||
return "Category not found."
|
||||
|
||||
def provide_feedback(transition, category, question, user_response, tokens_for_ai):
|
||||
feedback = ""
|
||||
if "ai_feedback" in transition:
|
||||
tokens_for_ai = (
|
||||
step["tokens_for_ai"] + " " + transition["ai_feedback"]["tokens_for_ai"]
|
||||
)
|
||||
tokens_for_ai += " " + transition["ai_feedback"]["tokens_for_ai"]
|
||||
ai_feedback = generate_ai_feedback(
|
||||
category, question, user_response, tokens_for_ai
|
||||
)
|
||||
feedback += f"\n\nAI Feedback: {ai_feedback}"
|
||||
|
||||
next_section_and_step = transition.get("next_section_and_step", None)
|
||||
return feedback, next_section_and_step
|
||||
return feedback
|
||||
|
||||
|
||||
def get_next_section_and_step(activity_content, current_section_id, current_step_id):
|
||||
|
|
@ -118,7 +101,6 @@ def get_next_section_and_step(activity_content, current_section_id, current_step
|
|||
return None, None
|
||||
|
||||
|
||||
# Simulate the activity
|
||||
def simulate_activity(yaml_file_path):
|
||||
yaml_content = load_yaml_activity(yaml_file_path)
|
||||
max_attempts = yaml_content.get("default_max_attempts_per_step", 3)
|
||||
|
|
@ -126,8 +108,12 @@ def simulate_activity(yaml_file_path):
|
|||
current_section_id = yaml_content["sections"][0]["section_id"]
|
||||
current_step_id = yaml_content["sections"][0]["steps"][0]["step_id"]
|
||||
|
||||
metadata = {}
|
||||
|
||||
while current_section_id and current_step_id:
|
||||
print(f"\n\nCurrent section: {current_section_id}, Current step: {current_step_id}\n\n")
|
||||
print(
|
||||
f"\n\nCurrent section: {current_section_id}, Current step: {current_step_id}\n\n"
|
||||
)
|
||||
section = next(
|
||||
(
|
||||
s
|
||||
|
|
@ -171,16 +157,47 @@ def simulate_activity(yaml_file_path):
|
|||
)
|
||||
print(f"\nCategory: {category}")
|
||||
|
||||
feedback, next_section_and_step = provide_feedback(
|
||||
yaml_content,
|
||||
section["section_id"],
|
||||
step["step_id"],
|
||||
category,
|
||||
question,
|
||||
user_response,
|
||||
transition = step["transitions"].get(category, None)
|
||||
if not transition:
|
||||
print("\nError: No valid transition found. Please try again.")
|
||||
continue
|
||||
|
||||
# Check metadata conditions
|
||||
if "metadata_conditions" in transition:
|
||||
conditions_met = all(
|
||||
metadata.get(key) == value
|
||||
for key, value in transition["metadata_conditions"].items()
|
||||
)
|
||||
if not conditions_met:
|
||||
print("\nYou do not meet the required conditions to proceed.")
|
||||
print(f"Current Metadata: {json.dumps(metadata, indent=2)}")
|
||||
continue
|
||||
|
||||
feedback = provide_feedback(
|
||||
transition, category, question, user_response, step["tokens_for_ai"]
|
||||
)
|
||||
print(f"\nFeedback: {feedback}")
|
||||
|
||||
# Update metadata based on user actions
|
||||
if "metadata_add" in transition:
|
||||
for key, value in transition["metadata_add"].items():
|
||||
if value == "the-users-response":
|
||||
value = user_response
|
||||
metadata[key] = value
|
||||
|
||||
if "metadata_remove" in transition:
|
||||
for key in transition["metadata_remove"]:
|
||||
if key in metadata:
|
||||
del metadata[key]
|
||||
|
||||
# Handle metadata_random
|
||||
if "metadata_random" in transition:
|
||||
random_key = random.choice(list(transition["metadata_random"].keys()))
|
||||
random_value = transition["metadata_random"][random_key]
|
||||
metadata[random_key] = random_value
|
||||
|
||||
print(f"\nMetadata: {json.dumps(metadata, indent=2)}")
|
||||
|
||||
if category not in [
|
||||
"off_topic",
|
||||
"asking_clarifying_questions",
|
||||
|
|
@ -188,11 +205,16 @@ def simulate_activity(yaml_file_path):
|
|||
]:
|
||||
break
|
||||
|
||||
attempts += 1
|
||||
# Access counts_as_attempt directly from the transition
|
||||
counts_as_attempt = transition.get("counts_as_attempt", True)
|
||||
if counts_as_attempt:
|
||||
attempts += 1
|
||||
|
||||
if attempts == max_attempts:
|
||||
print("\nMaximum attempts reached. Moving to the next step.")
|
||||
|
||||
# Access next_section_and_step directly from the transition
|
||||
next_section_and_step = transition.get("next_section_and_step", None)
|
||||
if next_section_and_step:
|
||||
current_section_id, current_step_id = next_section_and_step.split(":")
|
||||
else:
|
||||
|
|
@ -200,9 +222,14 @@ def simulate_activity(yaml_file_path):
|
|||
yaml_content, current_section_id, current_step_id
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Simulate an activity.")
|
||||
parser.add_argument("yaml_file_path", type=str, help="Path to the activity YAML file", default="activity0.yaml")
|
||||
parser.add_argument(
|
||||
"yaml_file_path",
|
||||
type=str,
|
||||
help="Path to the activity YAML file",
|
||||
default="activity0.yaml",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
simulate_activity(args.yaml_file_path)
|
||||
# simulate_activity("activity13-choose-adventure.yaml")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue