From 1c347ea06048eb79d8296d62391c46e5c8b2c917 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 8 Nov 2025 18:58:29 +0000 Subject: [PATCH] Add classifier_model and feedback_model support to YAML schema Allow activities to specify separate models for classification and feedback: - classifier_model: Used for categorizing user responses into buckets - feedback_model: Used for generating AI feedback and translations Both fields can be set at activity level (defaults) and overridden at step level. Updated activity37 to use: - MODEL_1 (Hermes) for classification - MODEL_3 (Qwen 3 Coder) for feedback This allows using specialized models for different tasks, e.g., fast classification with accurate feedback generation from domain-specific models. --- activity.py | 39 +++++++++++++------ activity_yaml_validator.py | 21 ++++++++++ .../activity37-programming-languages.yaml | 6 +++ 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/activity.py b/activity.py index ccee539..4a98546 100644 --- a/activity.py +++ b/activity.py @@ -91,7 +91,7 @@ def get_activity_content(file_path): def loop_through_steps_until_question( - activity_content, activity_state, room_name, username, model=None + activity_content, activity_state, room_name, username, classifier_model=None, feedback_model=None ): room = get_room(room_name) @@ -122,7 +122,7 @@ def loop_through_steps_until_question( # Emit the current step content blocks if "content_blocks" in step: content = "\n\n".join(step["content_blocks"]) - translated_content = translate_text(content, user_language, model) + translated_content = translate_text(content, user_language, feedback_model) new_message = Message( username="System", content=translated_content, room_id=room.id ) @@ -144,7 +144,7 @@ def loop_through_steps_until_question( if "question" in step: question_content = step["question"] translated_question_content = translate_text( - question_content, user_language, model + question_content, user_language, feedback_model ) new_message = Message( username="System (Question)", @@ -185,7 +185,7 @@ def loop_through_steps_until_question( # Activity completed # Display activity info before completing - display_activity_info(room_name, username, model) + display_activity_info(room_name, username, feedback_model) db.session.delete(activity_state) db.session.commit() @@ -222,9 +222,14 @@ def start_activity(room_name, s3_file_path, username): db.session.add(activity_state) db.session.commit() + # Get model configuration from activity content if specified + classifier_model = activity_content.get("classifier_model", None) + feedback_model = activity_content.get("feedback_model", None) + # Loop through steps until a question is found or the end is reached loop_through_steps_until_question( - activity_content, activity_state, room_name, username, model=None + activity_content, activity_state, room_name, username, + classifier_model=classifier_model, feedback_model=feedback_model ) # Emit activity status update @@ -340,6 +345,10 @@ def handle_activity_response(room_name, user_response, username, model=None): # Load the activity content activity_content = get_activity_content(activity_state.s3_file_path) + # Get activity-level model defaults + default_classifier_model = activity_content.get("classifier_model", None) + default_feedback_model = activity_content.get("feedback_model", None) + try: # Find the current section and step section = next( @@ -351,6 +360,10 @@ def handle_activity_response(room_name, user_response, username, model=None): s for s in section["steps"] if s["step_id"] == activity_state.step_id ) + # Get step-level model overrides (if specified), otherwise use activity defaults + classifier_model = step.get("classifier_model", default_classifier_model) + feedback_model = step.get("feedback_model", default_feedback_model) + feedback_tokens_for_ai = step.get("feedback_tokens_for_ai", "") # Check if the step has a question @@ -376,7 +389,7 @@ def handle_activity_response(room_name, user_response, username, model=None): user_response, step["buckets"], step.get("tokens_for_ai", ""), - model, + classifier_model, ) # Initialize transition to None @@ -684,7 +697,7 @@ def handle_activity_response(room_name, user_response, username, model=None): if "content_blocks" in transition: transition_content = "\n\n".join(transition["content_blocks"]) translated_transition_content = translate_text( - transition_content, user_language, model + transition_content, user_language, feedback_model ) new_message = Message( username="System", @@ -724,7 +737,7 @@ def handle_activity_response(room_name, user_response, username, model=None): json.dumps(activity_state.dict_metadata), # Pass full metadata json.dumps(new_metadata), feedback_tokens_for_ai, # Pass legacy tokens to be combined - model, + feedback_model, ) feedback_messages.extend(multi_feedback_messages) elif feedback_tokens_for_ai: @@ -748,7 +761,7 @@ def handle_activity_response(room_name, user_response, username, model=None): username, json.dumps(feedback_metadata), json.dumps(new_metadata), - model, + feedback_model, ) if feedback and feedback.strip(): feedback_messages.append( @@ -835,7 +848,8 @@ def handle_activity_response(room_name, user_response, username, model=None): # Loop through steps until a question is found or the end is reached loop_through_steps_until_question( - activity_content, activity_state, room_name, username, model + activity_content, activity_state, room_name, username, + classifier_model=classifier_model, feedback_model=feedback_model ) else: # the user response is any bucket other than correct. @@ -847,7 +861,7 @@ def handle_activity_response(room_name, user_response, username, model=None): # Emit the question again question_content = step["question"] translated_question_content = translate_text( - question_content, user_language, model + question_content, user_language, feedback_model ) new_message = Message( username="System (Question)", @@ -886,7 +900,8 @@ def handle_activity_response(room_name, user_response, username, model=None): else: # Handle steps without a question loop_through_steps_until_question( - activity_content, activity_state, room_name, username, model + activity_content, activity_state, room_name, username, + classifier_model=classifier_model, feedback_model=feedback_model ) except Exception as e: diff --git a/activity_yaml_validator.py b/activity_yaml_validator.py index 3fd84c7..d760143 100644 --- a/activity_yaml_validator.py +++ b/activity_yaml_validator.py @@ -110,6 +110,14 @@ class ActivityYAMLValidator: if not isinstance(data["tokens_for_ai_rubric"], str): self.errors.append("tokens_for_ai_rubric must be a string") + if "classifier_model" in data: + if not isinstance(data["classifier_model"], str): + self.errors.append("classifier_model must be a string") + + if "feedback_model" in data: + if not isinstance(data["feedback_model"], str): + self.errors.append("feedback_model must be a string") + def _validate_sections(self, sections: List[Dict[str, Any]]): """Validate sections structure""" if not isinstance(sections, list): @@ -189,6 +197,19 @@ class ActivityYAMLValidator: f"Section {section_id}, step {step_id}: Missing required field '{field}'" ) + # Validate optional model overrides at step level + if "classifier_model" in step: + if not isinstance(step["classifier_model"], str): + self.errors.append( + f"Section {section_id}, step {step_id}: classifier_model must be a string" + ) + + if "feedback_model" in step: + if not isinstance(step["feedback_model"], str): + self.errors.append( + f"Section {section_id}, step {step_id}: feedback_model must be a string" + ) + # Validate content_blocks or question has_content = "content_blocks" in step has_question = "question" in step diff --git a/research/activity37-programming-languages.yaml b/research/activity37-programming-languages.yaml index 8777966..2efbe9b 100644 --- a/research/activity37-programming-languages.yaml +++ b/research/activity37-programming-languages.yaml @@ -1,5 +1,11 @@ default_max_attempts_per_step: 3 +# Model configuration +# Use Hermes for classification (fast, accurate bucketing) +# Use Qwen 3 Coder for feedback (specialized for code generation) +classifier_model: "MODEL_1" +feedback_model: "MODEL_3" + tokens_for_ai_rubric: | Evaluate the student's understanding of programming concepts in their chosen language. Consider: