engineering/opencompletion.com
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add classifier_model and feedback_model support to YAML schema

Browse source 
Allow activities to specify separate models for classification and feedback:
- classifier_model: Used for categorizing user responses into buckets
- feedback_model: Used for generating AI feedback and translations

Both fields can be set at activity level (defaults) and overridden at step level.

Updated activity37 to use:
- MODEL_1 (Hermes) for classification
- MODEL_3 (Qwen 3 Coder) for feedback

This allows using specialized models for different tasks, e.g., fast classification
with accurate feedback generation from domain-specific models.
This commit is contained in:
Claude 2025-11-08 18:58:29 +00:00
parent 11e705be97
commit 1c347ea060
No known key found for this signature in database
3 changed files with 54 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

39
activity.py
View file

@ -91,7 +91,7 @@ def get_activity_content(file_path):
def loop_through_steps_until_question(
activity_content, activity_state, room_name, username, model=None
activity_content, activity_state, room_name, username, classifier_model=None, feedback_model=None
):
room = get_room(room_name)
@ -122,7 +122,7 @@ def loop_through_steps_until_question(
# Emit the current step content blocks
if "content_blocks" in step:
content = "\n\n".join(step["content_blocks"])
translated_content = translate_text(content, user_language, model)
translated_content = translate_text(content, user_language, feedback_model)
new_message = Message(
username="System", content=translated_content, room_id=room.id
)
@ -144,7 +144,7 @@ def loop_through_steps_until_question(
if "question" in step:
question_content = step["question"]
translated_question_content = translate_text(
question_content, user_language, model
question_content, user_language, feedback_model
)
new_message = Message(
username="System (Question)",
@ -185,7 +185,7 @@ def loop_through_steps_until_question(
# Activity completed
# Display activity info before completing
display_activity_info(room_name, username, model)
display_activity_info(room_name, username, feedback_model)
db.session.delete(activity_state)
db.session.commit()
@ -222,9 +222,14 @@ def start_activity(room_name, s3_file_path, username):
db.session.add(activity_state)
db.session.commit()
# Get model configuration from activity content if specified
classifier_model = activity_content.get("classifier_model", None)
feedback_model = activity_content.get("feedback_model", None)
# Loop through steps until a question is found or the end is reached
loop_through_steps_until_question(
activity_content, activity_state, room_name, username, model=None
activity_content, activity_state, room_name, username,
classifier_model=classifier_model, feedback_model=feedback_model
)
# Emit activity status update
@ -340,6 +345,10 @@ def handle_activity_response(room_name, user_response, username, model=None):
# Load the activity content
activity_content = get_activity_content(activity_state.s3_file_path)
# Get activity-level model defaults
default_classifier_model = activity_content.get("classifier_model", None)
default_feedback_model = activity_content.get("feedback_model", None)
try:
# Find the current section and step
section = next(
@ -351,6 +360,10 @@ def handle_activity_response(room_name, user_response, username, model=None):
s for s in section["steps"] if s["step_id"] == activity_state.step_id
)
# Get step-level model overrides (if specified), otherwise use activity defaults
classifier_model = step.get("classifier_model", default_classifier_model)
feedback_model = step.get("feedback_model", default_feedback_model)
feedback_tokens_for_ai = step.get("feedback_tokens_for_ai", "")
# Check if the step has a question
@ -376,7 +389,7 @@ def handle_activity_response(room_name, user_response, username, model=None):
user_response,
step["buckets"],
step.get("tokens_for_ai", ""),
model,
classifier_model,
)
# Initialize transition to None
@ -684,7 +697,7 @@ def handle_activity_response(room_name, user_response, username, model=None):
if "content_blocks" in transition:
transition_content = "\n\n".join(transition["content_blocks"])
translated_transition_content = translate_text(
transition_content, user_language, model
transition_content, user_language, feedback_model
)
new_message = Message(
username="System",
@ -724,7 +737,7 @@ def handle_activity_response(room_name, user_response, username, model=None):
json.dumps(activity_state.dict_metadata), # Pass full metadata
json.dumps(new_metadata),
feedback_tokens_for_ai, # Pass legacy tokens to be combined
model,
feedback_model,
)
feedback_messages.extend(multi_feedback_messages)
elif feedback_tokens_for_ai:
@ -748,7 +761,7 @@ def handle_activity_response(room_name, user_response, username, model=None):
username,
json.dumps(feedback_metadata),
json.dumps(new_metadata),
model,
feedback_model,
)
if feedback and feedback.strip():
feedback_messages.append(
@ -835,7 +848,8 @@ def handle_activity_response(room_name, user_response, username, model=None):
# Loop through steps until a question is found or the end is reached
loop_through_steps_until_question(
activity_content, activity_state, room_name, username, model
activity_content, activity_state, room_name, username,
classifier_model=classifier_model, feedback_model=feedback_model
)
else:
# the user response is any bucket other than correct.
@ -847,7 +861,7 @@ def handle_activity_response(room_name, user_response, username, model=None):
# Emit the question again
question_content = step["question"]
translated_question_content = translate_text(
question_content, user_language, model
question_content, user_language, feedback_model
)
new_message = Message(
username="System (Question)",
@ -886,7 +900,8 @@ def handle_activity_response(room_name, user_response, username, model=None):
else:
# Handle steps without a question
loop_through_steps_until_question(
activity_content, activity_state, room_name, username, model
activity_content, activity_state, room_name, username,
classifier_model=classifier_model, feedback_model=feedback_model
)
except Exception as e:

21
activity_yaml_validator.py
View file

@ -110,6 +110,14 @@ class ActivityYAMLValidator:
if not isinstance(data["tokens_for_ai_rubric"], str):
self.errors.append("tokens_for_ai_rubric must be a string")
if "classifier_model" in data:
if not isinstance(data["classifier_model"], str):
self.errors.append("classifier_model must be a string")
if "feedback_model" in data:
if not isinstance(data["feedback_model"], str):
self.errors.append("feedback_model must be a string")
def _validate_sections(self, sections: List[Dict[str, Any]]):
"""Validate sections structure"""
if not isinstance(sections, list):
@ -189,6 +197,19 @@ class ActivityYAMLValidator:
f"Section {section_id}, step {step_id}: Missing required field '{field}'"
)
# Validate optional model overrides at step level
if "classifier_model" in step:
if not isinstance(step["classifier_model"], str):
self.errors.append(
f"Section {section_id}, step {step_id}: classifier_model must be a string"
)
if "feedback_model" in step:
if not isinstance(step["feedback_model"], str):
self.errors.append(
f"Section {section_id}, step {step_id}: feedback_model must be a string"
)
# Validate content_blocks or question
has_content = "content_blocks" in step
has_question = "question" in step

6
research/activity37-programming-languages.yaml
View file

@ -1,5 +1,11 @@
default_max_attempts_per_step: 3
# Model configuration
# Use Hermes for classification (fast, accurate bucketing)
# Use Qwen 3 Coder for feedback (specialized for code generation)
classifier_model: "MODEL_1"
feedback_model: "MODEL_3"
tokens_for_ai_rubric: |
Evaluate the student's understanding of programming concepts in their chosen language.
Consider: