refactor the application to focus on advice

app.py

@@ -1,29 +1,23 @@
 import os
-from datetime import datetime
-import random
-from typing import List
+from typing import List, Optional
+from pydantic import BaseModel, Field
 import gradio as gr
-from datasets import load_dataset, Dataset, DatasetDict
-from huggingface_hub import whoami, InferenceClient
-import black  # Add black import
+from datasets import load_dataset
+from huggingface_hub import InferenceClient
+import black
 
 # Initialize the inference client
 HF_TOKEN = os.getenv("HF_TOKEN")
 HF_API_URL = os.getenv("HF_API_URL", "Qwen/Qwen2.5-Coder-32B-Instruct")
-client = InferenceClient(api_key=HF_TOKEN)
+client = InferenceClient(model=HF_API_URL, token=HF_TOKEN)
 
 # Load questions from Hugging Face dataset
-EXAM_MAX_QUESTIONS = int(
-    os.getenv("EXAM_MAX_QUESTIONS", 5)
-)  # Limit quiz to max questions
-EXAM_PASSING_SCORE = float(os.getenv("EXAM_PASSING_SCORE", 0.8))
+EXAM_MAX_QUESTIONS = int(os.getenv("EXAM_MAX_QUESTIONS", 1))
 EXAM_DATASET_ID = "agents-course/smolagents-quiz-data"
 
-
 # prep the dataset for the quiz
 ds = load_dataset(EXAM_DATASET_ID, split="train", download_mode="force_redownload")
-quiz_data = list(ds)  # Convert dataset to list instead of using to_list()
-# random.shuffle(quiz_data)
+quiz_data = list(ds)
 if EXAM_MAX_QUESTIONS:
     quiz_data = quiz_data[:EXAM_MAX_QUESTIONS]
 
@@ -31,6 +25,30 @@ if EXAM_MAX_QUESTIONS:
 HAS_IMAGE_FEATURE = "image" in ds.features
 
 
+class CriterionFeedback(BaseModel):
+    """Feedback for a single assessment criterion"""
+
+    criterion: str = Field(..., description="The assessment criterion being evaluated")
+    met: bool = Field(..., description="Whether the criterion was met")
+    explanation: str = Field(
+        ..., description="Detailed explanation of how well the criterion was met"
+    )
+    improvement_tips: Optional[str] = Field(
+        None, description="Specific tips for improvement if needed"
+    )
+
+
+class CodeFeedback(BaseModel):
+    """Structured feedback for code submission"""
+
+    overall_feedback: str = Field(
+        ..., description="Overall assessment of the code solution"
+    )
+    criteria_feedback: List[CriterionFeedback] = Field(
+        ..., description="Detailed feedback for each assessment criterion"
+    )
+
+
 def format_python_code(code: str) -> str:
     """Format Python code using black."""
     try:
@@ -40,80 +58,95 @@ def format_python_code(code: str) -> str:
         return code
 
 
+EVALUATION_TEMPLATE = """Evaluate this Python code solution:
+
+Challenge:
+{challenge}
+
+Reference Solution:
+```python
+
+{solution}
+
+```
+
+Student's Solution:
+
+```python
+
+{student_code}
+
+```
+
+Assessment Criteria:
+{criteria}
+
+Approach:
+Be highly tollerent of differences in approach, as long as they meet Assessment Criteria.
+
+Provide detailed feedback on how well each criterion was met."""
+
+
 def check_code(
     user_code: str, solution: str, challenge: str, assessment_criteria: List[str]
-):
+) -> dict:
     """
-    Use LLM to evaluate if the user's code solution is correct.
-    Returns True if the solution is correct, False otherwise.
+    Use LLM to evaluate the user's code solution and provide structured feedback.
     """
     # Format both user code and solution
     formatted_user_code = format_python_code(user_code)
     formatted_solution = format_python_code(solution)
 
-    assessment_criteria_str = "\n".join(
-        [f"{i + 1}. {c}" for i, c in enumerate(assessment_criteria)]
-    )
+    # Format criteria as bullet points
+    criteria_text = "\n".join(f"- {c}" for c in assessment_criteria)
 
-    prompt = f"""You are an expert Python programming instructor evaluating a student's code solution.
-
-    Challenge:
-    {challenge}
-
-    Reference Solution:
-    {formatted_solution}
-    
-    Student's Solution:
-    {formatted_user_code}
-    
-    Assessment Criteria:
-    {assessment_criteria_str}
-    
-    Evaluate if the student's solution is functionally equivalent to the reference solution.
-    Consider:
-    1. Does it solve the problem correctly?
-    2. Does it handle edge cases appropriately?
-    3. Does it follow the requirements of the challenge?
-    4. Does it meet the assessment criteria?
-
-    Respond with ONLY "CORRECT" or "INCORRECT" followed by a brief explanation.
-    """
-
-    messages = [{"role": "user", "content": prompt}]
+    # Fill the template
+    prompt = EVALUATION_TEMPLATE.format(
+        challenge=challenge,
+        solution=formatted_solution,
+        student_code=formatted_user_code,
+        criteria=criteria_text,
+    )
 
     try:
-        completion = client.chat.completions.create(
-            model=HF_API_URL,
-            messages=messages,
-            max_tokens=500,
+        # Get structured feedback using response_format with schema from Pydantic model
+        response = client.text_generation(
+            prompt=prompt,
+            grammar={
+                "type": "json_object",
+                "value": CodeFeedback.model_json_schema(),
+            },
         )
 
-        response = completion.choices[0].message.content.strip()
+        # Parse response into Pydantic model
+        feedback = CodeFeedback.model_validate_json(response)
 
-        # Extract the verdict from the response
-        is_correct = response.upper().startswith("CORRECT")
+        # Format the feedback for display
+        formatted_feedback = [
+            f"### Overall Assessment\n{feedback.overall_feedback}\n\n"
+        ]
 
-        # Add the explanation to the status text with emoji
-        explanation = response.split("CORRECT")[1]
-        status = "✅ Correct!" if is_correct else "❌ Incorrect!" + f"\n\n{explanation}"
-        gr.Info(f"{status}\n\n{explanation}")
+        for cf in feedback.criteria_feedback:
+            tip = cf.improvement_tips or ""
+            tip_text = f"\n💡 Tip: {tip}" if tip else ""
 
-        return is_correct
+            formatted_feedback.append(
+                f"### {cf.criterion}\n"
+                f"{'✅' if cf.met else '❌'} {cf.explanation}"
+                f"{tip_text}\n"
+            )
+
+        return {"feedback": "\n".join(formatted_feedback)}
 
     except Exception as e:
-        gr.Warning(f"Error checking code: {str(e)}")
-        # Fall back to simple string comparison if LLM fails
-        status = (
-            "✅ Correct!" if is_correct else "❌ Incorrect!" + f"\n\nError: {str(e)}"
-        )
-        gr.Info(f"{status} (Fallback comparison)")
-        return is_correct
+        gr.Warning(f"Error generating feedback: {str(e)}")
+        return {"feedback": "Unable to generate detailed feedback due to an error."}
 
 
 def on_user_logged_in(token: gr.OAuthToken | None):
     """
     Handle user login state.
-    On a valid token, hide the login button and reveal the Start button while keeping Next and Submit hidden.
+    On a valid token, hide the login button and reveal the Start button while keeping Next hidden.
     Also, clear the question text, code input, status, and image.
     """
     if token is not None:
@@ -121,7 +154,6 @@ def on_user_logged_in(token: gr.OAuthToken | None):
             gr.update(visible=False),  # login_btn hidden
             gr.update(visible=True),  # start_btn shown
             gr.update(visible=False),  # next_btn hidden
-            gr.update(visible=False),  # submit_btn hidden
             "",  # Clear question_text
             gr.update(value="", visible=False),  # Clear code_input
             "",  # Clear status_text
@@ -132,7 +164,6 @@ def on_user_logged_in(token: gr.OAuthToken | None):
             gr.update(visible=True),  # login_btn visible
             gr.update(visible=False),  # start_btn hidden
             gr.update(visible=False),  # next_btn hidden
-            gr.update(visible=False),  # submit_btn hidden
             "",
             gr.update(value="", visible=False),
             "",
@@ -140,92 +171,17 @@ def on_user_logged_in(token: gr.OAuthToken | None):
         )
 
 
-def push_results_to_hub(
-    user_answers: list, token: gr.OAuthToken | None, signed_in_message: str
-):
-    """Push results to Hugging Face Hub."""
-
-    print(f"signed_in_message: {signed_in_message}")
-
-    if not user_answers:  # Check if there are any answers to submit
-        gr.Warning("No answers to submit!")
-        return "No answers to submit!"
-
-    if token is None:
-        gr.Warning("Please log in to Hugging Face before pushing!")
-        return "Please log in to Hugging Face before pushing!"
-
-    # Calculate grade
-    correct_count = sum(1 for answer in user_answers if answer["is_correct"])
-    total_questions = len(user_answers)
-    grade = correct_count / total_questions if total_questions > 0 else 0
-
-    if grade < float(EXAM_PASSING_SCORE):
-        gr.Warning(
-            f"Score {grade:.1%} below passing threshold of {float(EXAM_PASSING_SCORE):.1%}"
-        )
-        return f"You scored {grade:.1%}. Please try again to achieve at least {float(EXAM_PASSING_SCORE):.1%}"
-
-    gr.Info("Submitting answers to the Hub. Please wait...", duration=2)
-
-    user_info = whoami(token=token.token)
-    username = user_info["name"]
-    repo_id = f"{EXAM_DATASET_ID}_responses"
-    submission_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
-    # Create a dataset with the user's answers and metadata
-    submission_data = [
-        {
-            "username": username,
-            "datetime": submission_time,
-            "grade": grade,
-            **answer,  # Include all answer data
-        }
-        for answer in user_answers
-    ]
-
-    try:
-        # Try to load existing dataset
-        existing_ds = load_dataset(repo_id)
-        # Convert to DatasetDict if it isn't already
-        if not isinstance(existing_ds, dict):
-            existing_ds = DatasetDict({"default": existing_ds})
-    except Exception:
-        # If dataset doesn't exist, create empty DatasetDict
-        existing_ds = DatasetDict()
-
-    # Create new dataset from submission
-    new_ds = Dataset.from_list(submission_data)
-
-    # Add or update the split for this user
-    existing_ds[username] = new_ds
-
-    # Push the updated dataset to the Hub
-    existing_ds.push_to_hub(
-        repo_id,
-        private=True,  # Make it private by default since it contains student submissions
-    )
-
-    return f"Your responses have been submitted to the Hub! Final grade: {grade:.1%}"
-
-
 def handle_quiz(question_idx, user_answers, submitted_code, is_start):
     """Handle quiz state and progression"""
-    # Hide the start button once the first question is shown
-    start_btn_update = gr.update(visible=False) if is_start else None
-
-    # If this is the first time (start=True), begin at question_idx=0
     if is_start:
         question_idx = 0
     else:
-        # If not the first question and there's a submission, store the user's last submission
-        if (
-            question_idx < len(quiz_data) and submitted_code.strip()
-        ):  # Only check if there's code
+        # If not the first question and there's a submission, store it
+        if question_idx < len(quiz_data) and submitted_code.strip():
             current_q = quiz_data[question_idx]
             # Format the submitted code before checking
             formatted_code = format_python_code(submitted_code)
-            is_correct = check_code(
+            feedback_dict = check_code(
                 formatted_code,
                 current_q["solution"],
                 current_q["challenge"],
@@ -234,47 +190,69 @@ def handle_quiz(question_idx, user_answers, submitted_code, is_start):
             user_answers.append(
                 {
                     "challenge": current_q["challenge"],
-                    "submitted_code": formatted_code,  # Store formatted code
+                    "submitted_code": formatted_code,
                     "correct_solution": current_q["solution"],
                     "assessment_criteria": current_q["assessment_criteria"],
-                    "is_correct": is_correct,
+                    "feedback": feedback_dict["feedback"],
                 }
             )
         question_idx += 1
 
     # If we've reached the end, show final results
     if question_idx >= len(quiz_data):
-        correct_count = sum(1 for answer in user_answers if answer["is_correct"])
-        grade = correct_count / len(user_answers)
-        results_text = (
-            f"**Quiz Complete!**\n\n"
-            f"Your score: {grade:.1%}\n"
-            f"Passing score: {float(EXAM_PASSING_SCORE):.1%}\n\n"
-            f"Your answers:\n\n"
-        )
+        results_text = """## Quiz Complete! 📚
+This feedback should help you improve your skills. 
+
+⛔️ The feedback uses Qwen/Qwen2.5-Coder-32B-Instruct to compare your response to a gold
+standard solution. As we know, LLMs are not perfect. You should compare your work against
+the assessment criteria if you doubt the feedback. 
+
+Here's your detailed feedback:"""
+
         for idx, answer in enumerate(user_answers):
+            # Format assessment criteria as bullet points
+            criteria_bullets = "\n".join(
+                f"- {c}" for c in answer["assessment_criteria"]
+            )
+
+            # Build the results text piece by piece
             results_text += (
-                f"Question {idx + 1}: {'✅' if answer['is_correct'] else '❌'}\n"
+                f"### Question {idx + 1}: {answer['challenge']}\n\n"
+                "#### Your Solution:\n```python\n"
+                f"{answer['submitted_code']}\n```\n\n"
+                "#### Reference Solution:\n```python\n"
+                f"{answer['correct_solution']}\n```\n\n"
+                "#### Assessment Criteria:\n"
+                f"{criteria_bullets}\n\n"
+                "#### Feedback:\n"
+                f"{answer['feedback']}\n\n"
+                "---\n\n"
             )
 
         return (
             "",  # question_text cleared
             gr.update(value="", visible=False),  # hide code_input
-            f"{'✅ Passed!' if grade >= EXAM_PASSING_SCORE else '❌ Did not pass'}",  # status_text
+            "Review your feedback below to improve your coding skills!",
             question_idx,  # updated question index
             user_answers,  # accumulated answers
-            gr.update(visible=False),  # start_btn hidden for quiz-in-progress
-            gr.update(visible=False),  # next_btn hidden on completion
-            gr.update(visible=True),  # submit_btn shown
-            gr.update(value=results_text, visible=True),  # final_markdown with results
-            gr.update(visible=False),  # question_image hidden on completion
+            gr.update(visible=False),  # start_btn hidden
+            gr.update(visible=False),  # next_btn hidden
+            gr.update(value=results_text, visible=True),  # final_markdown
+            gr.update(visible=False),  # question_image hidden
         )
     else:
         # Show the next question
         q = quiz_data[question_idx]
-        challenge_text = f"## Question {question_idx + 1} \n### {q['challenge']}"
+        # Format assessment criteria as bullet points
+        criteria_bullets = "\n".join(f"- {c}" for c in q["assessment_criteria"])
+        challenge_text = (
+            f"## Question {question_idx + 1}\n\n"
+            f"### Challenge:\n{q['challenge']}\n\n"
+            "### Assessment Criteria:\n"
+            f"{criteria_bullets}"
+        )
 
-        # Only show image if the feature exists and the question has an image
+        # Only show image if the feature exists and question has an image
         show_image = HAS_IMAGE_FEATURE and q.get("image") is not None
         image_update = gr.update(
             value=q.get("image") if show_image else None, visible=show_image
@@ -283,14 +261,13 @@ def handle_quiz(question_idx, user_answers, submitted_code, is_start):
         return (
             challenge_text,  # question_text
             gr.update(value=q["placeholder"], visible=True),  # code_input
-            "Submit your code solution and click 'Next' to continue.",  # status_text
+            "Submit your solution and click 'Next' to continue.",
             question_idx,  # updated question_idx
             user_answers,  # user_answers
             gr.update(visible=False),  # start_btn hidden
             gr.update(visible=True),  # next_btn visible
-            gr.update(visible=False),  # submit_btn hidden
             gr.update(visible=False),  # final_markdown hidden
-            image_update,  # question_image with current question image
+            image_update,  # question_image
         )
 
 
@@ -304,8 +281,8 @@ with gr.Blocks() as demo:
         gr.Markdown(f"## Welcome to the {EXAM_DATASET_ID} Quiz")
     with gr.Row(variant="compact"):
         gr.Markdown(
-            "Log in first, then click 'Start' to begin. Complete each coding challenge, click 'Next', "
-            "and finally click 'Submit' to publish your results to the Hugging Face Hub."
+            "Log in first, then click 'Start' to begin. Complete each coding challenge "
+            "and click 'Next' to proceed. You'll get feedback on your solutions at the end."
         )
 
     with gr.Row(variant="panel"):
@@ -328,7 +305,6 @@ with gr.Blocks() as demo:
         login_btn = gr.LoginButton()
         start_btn = gr.Button("Start")
         next_btn = gr.Button("Next ⏭️", visible=False)
-        submit_btn = gr.Button("Submit ✅", visible=False)
 
     with gr.Row(variant="compact"):
         final_markdown = gr.Markdown("", visible=False)
@@ -340,7 +316,6 @@ with gr.Blocks() as demo:
             login_btn,
             start_btn,
             next_btn,
-            submit_btn,
             question_text,
             code_input,
             status_text,
@@ -359,7 +334,6 @@ with gr.Blocks() as demo:
             user_answers,  # Updated user answers (state)
             start_btn,  # Update for start button (will be hidden)
             next_btn,  # Update for next button (shown for in-progress quiz)
-            submit_btn,  # Update for submit button (hidden until end)
             final_markdown,  # Final results markdown (hidden until quiz ends)
             question_image,  # Image update for the quiz question
         ],
@@ -376,18 +350,10 @@ with gr.Blocks() as demo:
             user_answers,
             start_btn,
             next_btn,
-            submit_btn,
             final_markdown,
             question_image,
         ],
     )
 
-    submit_btn.click(
-        fn=push_results_to_hub,
-        inputs=[user_answers, login_btn],
-        outputs=status_text,
-    )
-
-
 if __name__ == "__main__":
     demo.launch()