llm_prompter/demos/flashcards.py

#!/usr/bin/env python3
import random
import readline
import json
import argparse
import sys
import textwrap
import llm_prompter
from blessings import Terminal

# Prompt written with assistance from ChatGPT; I asked ChatGPT to improve the
# previous, manually-written prompt, and this is what it gave.

prompt = """Determine whether the student's answer is correct based on the given
book answer. If the student's answer is clear, spelling errors and
abbreviations are acceptable; only mark the answer wrong if the student did not
provide the same information or gave less information than the book answer.
Keep in mind that the given question should only be used as context for
interpreting abbreviated and misspelled words in the student's response. If
both the student's answer and the book answer are incorrect but match each
other, mark the student's answer as correct. Your evaluation should be based on
a comparison of the student's answer against the book answer, not against the
question."""

check_function = llm_prompter.LLMFunction(
    prompt,
    llm_prompter.Dictionary(
        question=llm_prompter.String("the question"),
        book_answer=llm_prompter.String("the correct book answer"),
        student_answer=llm_prompter.String("the student's answer"),
    ),
    llm_prompter.Dictionary(
        is_student_correct=llm_prompter.Boolean(
            "whether or not the student is correct"
        )
    ),
)


def check(question, book, student):
    # No sense in using API credits if the answer is obviously right or wrong
    if book.casefold().strip() == student.casefold().strip():
        return True
    if not student.strip():
        return False

    return check_function(
        {"question": question, "book_answer": book, "student_answer": student}
    )["is_student_correct"]


parser = argparse.ArgumentParser()
parser.add_argument("file", help="File containing questions and answers")
parser.add_argument(
    "--no-shuffle",
    "-n",
    help="don't shuffle questions (default is to shuffle)",
    action="store_true",
)
args = parser.parse_args()

t = Terminal()

with open(args.file) as f:
    questions = []

    for number, line in enumerate(f.readlines()):
        if line.strip():
            try:
                question, answer = line.split("::")
            except ValueError:
                print(
                    textwrap.fill(
                        f"Syntax error on line {number+1}: lines must contain `::` exactly once",
                        width=t.width,
                    ),
                    file=sys.stderr,
                )
                sys.exit(1)

            question = question.strip()
            answer = answer.strip()

            if not question:
                print(
                    textwrap.fill(
                        f"Syntax error on line {number+1}: question must not be empty",
                        width=t.width,
                    ),
                    file=sys.stderr,
                )
                sys.exit(1)

            if not answer:
                print(
                    textwrap.fill(
                        f"Syntax error on line {number+1}: answer must not be empty",
                        width=t.width,
                    ),
                    file=sys.stderr,
                )
                sys.exit(1)

            questions.append((question, answer))

if not args.no_shuffle:
    random.shuffle(questions)

print(t.normal + "=" * t.width)
print()

with_answers = []
for question, book_answer in questions:
    print(t.bold_bright_green(textwrap.fill(question, width=t.width)))
    student_answer = input(t.bright_cyan(">>> ") + t.bright_yellow).strip()
    with_answers.append((question, book_answer, student_answer))
    print()

print(t.normal + "=" * t.width)
print()

total = len(with_answers)
right = 0

for question, book_answer, student_answer in with_answers:
    print(t.bright_cyan(textwrap.fill(question, width=t.width)))
    if check(question, book_answer, student_answer):
        print(t.bold_white_on_green(textwrap.fill(book_answer, width=t.width)))
        right += 1
    else:
        print(
            t.bold_white_on_red(
                textwrap.fill(student_answer or "[no response]", width=t.width),
            )
        )
        print(
            t.bold_white_on_green(
                textwrap.fill(
                    book_answer,
                    width=t.width,
                )
            )
        )
    print()

print(f"Correct: {right}/{total} ({round(100*right/total)}%)")
print()
print(t.normal + "=" * t.width)