ai-agent-book/chapter2/prompt-engineering/tau_bench/model_utils/model/chat.py

import abc
import enum
import json
from typing import Any, TypeVar

from pydantic import BaseModel

from tau_bench.model_utils.api.datapoint import (
    BinaryClassifyDatapoint,
    ClassifyDatapoint,
    Datapoint,
    GenerateDatapoint,
    ParseDatapoint,
    ParseForceDatapoint,
    ScoreDatapoint,
)
from tau_bench.model_utils.api.types import PartialObj
from tau_bench.model_utils.model.exception import ModelError
from tau_bench.model_utils.model.general_model import GeneralModel
from tau_bench.model_utils.model.utils import (
    add_md_tag,
    clean_top_level_keys,
    display_choices,
    json_response_to_obj_or_partial_obj,
    optionalize_type,
    parse_json_or_json_markdown,
    try_classify_recover,
    type_to_json_schema_string,
)

T = TypeVar("T", bound=BaseModel)


class Role(str, enum.Enum):
    SYSTEM = "system"
    ASSISTANT = "assistant"
    USER = "user"


class Message(BaseModel):
    role: Role
    content: str
    obj: dict[str, Any] | None = None

    def model_dump(self, **kwargs) -> dict[str, Any]:
        if self.obj is not None:
            return super().model_dump(**kwargs)
        return {"role": self.role, "content": self.content}


class PromptSuffixStrategy(str, enum.Enum):
    JSON = "json"
    JSON_MD_BLOCK = "json_md_block"


def force_json_prompt(
    text: str,
    suffix_strategy: PromptSuffixStrategy = PromptSuffixStrategy.JSON,
) -> str:
    if suffix_strategy == PromptSuffixStrategy.JSON:
        return f"{text}\n\nValid JSON:"
    elif suffix_strategy == PromptSuffixStrategy.JSON_MD_BLOCK:
        return f'{text}\n\nThe result should be a valid JSON object (according to the definition in the provided schema) in a markdown block only. For example:\nassistant:```json\n{{"items": ["value"]}}\n```'
    else:
        raise ValueError(f"Invalid suffix strategy: {suffix_strategy}")


def build_generate_state(
    instruction: str,
    text: str,
    examples: list[GenerateDatapoint] | None = None,
) -> list[Message]:
    messages = []
    if examples is not None:
        for example in examples:
            example_msgs = [
                Message(role=Role.SYSTEM, content=example.instruction),
                Message(role=Role.USER, content=example.text),
                Message(role=Role.ASSISTANT, content=example.response),
            ]
            messages.extend(example_msgs)
    messages.append(Message(role=Role.SYSTEM, content=instruction))
    messages.append(Message(role=Role.USER, content=text))
    return messages


def build_parse_force_state(
    instruction: str,
    typ: type[T] | dict[str, Any],
    text: str | None = None,
    examples: list[ParseForceDatapoint] | None = None,
    suffix_strategy: PromptSuffixStrategy = PromptSuffixStrategy.JSON,
) -> list[Message]:
    def display_sample(
        instr: str,
        ty: type[T] | dict[str, Any],
        t: str | None = None,
        response: T | dict[str, Any] | None = None,
    ) -> Message | list[Message]:
        if isinstance(ty, dict):
            json_schema_string = json.dumps(ty)
        else:
            json_schema_string = type_to_json_schema_string(ty)
        text_insert = "" if t is None else f"\n\nText:\n{t}"
        input_text = force_json_prompt(
            text=f"Instruction:\n{instr}{text_insert}\n\nSchema:\n{json_schema_string}",
            suffix_strategy=suffix_strategy,
        )
        if response is not None:
            if isinstance(response, dict):
                response_display = json.dumps(response)
            else:
                response_display = json.dumps(response.model_dump())
            return [
                Message(role=Role.USER, content=input_text),
                Message(role=Role.ASSISTANT, content=response_display),
            ]
        else:
            return Message(role=Role.USER, content=input_text)

    messages = [
        Message(
            role=Role.SYSTEM,
            content="Generate an object with the provided instruction, text, and schema.",
        ),
    ]
    if examples is not None:
        for example in examples:
            example_msgs = display_sample(
                instr=example.instruction,
                ty=example.typ,
                t=example.text,
                response=example.response,
            )
            assert isinstance(example_msgs, list) and all(
                isinstance(msg, Message) for msg in example_msgs
            )
            messages.extend(example_msgs)
    messages.append(display_sample(instr=instruction, ty=typ, t=text))
    return messages


def build_score_state(
    instruction: str,
    text: str,
    min: int,
    max: int,
    examples: list[ScoreDatapoint] | None = None,
    suffix_strategy: PromptSuffixStrategy = PromptSuffixStrategy.JSON,
) -> list[Message]:
    def display_sample(
        instr: str, t: str, mn: int, mx: int, response: int | None = None
    ) -> list[Message] | Message:
        if mn > mx:
            raise ValueError(f"Invalid range: [{mn}, {mx}]")
        input_text = force_json_prompt(
            f"Instruction:\n{instr}\n\nText:\n{t}\n\nRange:\n[{mn}, {mx}]",
            suffix_strategy,
        )
        if response is not None:
            return [
                Message(role=Role.USER, content=input_text),
                Message(role=Role.ASSISTANT, content=f'{{"score": {response}}}'),
            ]
        else:
            return Message(role=Role.USER, content=input_text)

    messages = [
        Message(
            role=Role.SYSTEM,
            content='Score the following text with the provided instruction and range as an integer value in valid JSON:\n{"score": number}',
        ),
    ]
    if examples is not None:
        for example in examples:
            example_msgs = display_sample(
                instr=example.instruction,
                t=example.text,
                mn=example.min,
                mx=example.max,
                response=example.response,
            )
            assert isinstance(example_msgs, list) and all(
                isinstance(msg, Message) for msg in example_msgs
            ), example_msgs
            messages.extend(example_msgs)
    messages.append(display_sample(instr=instruction, t=text, mn=min, mx=max))
    return messages


def build_parse_state(
    text: str,
    typ: type[T] | dict[str, Any],
    examples: list[ParseDatapoint] | None = None,
    suffix_strategy: PromptSuffixStrategy = PromptSuffixStrategy.JSON,
) -> list[Message]:
    def display_sample(
        t: str,
        ty: type[T] | dict[str, Any],
        response: T | PartialObj | dict[str, Any] | None = None,
    ) -> Message | list[Message]:
        if isinstance(ty, dict):
            json_schema_string = json.dumps(ty)
        else:
            optionalized_typ = optionalize_type(ty)
            json_schema_string = type_to_json_schema_string(optionalized_typ)
        input_text = force_json_prompt(
            f"Text:\n{t}\n\nSchema:\n{json_schema_string}",
            suffix_strategy=suffix_strategy,
        )
        if response is not None:
            if isinstance(response, dict):
                response_display = json.dumps(response)
            else:
                response_display = response.model_dump_json()
            return [
                Message(role=Role.USER, content=input_text),
                Message(role=Role.ASSISTANT, content=response_display),
            ]
        else:
            return Message(role=Role.USER, content=input_text)

    messages = [
        Message(
            role=Role.SYSTEM,
            content="Parse the following text with the provided JSON schema.",
        ),
    ]
    if examples is not None:
        for example in examples:
            example_msgs = display_sample(t=example.text, ty=typ, response=example.response)
            assert isinstance(example_msgs, list) and all(
                isinstance(msg, Message) for msg in example_msgs
            ), example_msgs
            messages.extend(example_msgs)
    messages.append(display_sample(t=text, ty=typ))
    return messages


def build_classify_state(
    instruction: str,
    text: str,
    options: list[str],
    examples: list[ClassifyDatapoint] | None = None,
    suffix_strategy: PromptSuffixStrategy = PromptSuffixStrategy.JSON,
) -> tuple[list[Message], dict[str, int]]:
    def display_sample(
        instr: str, t: str, opts: list[str], response: int | None = None
    ) -> list[Message] | tuple[Message, dict[str, int]]:
        choices_display, decode_map = display_choices(opts)
        input_text = force_json_prompt(
            f"Instruction:\n{instr}\n\nText:\n{t}\n\nChoices:\n{choices_display}",
            suffix_strategy=suffix_strategy,
        )
        if response is not None:
            response_label = None
            for label, idx in decode_map.items():
                if idx == response:
                    response_label = label
                    break
            assert response_label is not None, f"Invalid response: {response}"
            return [
                Message(role=Role.USER, content=input_text),
                Message(
                    role=Role.ASSISTANT,
                    content=f'{{"classification": "{response_label}"}}',
                ),
            ]
        else:
            return Message(role=Role.USER, content=input_text), decode_map

    messages = [
        Message(
            role=Role.SYSTEM,
            content='Classify the following text with the provided instruction and choices. To classify, provide the key of the choice:\n{"classification": string}\n\nFor example, if the correct choice is \'Z. description of choice Z\', then provide \'Z\' as the classification as valid JSON:\n{"classification": "Z"}',
        ),
    ]
    if examples is not None:
        for example in examples:
            example_msgs = display_sample(
                instr=example.instruction,
                t=example.text,
                opts=example.options,
                response=example.response,
            )
            assert isinstance(example_msgs, list) and all(
                isinstance(msg, Message) for msg in example_msgs
            ), example_msgs
            messages.extend(example_msgs)
    message, decode_map = display_sample(instr=instruction, t=text, opts=options)
    messages.append(message)
    return messages, decode_map


class ChatModel(GeneralModel):
    @abc.abstractmethod
    def generate_message(
        self, messages: list[Message], force_json: bool, temperature: float | None = None
    ) -> Message:
        raise NotImplementedError

    def handle_generate_message_response(
        self, prompt: list[dict[str, str] | Message], content: str, force_json: bool
    ) -> Message:
        if force_json:
            try:
                parsed = parse_json_or_json_markdown(content)
            except (json.JSONDecodeError, ValueError) as e:
                msgs = []
                for msg in prompt:
                    if isinstance(msg, Message):
                        msgs.append(msg.model_dump())
                    else:
                        msgs.append(msg)
                raise ModelError(
                    short_message=f"Failed to parse JSON: {content}",
                    prompt=msgs,
                    response=content,
                ) from e
            cleaned = clean_top_level_keys(parsed)
            return Message(role=Role.ASSISTANT, content=content, obj=cleaned)
        return Message(role=Role.ASSISTANT, content=content, obj=None)

    def build_generate_message_state(self, messages: list[Message]) -> list[dict[str, str]]:
        msgs: list[dict[str, str]] = []
        for msg in messages:
            if msg.obj is not None:
                content = json.dumps(msg.obj)
            else:
                content = msg.content
            msgs.append({"role": msg.role.value, "content": content})
        return msgs

    def _handle_classify_response(self, res: Message, decode_map: dict[str, int]) -> int:
        assert res.obj is not None
        if "classification" not in res.obj:
            raise ModelError(f"Invalid response from model: {res.content}")
        choice = res.obj["classification"]
        if choice not in decode_map:
            key = try_classify_recover(s=choice, decode_map=decode_map)
            if key is not None:
                return decode_map[key]
            raise ModelError(f"Invalid choice: {choice}")
        return decode_map[choice]

    def classify(
        self,
        instruction: str,
        text: str,
        options: list[str],
        examples: list[ClassifyDatapoint] | None = None,
        temperature: float | None = None,
    ) -> int:
        messages, decode_map = build_classify_state(instruction, text, options, examples=examples)
        res = self.generate_message(messages, force_json=True, temperature=temperature)
        return self._handle_classify_response(res, decode_map)

    def parse(
        self,
        text: str,
        typ: type[T] | dict[str, Any],
        examples: list[ParseDatapoint] | None = None,
        temperature: float | None = None,
    ) -> T | PartialObj | dict[str, Any]:
        messages = build_parse_state(text, typ, examples=examples)
        res = self.generate_message(messages, force_json=True, temperature=temperature)
        assert res.obj is not None
        return json_response_to_obj_or_partial_obj(response=res.obj, typ=typ)

    def generate(
        self,
        instruction: str,
        text: str,
        examples: list[GenerateDatapoint] | None = None,
        temperature: float | None = None,
    ) -> str:
        messages = build_generate_state(instruction=instruction, text=text, examples=examples)
        return self.generate_message(messages, force_json=False, temperature=temperature).content

    def _handle_parse_force_response(
        self, res: Message, typ: type[T] | dict[str, Any]
    ) -> T | dict[str, Any]:
        assert res.obj is not None
        obj = json_response_to_obj_or_partial_obj(response=res.obj, typ=typ)
        if not isinstance(typ, dict) and isinstance(obj, dict):
            raise ModelError(f"Invalid response from model: {res.content}")
        return obj

    def parse_force(
        self,
        instruction: str,
        typ: type[T] | dict[str, Any],
        text: str | None = None,
        examples: list[ParseForceDatapoint] | None = None,
        temperature: float | None = None,
    ) -> T | dict[str, Any]:
        messages = build_parse_force_state(
            instruction=instruction,
            typ=typ,
            text=text,
            examples=examples,
        )
        res = self.generate_message(messages, force_json=True, temperature=temperature)
        return self._handle_parse_force_response(res, typ)

    def _handle_score_response(
        self,
        res: Message,
        min: int,
        max: int,
    ) -> int:
        if res.obj is None or "score" not in res.obj:
            raise ModelError(f"Invalid response from model: {res.content}")
        score = res.obj["score"]
        if not isinstance(score, int):
            raise ModelError(f"Invalid score type: {type(score)}")
        if score < min or score > max:
            raise ModelError(f"Invalid score value: {score}")
        return score

    def score(
        self,
        instruction: str,
        text: str,
        min: int,
        max: int,
        examples: list[ScoreDatapoint] | None = None,
        temperature: float | None = None,
    ) -> int:
        messages = build_score_state(instruction, text, min, max, examples=examples)
        res = self.generate_message(messages, force_json=True, temperature=temperature)
        return self._handle_score_response(res, min, max)


def build_prompts(
    dps: list[Datapoint], prompt_suffix_strategy: PromptSuffixStrategy | None
) -> list[str | list[Message]]:
    if len(dps) == 0:
        return []
    typ = type(dps[0])
    for i, dp in enumerate(dps):
        if not isinstance(dp, typ):
            raise ValueError(
                f"All elements must be of type Datapoint, expected type {typ} at index {i}, got {type(dp)}"
            )
    if isinstance(dps[0], ParseDatapoint):
        build_func = build_parse_prompts
    elif isinstance(dps[0], BinaryClassifyDatapoint):
        build_func = build_binary_classify_prompts
    elif isinstance(dps[0], ClassifyDatapoint):
        build_func = build_classify_prompts
    elif isinstance(dps[0], ParseForceDatapoint):
        build_func = build_parse_force_prompts
    elif isinstance(dps[0], GenerateDatapoint):
        build_func = build_generate_prompts
    elif isinstance(dps[0], ScoreDatapoint):
        build_func = build_score_prompts
    else:
        raise ValueError(f"Unknown datapoint type: {type(dps[0])}")
    return build_func(dps, suffix_strategy=prompt_suffix_strategy)


def build_parse_prompts(
    dps: list[ParseDatapoint],
    suffix_strategy: PromptSuffixStrategy | None = None,
) -> list[str | list[Message]]:
    datapoints = []
    for dp in dps:
        json_response_object = (
            dp.response.model_dump_json()
            if isinstance(dp.response, BaseModel)
            else json.dumps(dp.response)
        )
        prompt_msgs = build_parse_state(
            text=dp.text,
            typ=dp.typ,
            suffix_strategy=(
                suffix_strategy if suffix_strategy is not None else PromptSuffixStrategy.JSON
            ),
        )
        json_response = apply_suffix_strategy(
            response=json_response_object, suffix_strategy=suffix_strategy
        )
        datapoints.append(prompt_msgs + [Message(role=Role.ASSISTANT, content=json_response)])
    return datapoints


def build_binary_classify_prompts(
    dps: list[BinaryClassifyDatapoint],
    suffix_strategy: PromptSuffixStrategy | None = None,
) -> list[str | list[Message]]:
    return build_classify_prompts(
        [
            ClassifyDatapoint(
                instruction=dp.instruction,
                text=dp.text,
                options=["true", "false"],
                response=0 if dp.response else 1,
            )
            for dp in dps
        ],
        suffix_strategy=suffix_strategy,
    )


def build_classify_prompts(
    dps: list[ClassifyDatapoint],
    suffix_strategy: PromptSuffixStrategy | None = None,
) -> list[str | list[Message]]:
    def label_idx_to_label_json(idx: int, decode_map: dict[str, int]) -> str:
        label = None
        for k, v in decode_map.items():
            if v == idx:
                label = k
                break
        if label is None:
            raise ValueError(f"Label index {idx} not found in decode map")
        return f'{{"classification": "{label}"}}'

    datapoints = []
    for dp in dps:
        suffix_strategy = PromptSuffixStrategy.JSON if suffix_strategy is None else suffix_strategy
        prompt_msgs, decode_map = build_classify_state(
            instruction=dp.instruction,
            text=dp.text,
            options=dp.options,
            suffix_strategy=suffix_strategy,
        )
        json_response_object = label_idx_to_label_json(idx=dp.response, decode_map=decode_map)
        json_response = apply_suffix_strategy(
            response=json_response_object, suffix_strategy=suffix_strategy
        )
        datapoints.append(
            prompt_msgs
            + [
                Message(
                    role=Role.ASSISTANT,
                    content=json_response,
                )
            ]
        )
    return datapoints


def build_parse_force_prompts(
    dps: list[ParseForceDatapoint],
    suffix_strategy: PromptSuffixStrategy | None = None,
) -> list[str | list[Message]]:
    datapoints = []
    for dp in dps:
        json_response_obj = (
            dp.response.model_dump_json()
            if isinstance(dp.response, BaseModel)
            else json.dumps(dp.response)
        )
        suffix_strategy = PromptSuffixStrategy.JSON if suffix_strategy is None else suffix_strategy
        prompt_msgs = build_parse_force_state(
            instruction=dp.instruction,
            text=dp.text,
            typ=dp.typ,
            suffix_strategy=suffix_strategy,
        )
        json_response = apply_suffix_strategy(
            response=json_response_obj, suffix_strategy=suffix_strategy
        )
        datapoints.append(prompt_msgs + [Message(role=Role.ASSISTANT, content=json_response)])
    return datapoints


def build_generate_prompts(dps: list[GenerateDatapoint]) -> list[str | list[Message]]:
    datapoints = []
    for dp in dps:
        prompt_msgs = build_generate_state(instruction=dp.instruction, text=dp.text)
        datapoints.append(prompt_msgs + [Message(role=Role.ASSISTANT, content=dp.response)])
    return datapoints


def build_score_prompts(
    dps: list[ScoreDatapoint],
    suffix_strategy: PromptSuffixStrategy | None = None,
) -> list[str | list[Message]]:
    datapoints = []
    for dp in dps:
        json_response_object = f'{{"score": {dp.response}}}'
        suffix_strategy = (
            suffix_strategy if suffix_strategy is not None else PromptSuffixStrategy.JSON
        )
        prompt_msgs = build_score_state(
            instruction=dp.instruction,
            text=dp.text,
            min=dp.min,
            max=dp.max,
            suffix_strategy=suffix_strategy,
        )
        json_response = apply_suffix_strategy(
            response=json_response_object, suffix_strategy=suffix_strategy
        )
        datapoints.append(prompt_msgs + [Message(role=Role.ASSISTANT, content=json_response)])
    return datapoints


def apply_suffix_strategy(response: str, suffix_strategy: PromptSuffixStrategy) -> str:
    if suffix_strategy == PromptSuffixStrategy.JSON:
        return response
    elif suffix_strategy == PromptSuffixStrategy.JSON_MD_BLOCK:
        return add_md_tag(response)
    else:
        raise ValueError(f"Unknown suffix strategy: {suffix_strategy}")