From eedf6e8501337d80f5f4ca9c95a50e4341d574e1 Mon Sep 17 00:00:00 2001 From: SumanthRH Date: Mon, 31 Mar 2025 18:53:07 +0000 Subject: [PATCH 1/2] fix issue with reading from system prompt in file Signed-off-by: SumanthRH --- scripts/response_rewrite.py | 2 +- skythought/evals/inference_and_check.py | 2 +- skythought/evals/models/base.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/response_rewrite.py b/scripts/response_rewrite.py index 31945cb1..6c19f568 100644 --- a/scripts/response_rewrite.py +++ b/scripts/response_rewrite.py @@ -507,7 +507,7 @@ def main(): variants_dataset, ["fcs", "fcs_plus1", "fcs_reflection"], llm ) - system_prompt = ModelConfig.from_model_id(args.target_model).system_prompt + system_prompt = str(ModelConfig.from_model_id(args.target_model).system_prompt) # Generate conversation format for each variant, which can be used in SimPO/DPO/etc. fcs_convo = make_preference_conversations(final_dataset, "fcs", system_prompt) diff --git a/skythought/evals/inference_and_check.py b/skythought/evals/inference_and_check.py index 9f8d47b2..7a534565 100644 --- a/skythought/evals/inference_and_check.py +++ b/skythought/evals/inference_and_check.py @@ -237,7 +237,7 @@ def generate_responses_for_dataset( # Prepare conversations conversations = handler.make_conversations( remaining_data, - model_config.system_prompt, + str(model_config.system_prompt), model_config.user_template, model_config.assistant_prefill, ) diff --git a/skythought/evals/models/base.py b/skythought/evals/models/base.py index 3f85cf58..39f28008 100644 --- a/skythought/evals/models/base.py +++ b/skythought/evals/models/base.py @@ -27,6 +27,9 @@ def validate_and_extract_string(self): @property def string(self): return self._string + + def __str__(self) -> str: + return self._string def read_yaml(path: str): @@ -49,7 +52,7 @@ def validate_name(self): if self.name is None: self.name = self.model_id.split("/")[-1] return self - + @classmethod def from_model_id( cls, From 6a9558c47443e68e5a389c5c89920f588f5b6a89 Mon Sep 17 00:00:00 2001 From: SumanthRH Date: Mon, 31 Mar 2025 18:54:24 +0000 Subject: [PATCH 2/2] lint Signed-off-by: SumanthRH --- skythought/evals/models/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/skythought/evals/models/base.py b/skythought/evals/models/base.py index 39f28008..425b215f 100644 --- a/skythought/evals/models/base.py +++ b/skythought/evals/models/base.py @@ -27,7 +27,7 @@ def validate_and_extract_string(self): @property def string(self): return self._string - + def __str__(self) -> str: return self._string @@ -52,7 +52,7 @@ def validate_name(self): if self.name is None: self.name = self.model_id.split("/")[-1] return self - + @classmethod def from_model_id( cls,