MMLU-Amateur

This dataset is derived from [TIGER-Lab/MMLU-Pro](https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro) as part of our [MMLU-Leagues]() Encoder benchmark series, containing:

- MMLU-Amateur (this dataset), where the train set contains all questions Llama-3-8B-Instruct (5-shot) gets wrong and the test set contains all questions it gets right. The aim is to measure the ability of an encoder, with relatively limited training data, to match the performance of a small frontier model.
- [MMLU-SemiPro](https://huggingface.co/datasets/answerdotai/MMLU-SemiPro), where the data is evenly split between a train and a test set. Both splits contain exactly the same proportions of questions that Llama-3-8B-Instruct (5-shot) answers correctly, to ensure an even difficulty distribution.

The data is stratified into categories, to ensure that there's the exact same number (+/-) of questions from each category in both splits.

This dataset was processed with the following script:

```python
from datasets import Dataset, load_dataset
import srsly
from sklearn.model_selection import GroupKFold

# Load original MMLU
data_df = load_dataset("TIGER-Lab/MMLU-Pro", split="test").to_pandas()

# Load llama cached predictions
# You can get the llama outputs from https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/main/eval_results/model_outputs_Meta-Llama-3-8B-Instruct_5shots.json
llama_outputs = srsly.read_json('llm_outputs/model_outputs_Meta-Llama-3-8B-Instruct_5shots.json')

# Enrich the df with the llama predictions
llama_pred_dict = {item['question_id']: item['pred'] for item in llama_outputs}
data_df['llama_pred'] = data_df['question_id'].map(llama_pred_dict)
data_df['llama_pred'] = data_df['llama_pred'].fillna("NoAnswer")
data_df['llama_correct'] = data_df.apply(lambda row: row['llama_pred'] == row['answer'], axis=1)
data_df = data_df.reset_index(drop=True)

# Filter down to only questions with exactly 10 answers
data_df = data_df[data_df["options"].apply(len) == 10].copy()
data_df = data_df.reset_index(drop=True)

# train-test split
from sklearn.model_selection import GroupShuffleSplit

def add_fold(df, group_col="category", fold_method="semipro"):
if fold_method not in ["amateur", "semipro"]:
raise ValueError("fold_method must be either 'amateur' or 'semipro'")

if fold_method == "amateur":
df["kfold"] = df["llama_correct"].astype(int)
return df

df["kfold"] = -1 # Initialize all folds to -1

for category in df[group_col].unique():
category_df = df[df[group_col] == category]

# Split correct predictions
correct_df = category_df[category_df["llama_correct"] == True]
correct_count = len(correct_df)
correct_per_fold = correct_count // 2
extra_correct = correct_count % 2

correct_df = correct_df.sample(frac=1, random_state=42) # Shuffle
df.loc[correct_df.index[:correct_per_fold], "kfold"] = 0
df.loc[correct_df.index[correct_per_fold:correct_per_fold+extra_correct], "kfold"] = 1
df.loc[correct_df.index[correct_per_fold+extra_correct:], "kfold"] = 1

# Split incorrect predictions
incorrect_df = category_df[category_df["llama_correct"] == False]
incorrect_count = len(incorrect_df)
incorrect_per_fold = incorrect_count // 2
extra_incorrect = incorrect_count % 2

incorrect_df = incorrect_df.sample(frac=1, random_state=42) # Shuffle
df.loc[incorrect_df.index[:incorrect_per_fold+extra_incorrect], "kfold"] = 0
df.loc[incorrect_df.index[incorrect_per_fold+extra_incorrect:], "kfold"] = 1

assert (df["kfold"] != -1).all(), "Not all rows were assigned a fold"

return df

semipro_processed_df = add_fold(data_df, fold_method="semipro")

semipro_test_df = semipro_processed_df[semipro_processed_df["kfold"] == 1].drop(columns="kfold")
semipro_train_df = semipro_processed_df[semipro_processed_df["kfold"] == 0].drop(columns="kfold")

semipro_train_ds = Dataset.from_pandas(semipro_train_df, preserve_index=False)
semipro_test_ds = Dataset.from_pandas(semipro_test_df, preserve_index=False)

semipro_processed_df = add_fold(data_df, fold_method="semipro")

semipro_test_df = semipro_processed_df[semipro_processed_df["kfold"] == 1].drop(columns="kfold")
semipro_train_df = semipro_processed_df[semipro_processed_df["kfold"] == 0].drop(columns="kfold")

semipro_train_ds = Dataset.from_pandas(semipro_train_df, preserve_index=False)
semipro_test_ds = Dataset.from_pandas(semipro_test_df, preserve_index=False)

# Sanity check: Assert that the number of llama_correct==True is within 1 for each category in train and test
# Also check that the total number of examples is within 1 for each category in train and test
for category in semipro_train_df['category'].unique():
train_correct = semipro_train_df[(semipro_train_df['category'] == category) & (semipro_train_df['llama_correct'] == True)].shape[0]
test_correct = semipro_test_df[(semipro_test_df['category'] == category) & (semipro_test_df['llama_correct'] == True)].shape[0]

train_total = semipro_train_df[semipro_train_df['category'] == category].shape[0]
test_total = semipro_test_df[semipro_test_df['category'] == category].shape[0]

assert abs(train_correct - test_correct) <= 1, f"Category {category}: train_correct ({train_correct}) and test_correct ({test_correct}) differ by more than 1"
assert abs(train_total - test_total) <= 1, f"Category {category}: train_total ({train_total}) and test_total ({test_total}) differ by more than 1"

print("Sanity check passed: The number of llama_correct==True and total examples are within 1 for each category in train and test sets.")
semipro_processed_df = add_fold(data_df, fold_method="semipro")

semipro_test_df = semipro_processed_df[semipro_processed_df["kfold"] == 1].drop(columns="kfold")
semipro_train_df = semipro_processed_df[semipro_processed_df["kfold"] == 0].drop(columns="kfold")

semipro_train_ds = Dataset.from_pandas(semipro_train_df, preserve_index=False)
semipro_test_ds = Dataset.from_pandas(semipro_test_df, preserve_index=False)

# Sanity check: Assert that the number of llama_correct==True is within 1 for each category in train and test
# Also check that the total number of examples is within 1 for each category in train and test
for category in semipro_train_df['category'].unique():
train_correct = semipro_train_df[(semipro_train_df['category'] == category) & (semipro_train_df['llama_correct'] == True)].shape[0]
test_correct = semipro_test_df[(semipro_test_df['category'] == category) & (semipro_test_df['llama_correct'] == True)].shape[0]

train_total = semipro_train_df[semipro_train_df['category'] == category].shape[0]
test_total = semipro_test_df[semipro_test_df['category'] == category].shape[0]

assert abs(train_correct - test_correct) <= 1, f"Category {category}: train_correct ({train_correct}) and test_correct ({test_correct}) differ by more than 1"
assert abs(train_total - test_total) <= 1, f"Category {category}: train_total ({train_total}) and test_total ({test_total}) differ by more than 1"

print("Sanity check passed: The number of llama_correct==True and total examples are within 1 for each category in train and test sets.")

semipro_train_ds.push_to_hub(
repo_id="answerdotai/MMLU-SemiPro",
split="train",
)

semipro_test_ds.push_to_hub(
repo_id="answerdotai/MMLU-SemiPro",
split="test",
)
```

本数据集源自[TIGER-Lab/MMLU-Pro](https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro)，隶属于我们的[MMLU-Leagues]()编码器基准测试系列，具体涵盖以下两个子集：
- **MMLU-Amateur（本数据集）**：其训练集涵盖Llama-3-8B-Instruct（5样本）模型答错的全部题目，测试集则包含该模型答对的全部题目。本数据集的设计目标为评估训练数据相对有限的编码器，能否媲美小型前沿大语言模型的性能。
- **[MMLU-SemiPro](https://huggingface.co/datasets/answerdotai/MMLU-SemiPro)**：其训练集与测试集的数据划分均匀，两个拆分集中均包含完全相同比例的、被Llama-3-8B-Instruct（5样本）模型答对的题目，以此确保难度分布均衡。

所有数据已按类别进行分层处理，以保证两个拆分集中每个类别的题目数量完全一致（误差范围为±1）。

本数据集的处理脚本如下：
python
from datasets import Dataset, load_dataset
import srsly
from sklearn.model_selection import GroupKFold

# 加载原始MMLU数据集
data_df = load_dataset("TIGER-Lab/MMLU-Pro", split="test").to_pandas()

# 加载Llama缓存的预测结果
# 你可以从https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/main/eval_results/model_outputs_Meta-Llama-3-8B-Instruct_5shots.json获取Llama的输出结果
llama_outputs = srsly.read_json('llm_outputs/model_outputs_Meta-Llama-3-8B-Instruct_5shots.json')

# 为数据框添加Llama的预测结果
llama_pred_dict = {item['question_id']: item['pred'] for item in llama_outputs}
data_df['llama_pred'] = data_df['question_id'].map(llama_pred_dict)
data_df['llama_pred'] = data_df['llama_pred'].fillna("NoAnswer")
data_df['llama_correct'] = data_df.apply(lambda row: row['llama_pred'] == row['answer'], axis=1)
data_df = data_df.reset_index(drop=True)

# 仅保留选项数量恰好为10的题目
data_df = data_df[data_df["options"].apply(len) == 10].copy()
data_df = data_df.reset_index(drop=True)

# 训练集-测试集划分
from sklearn.model_selection import GroupShuffleSplit

def add_fold(df, group_col="category", fold_method="semipro"):
if fold_method not in ["amateur", "semipro"]:
raise ValueError("fold_method必须为'amateur'或'semipro'")

if fold_method == "amateur":
df["kfold"] = df["llama_correct"].astype(int)
return df

df["kfold"] = -1 # 将所有折初始化为-1

for category in df[group_col].unique():
category_df = df[df[group_col] == category]

# 划分预测正确的样本
correct_df = category_df[category_df["llama_correct"] == True]
correct_count = len(correct_df)
correct_per_fold = correct_count // 2
extra_correct = correct_count % 2

correct_df = correct_df.sample(frac=1, random_state=42) # 打乱顺序
df.loc[correct_df.index[:correct_per_fold], "kfold"] = 0
df.loc[correct_df.index[correct_per_fold:correct_per_fold+extra_correct], "kfold"] = 1
df.loc[correct_df.index[correct_per_fold+extra_correct:], "kfold"] = 1

# 划分预测错误的样本
incorrect_df = category_df[category_df["llama_correct"] == False]
incorrect_count = len(incorrect_df)
incorrect_per_fold = incorrect_count // 2
extra_incorrect = incorrect_count % 2

incorrect_df = incorrect_df.sample(frac=1, random_state=42) # 打乱顺序
df.loc[incorrect_df.index[:incorrect_per_fold+extra_incorrect], "kfold"] = 0
df.loc[incorrect_df.index[incorrect_per_fold+extra_incorrect:], "kfold"] = 1

assert (df["kfold"] != -1).all(), "并非所有样本都被分配了折分组"

return df

semipro_processed_df = add_fold(data_df, fold_method="semipro")

semipro_test_df = semipro_processed_df[semipro_processed_df["kfold"] == 1].drop(columns="kfold")
semipro_train_df = semipro_processed_df[semipro_processed_df["kfold"] == 0].drop(columns="kfold")

semipro_train_ds = Dataset.from_pandas(semipro_train_df, preserve_index=False)
semipro_test_ds = Dataset.from_pandas(semipro_test_df, preserve_index=False)

# 合理性检查：断言训练集和测试集中每个类别的llama_correct==True的样本数量差异不超过1
# 同时检查训练集和测试集中每个类别的总样本数量差异不超过1
for category in semipro_train_df['category'].unique():
train_correct = semipro_train_df[(semipro_train_df['category'] == category) & (semipro_train_df['llama_correct'] == True)].shape[0]
test_correct = semipro_test_df[(semipro_test_df['category'] == category) & (semipro_test_df['llama_correct'] == True)].shape[0]

train_total = semipro_train_df[semipro_train_df['category'] == category].shape[0]
test_total = semipro_test_df[semipro_test_df['category'] == category].shape[0]

assert abs(train_correct - test_correct) <= 1, f"类别 {category}：训练集正确样本数 ({train_correct}) 与测试集正确样本数 ({test_correct}) 差异超过1"
assert abs(train_total - test_total) <= 1, f"类别 {category}：训练集总样本数 ({train_total}) 与测试集总样本数 ({test_total}) 差异超过1"

print("合理性检查通过：训练集和测试集中每个类别的llama_correct==True的样本数量及总样本数量差异均不超过1。")
semipro_processed_df = add_fold(data_df, fold_method="semipro")

semipro_test_df = semipro_processed_df[semipro_processed_df["kfold"] == 1].drop(columns="kfold")
semipro_train_df = semipro_processed_df[semipro_processed_df["kfold"] == 0].drop(columns="kfold")

semipro_train_ds = Dataset.from_pandas(semipro_train_df, preserve_index=False)
semipro_test_ds = Dataset.from_pandas(semipro_test_df, preserve_index=False)

# 合理性检查：断言训练集和测试集中每个类别的llama_correct==True的样本数量差异不超过1
# 同时检查训练集和测试集中每个类别的总样本数量差异不超过1
for category in semipro_train_df['category'].unique():
train_correct = semipro_train_df[(semipro_train_df['category'] == category) & (semipro_train_df['llama_correct'] == True)].shape[0]
test_correct = semipro_test_df[(semipro_test_df['category'] == category) & (semipro_test_df['llama_correct'] == True)].shape[0]

train_total = semipro_train_df[semipro_train_df['category'] == category].shape[0]
test_total = semipro_test_df[semipro_test_df['category'] == category].shape[0]

assert abs(train_correct - test_correct) <= 1, f"类别 {category}：训练集正确样本数 ({train_correct}) 与测试集正确样本数 ({test_correct}) 差异超过1"
assert abs(train_total - test_total) <= 1, f"类别 {category}：训练集总样本数 ({train_total}) 与测试集总样本数 ({test_total}) 差异超过1"

print("合理性检查通过：训练集和测试集中每个类别的llama_correct==True的样本数量及总样本数量差异均不超过1。")

semipro_train_ds.push_to_hub(
repo_id="answerdotai/MMLU-SemiPro",
split="train",
)

semipro_test_ds.push_to_hub(
repo_id="answerdotai/MMLU-SemiPro",
split="test",
)