THIQAH-RD/ArabLegalEval

---
dataset_info:
- config_name: ArLegalBench
features:
- name: index
dtype: string
- name: question_english
dtype: string
- name: answer_english
dtype: string
- name: contract_english
dtype: string
- name: Question
dtype: string
- name: Answer
dtype: string
- name: Context
dtype: string
- name: choices
sequence: string
- name: choices_english
sequence: string
- name: subject
dtype: string
- name: task
dtype: string
splits:
- name: test
num_bytes: 19243593
num_examples: 15311
- name: train
num_bytes: 51939
num_examples: 28
download_size: 3389747
dataset_size: 19295532
- config_name: MCQs
features:
- name: Engine
dtype: string
- name: Context
dtype: string
- name: Question
dtype: string
- name: Answer
dtype: string
- name: Option 1
dtype: string
- name: Option 2
dtype: string
- name: Option 3
dtype: string
- name: Option 4
dtype: string
- name: Answer Key
dtype: int64
- name: task
dtype: string
- name: subject
dtype: string
- name: choices
sequence: string
splits:
- name: train
num_bytes: 12185640.889357671
num_examples: 9291
- name: test
num_bytes: 3046738.1106423284
num_examples: 2323
download_size: 7038064
dataset_size: 15232379.0
- config_name: QA
features:
- name: Question
dtype: string
- name: Answer
dtype: string
- name: Context
dtype: string
- name: task
dtype: string
- name: subject
dtype: string
splits:
- name: train
num_bytes: 81006.03797468354
num_examples: 63
- name: test
num_bytes: 20572.962025316454
num_examples: 16
download_size: 56179
dataset_size: 101579.0
configs:
- config_name: ArLegalBench
data_files:
- split: train
path: ArLegalBench/train-*
- split: test
path: ArLegalBench/test-*
- config_name: MCQs
data_files:
- split: train
path: MCQs/train-*
- split: test
path: MCQs/test-*
- config_name: QA
data_files:
- split: train
path: QA/train-*
- split: test
path: QA/test-*
---

# ArabLegalEval

## Populating prompt templates

```py
import random
import requests
import yaml
from datasets import load_dataset

def render_prompt_template(dataset_entry, technique_map):
subject = dataset_entry['subject']
task = dataset_entry['task']
technique = technique_map[task]
prompt_template = prompts[task][subject][technique]
prompt_template = (
prompt_template
.replace('{correct_answer}', '{Answer}')
.replace('{question}', '{Question}')
.replace('{query_str}', '{Question}')
.replace('{clause}', '{Context}')
.replace('{context}', '{Context}')
.replace('{context_str}', '{Context}')
.replace('{options}', '{choices}')
)
assert '{Question}' in prompt_template, f"Question not found in prompt template for {subject}/{task}"
rendered_instruction = prompt_template.replace('{Question}', dataset_entry['Question'])
rendered_instruction = rendered_instruction.replace('{Context}', dataset_entry['Context'])
dataset_entry['rendered_instruction'] = rendered_instruction
return dataset_entry

def populate_choices(dataset_entry):
assert 'rendered_instruction' in dataset_entry, f"rendered_instruction not found for {subject}/{task}"
subject = dataset_entry['subject']
task = dataset_entry['task']
if '{choices}' not in dataset_entry['rendered_instruction']:
return dataset_entry

if task == "MCQs":
alpa_en = ['A', 'B', 'C', 'D', 'E']
# Get the actual option values (not column names) that aren't None/empty
choices_columns = [
c for c in
[
'Option 1', 'Option 2', 'Option 3', 'Option 4'
]
if c in dataset_entry.keys() and dataset_entry[c] is not None and dataset_entry[c] != ''
]
choices = [
dataset_entry[c] for c in choices_columns
]
correct_answer_index_before_shuffling = int(dataset_entry['Answer Key']) - 1
# assert choices[correct_answer_index_before_shuffling] == dataset_entry['Answer'], f"Answer '{dataset_entry['Answer']}' not found at index {correct_answer_index_before_shuffling} in choices {choices}"
# print("Choices before shuffling:", choices, "Answer Key:", dataset_entry['Answer Key'], "Answer:", dataset_entry['Answer'])
# Randomly shuffle the choices
shuffled_indices = list(range(len(choices)))
random.shuffle(shuffled_indices)
correct_answer_index_after_shuffling = shuffled_indices.index(correct_answer_index_before_shuffling)
shuffled_choices = [choices[i] for i in shuffled_indices]
choices_str = '\n'.join([f"{alpa_en[i]}. {choice}" for i, choice in enumerate(shuffled_choices)])
dataset_entry['Answer Key'] = alpa_en[correct_answer_index_after_shuffling]
dataset_entry['rendered_instruction'] = dataset_entry['rendered_instruction'].format(choices=choices_str)
dataset_entry['Answer'] = shuffled_choices[correct_answer_index_after_shuffling]
assert shuffled_choices[correct_answer_index_after_shuffling] == dataset_entry['Answer'], f"Answer '{dataset_entry['Answer']}' not found at index {correct_answer_index_after_shuffling} in shuffled choices {shuffled_choices}"
assert dataset_entry['Answer Key'] == alpa_en[correct_answer_index_after_shuffling], f"Answer Key '{dataset_entry['Answer Key']}' not found in shuffled choices {shuffled_choices}"
index_from_alpha = alpa_en.index(dataset_entry['Answer Key'])
assert shuffled_choices[index_from_alpha] == dataset_entry['Answer'], f"Answer '{dataset_entry['Answer']}' not found in shuffled choices {shuffled_choices}"
elif task == "ArLegalBench":
raise Exception("ArLegalBench is not supported")
else:
raise ValueError(f"Unsupported task: {task}")

return dataset_entry

# download and load prompts.yaml
prompts = yaml.safe_load(requests.get("https://huggingface.co/datasets/THIQAH-Research/ArabLegalEval/resolve/main/prompts.yaml?download=true").content)

## feel free to change the technique map to your liking
fn_kwargs = {
"technique_map": {
"QA": "TEXT_QA_TEMPLATE",
"MCQs": "MCQ_PROMPT_WITH_CONTEXT",
"ArLegalBench": "Zero_shot"
}
}

arabLegalEvalDatasetKeys = ["MCQs", "QA", "ArLegalBench"]
arabLegalEval = {
key: load_dataset("THIQAH-Research/ArabLegalEval", key) for key in arabLegalEvalDatasetKeys
}

for name, ds in arabLegalEval.items():
for split in ds.keys():
ds[split] = ds[split].map(render_prompt_template, fn_kwargs=fn_kwargs)
ds[split] = ds[split].map(populate_choices)
x = ds["test"][0]
print(f'rendered instruction for {name}:\n' + x['rendered_instruction'], "Answer:", x['Answer'])
```

```
rendered instruction for MCQs:
هذا سؤال قانوني للعامة في المملكة العربية السعودية. اختر الإجابة الصحيحة!

سياق: يكون تبليغ غير السعودي المتحقق وجوده داخل المملكة وليس له مكان إقامة معروف وفق الفقرة (ط) من المادة السابعة عشرة من هذا النظام.
سؤال: ما هو شرط تبليغ غير السعودي وفق الفقرة (ط) من المادة السابعة عشرة من هذا النظام؟
A. أن يكون متحقق وجوده خارج المملكة وليس له مكان إقامة معروف
B. أن يكون متحقق وجوده داخل المملكة وله مكان إقامة معروف
C. أن يكون متحقق وجوده خارج المملكة وله مكان إقامة معروف
D. أن يكون متحقق وجوده داخل المملكة وليس له مكان إقامة معروف

إجابة:
Answer: أن يكون متحقق وجوده داخل المملكة وليس له مكان إقامة معروف
```

### 数据集信息
- 配置名称：ArLegalBench
特征：
- 索引（index）：字符串类型
- 英文问题（question_english）：字符串类型
- 英文答案（answer_english）：字符串类型
- 英文合同（contract_english）：字符串类型
- 问题（Question）：字符串类型
- 答案（Answer）：字符串类型
- 上下文（Context）：字符串类型
- 选项（choices）：字符串序列
- 英文选项（choices_english）：字符串序列
- 主题（subject）：字符串类型
- 任务（task）：字符串类型
数据划分：
- 测试集（test）：字节数19243593，样本量15311
- 训练集（train）：字节数51939，样本量28
下载大小：3389747，数据集总大小：19295532
- 配置名称：多项选择题（Multiple Choice Questions, MCQs）
特征：
- 引擎（Engine）：字符串类型
- 上下文（Context）：字符串类型
- 问题（Question）：字符串类型
- 答案（Answer）：字符串类型
- 选项1（Option 1）：字符串类型
- 选项2（Option 2）：字符串类型
- 选项3（Option 3）：字符串类型
- 选项4（Option 4）：字符串类型
- 答案索引（Answer Key）：整数类型
- 任务（task）：字符串类型
- 主题（subject）：字符串类型
- 选项（choices）：字符串序列
数据划分：
- 训练集（train）：字节数12185640.889357671，样本量9291
- 测试集（test）：字节数3046738.1106423284，样本量2323
下载大小：7038064，数据集总大小：15232379.0
- 配置名称：问答（Question Answering, QA）
特征：
- 问题（Question）：字符串类型
- 答案（Answer）：字符串类型
- 上下文（Context）：字符串类型
- 任务（task）：字符串类型
- 主题（subject）：字符串类型
数据划分：
- 训练集（train）：字节数81006.03797468354，样本量63
- 测试集（test）：字节数20572.962025316454，样本量16
下载大小：56179，数据集总大小：101579.0

### 配置详情
- 配置名称：ArLegalBench
数据文件：
- 训练集划分：路径为ArLegalBench/train-*
- 测试集划分：路径为ArLegalBench/test-*
- 配置名称：多项选择题（Multiple Choice Questions, MCQs）
数据文件：
- 训练集划分：路径为MCQs/train-*
- 测试集划分：路径为MCQs/test-*
- 配置名称：问答（Question Answering, QA）
数据文件：
- 训练集划分：路径为QA/train-*
- 测试集划分：路径为QA/test-*

---

# 阿拉伯法律评估（ArabLegalEval）

## 填充提示模板
py
import random
import requests
import yaml
from datasets import load_dataset

def render_prompt_template(dataset_entry, technique_map):
subject = dataset_entry['subject']
task = dataset_entry['task']
technique = technique_map[task]
prompt_template = prompts[task][subject][technique]
prompt_template = (
prompt_template
.replace('{correct_answer}', '{Answer}')
.replace('{question}', '{Question}')
.replace('{query_str}', '{Question}')
.replace('{clause}', '{Context}')
.replace('{context}', '{Context}')
.replace('{context_str}', '{Context}')
.replace('{options}', '{choices}')
)
assert '{Question}' in prompt_template, f"未找到对应{subject}/{task}的提示模板中的Question占位符"
rendered_instruction = prompt_template.replace('{Question}', dataset_entry['Question'])
rendered_instruction = rendered_instruction.replace('{Context}', dataset_entry['Context'])
dataset_entry['rendered_instruction'] = rendered_instruction
return dataset_entry

def populate_choices(dataset_entry):
assert 'rendered_instruction' in dataset_entry, f"未找到对应{subject}/{task}的rendered_instruction字段"
subject = dataset_entry['subject']
task = dataset_entry['task']
if '{choices}' not in dataset_entry['rendered_instruction']:
return dataset_entry

if task == "MCQs":
alpa_en = ['A', 'B', 'C', 'D', 'E']
# 获取非空的实际选项值（而非列名）
choices_columns = [
c for c in
[
'Option 1', 'Option 2', 'Option 3', 'Option 4'
]
if c in dataset_entry.keys() and dataset_entry[c] is not None and dataset_entry[c] != ''
]
choices = [
dataset_entry[c] for c in choices_columns
]
correct_answer_index_before_shuffling = int(dataset_entry['Answer Key']) - 1
# assert choices[correct_answer_index_before_shuffling] == dataset_entry['Answer'], f"答案'{dataset_entry['Answer']}'未在索引{correct_answer_index_before_shuffling}的选项{choices}中找到"
# print("打乱前的选项:", choices, "答案索引:", dataset_entry['Answer Key'], "答案:", dataset_entry['Answer'])
# 随机打乱选项顺序
shuffled_indices = list(range(len(choices)))
random.shuffle(shuffled_indices)
correct_answer_index_after_shuffling = shuffled_indices.index(correct_answer_index_before_shuffling)
shuffled_choices = [choices[i] for i in shuffled_indices]
choices_str = '
'.join([f"{alpa_en[i]}. {choice}" for i, choice in enumerate(shuffled_choices)])
dataset_entry['Answer Key'] = alpa_en[correct_answer_index_after_shuffling]
dataset_entry['rendered_instruction'] = dataset_entry['rendered_instruction'].format(choices=choices_str)
dataset_entry['Answer'] = shuffled_choices[correct_answer_index_after_shuffling]
assert shuffled_choices[correct_answer_index_after_shuffling] == dataset_entry['Answer'], f"答案'{dataset_entry['Answer']}'未在打乱后的选项{shuffled_choices}的索引{correct_answer_index_after_shuffling}位置找到"
assert dataset_entry['Answer Key'] == alpa_en[correct_answer_index_after_shuffling], f"答案键'{dataset_entry['Answer Key']}'未在打乱后的选项{shuffled_choices}中找到"
index_from_alpha = alpa_en.index(dataset_entry['Answer Key'])
assert shuffled_choices[index_from_alpha] == dataset_entry['Answer'], f"答案'{dataset_entry['Answer']}'未在打乱后的选项{shuffled_choices}中找到"
elif task == "ArLegalBench":
raise Exception("不支持ArLegalBench任务类型")
else:
raise ValueError(f"不支持的任务类型: {task}")

return dataset_entry

# 下载并加载prompts.yaml配置文件
prompts = yaml.safe_load(requests.get("https://huggingface.co/datasets/THIQAH-Research/ArabLegalEval/resolve/main/prompts.yaml?download=true").content)

## 可根据需求自定义技术映射表
fn_kwargs = {
"technique_map": {
"QA": "TEXT_QA_TEMPLATE",
"MCQs": "MCQ_PROMPT_WITH_CONTEXT",
"ArLegalBench": "零样本(Zero-shot)"
}
}

arabLegalEvalDatasetKeys = ["MCQs", "QA", "ArLegalBench"]
arabLegalEval = {
key: load_dataset("THIQAH-Research/ArabLegalEval", key) for key in arabLegalEvalDatasetKeys
}

for name, ds in arabLegalEval.items():
for split in ds.keys():
ds[split] = ds[split].map(render_prompt_template, fn_kwargs=fn_kwargs)
ds[split] = ds[split].map(populate_choices)
x = ds["test"][0]
print(f'{name}的渲染后提示指令:
' + x['rendered_instruction'], "答案:", x['Answer'])

多项选择题渲染后示例：
这是沙特阿拉伯王国的公共法律问题，请选择正确答案！

上下文：非沙特国民在王国境内被核实存在，且无已知居留地点，符合本法令第十七条第（ط）款的规定。
问题：根据本法令第十七条第（ط）款，非沙特国民的报备条件是什么？
A. 被核实存在于王国境外且无已知居留地点
B. 被核实存在于王国内且有已知居留地点
C. 被核实存在于王国境外且有已知居留地点
D. 被核实存在于王国内且无已知居留地点

答案：
正确答案为：被核实存在于王国内且无已知居留地点