miulab/tmlu

---
task_categories:
- question-answering
- text-classification
language:
- zh
pretty_name: TMLU
size_categories:
- 1K<n<10K
configs:
- config_name: AST_chinese
data_files:
- split: test
path: "AST_chinese_test.jsonl"
- split: dev
path: "AST_chinese_dev.jsonl"
- config_name: AST_mathematics
data_files:
- split: test
path: "AST_mathematics_test.jsonl"
- split: dev
path: "AST_mathematics_dev.jsonl"
- config_name: AST_biology
data_files:
- split: test
path: "AST_biology_test.jsonl"
- split: dev
path: "AST_biology_dev.jsonl"
- config_name: AST_chemistry
data_files:
- split: test
path: "AST_chemistry_test.jsonl"
- split: dev
path: "AST_chemistry_dev.jsonl"
- config_name: AST_physics
data_files:
- split: test
path: "AST_physics_test.jsonl"
- split: dev
path: "AST_physics_dev.jsonl"
- config_name: AST_civics
data_files:
- split: test
path: "AST_civics_test.jsonl"
- split: dev
path: "AST_civics_dev.jsonl"
- config_name: AST_geography
data_files:
- split: test
path: "AST_geography_test.jsonl"
- split: dev
path: "AST_geography_dev.jsonl"
- config_name: AST_history
data_files:
- split: test
path: "AST_history_test.jsonl"
- split: dev
path: "AST_history_dev.jsonl"
- config_name: GSAT_chinese
data_files:
- split: test
path: "GSAT_chinese_test.jsonl"
- split: dev
path: "GSAT_chinese_dev.jsonl"
- config_name: GSAT_chemistry
data_files:
- split: test
path: "GSAT_chemistry_test.jsonl"
- split: dev
path: "GSAT_chemistry_dev.jsonl"
- config_name: GSAT_biology
data_files:
- split: test
path: "GSAT_biology_test.jsonl"
- split: dev
path: "GSAT_biology_dev.jsonl"
- config_name: GSAT_physics
data_files:
- split: test
path: "GSAT_physics_test.jsonl"
- split: dev
path: "GSAT_physics_dev.jsonl"
- config_name: GSAT_earth_science
data_files:
- split: test
path: "GSAT_earth_science_test.jsonl"
- split: dev
path: "GSAT_earth_science_dev.jsonl"
- config_name: GSAT_mathematics
data_files:
- split: test
path: "GSAT_mathematics_test.jsonl"
- split: dev
path: "GSAT_mathematics_dev.jsonl"
- config_name: GSAT_geography
data_files:
- split: test
path: "GSAT_geography_test.jsonl"
- split: dev
path: "GSAT_geography_dev.jsonl"
- config_name: GSAT_history
data_files:
- split: test
path: "GSAT_history_test.jsonl"
- split: dev
path: "GSAT_history_dev.jsonl"
- config_name: GSAT_civics
data_files:
- split: test
path: "GSAT_civics_test.jsonl"
- split: dev
path: "GSAT_civics_dev.jsonl"
- config_name: CAP_mathematics
data_files:
- split: test
path: "CAP_mathematics_test.jsonl"
- split: dev
path: "CAP_mathematics_dev.jsonl"
- config_name: CAP_biology
data_files:
- split: test
path: "CAP_biology_test.jsonl"
- split: dev
path: "CAP_biology_dev.jsonl"
- config_name: CAP_physics
data_files:
- split: test
path: "CAP_physics_test.jsonl"
- split: dev
path: "CAP_physics_dev.jsonl"
- config_name: CAP_chemistry
data_files:
- split: test
path: "CAP_chemistry_test.jsonl"
- split: dev
path: "CAP_chemistry_dev.jsonl"
- config_name: CAP_earth_science
data_files:
- split: test
path: "CAP_earth_science_test.jsonl"
- split: dev
path: "CAP_earth_science_dev.jsonl"
- config_name: CAP_civics
data_files:
- split: test
path: "CAP_civics_test.jsonl"
- split: dev
path: "CAP_civics_dev.jsonl"
- config_name: CAP_history
data_files:
- split: test
path: "CAP_history_test.jsonl"
- split: dev
path: "CAP_history_dev.jsonl"
- config_name: CAP_geography
data_files:
- split: test
path: "CAP_geography_test.jsonl"
- split: dev
path: "CAP_geography_dev.jsonl"
- config_name: CAP_chinese
data_files:
- split: test
path: "CAP_chinese_test.jsonl"
- split: dev
path: "CAP_chinese_dev.jsonl"
- config_name: driving_rule
data_files:
- split: test
path: "driving_rule_test.jsonl"
- split: dev
path: "driving_rule_dev.jsonl"
- config_name: basic_traditional_chinese_medicine
data_files:
- split: test
path: "basic_traditional_chinese_medicine_test.jsonl"
- split: dev
path: "basic_traditional_chinese_medicine_dev.jsonl"
- config_name: clinical_traditional_chinese_medicine
data_files:
- split: test
path: "clinical_traditional_chinese_medicine_test.jsonl"
- split: dev
path: "clinical_traditional_chinese_medicine_dev.jsonl"
- config_name: lawyer_qualification
data_files:
- split: test
path: "lawyer_qualification_test.jsonl"
- split: dev
path: "lawyer_qualification_dev.jsonl"
- config_name: nutritionist
data_files:
- split: test
path: "nutritionist_test.jsonl"
- split: dev
path: "nutritionist_dev.jsonl"
- config_name: tour_leader
data_files:
- split: test
path: "tour_leader_test.jsonl"
- split: dev
path: "tour_leader_dev.jsonl"
- config_name: tour_guide
data_files:
- split: test
path: "tour_guide_test.jsonl"
- split: dev
path: "tour_guide_dev.jsonl"
- config_name: taiwan_tourist_resources
data_files:
- split: test
path: "taiwan_tourist_resources_test.jsonl"
- split: dev
path: "taiwan_tourist_resources_dev.jsonl"
- config_name: clinical_psychologist
data_files:
- split: test
path: "clinical_psychologist_test.jsonl"
- split: dev
path: "clinical_psychologist_dev.jsonl"
- config_name: teacher_qualification
data_files:
- split: test
path: "teacher_qualification_test.jsonl"
- split: dev
path: "teacher_qualification_dev.jsonl"
- config_name: accountant
data_files:
- split: test
path: "accountant_test.jsonl"
- split: dev
path: "accountant_dev.jsonl"

---
# Dataset Card for Dataset Name

<!-- Provide a quick summary of the dataset. -->

This dataset card aims to be a base template for new datasets. It has been generated using [this raw template](https://github.com/huggingface/huggingface_hub/blob/main/src/huggingface_hub/templates/datasetcard_template.md?plain=1).

## Dataset Details

- AST: 分科測驗（110前指考）
- GSAT: 學科能力測驗
- CAP: 國中教育會考

### Dataset Description

<!-- Provide a longer summary of what this dataset is. -->

- **Curated by:** [More Information Needed]
- **Funded by [optional]:** [More Information Needed]
- **Shared by [optional]:** [More Information Needed]
- **Language(s) (NLP):** [More Information Needed]
- **License:** [More Information Needed]

### Dataset Sources [optional]

<!-- Provide the basic links for the dataset. -->

- **Repository:** [More Information Needed]
- **Paper [optional]:** [More Information Needed]
- **Demo [optional]:** [More Information Needed]

## Uses

<!-- Address questions around how the dataset is intended to be used. -->

### Direct Use

<!-- This section describes suitable use cases for the dataset. -->

[More Information Needed]

### Out-of-Scope Use

<!-- This section addresses misuse, malicious use, and uses that the dataset will not work well for. -->

[More Information Needed]

## Dataset Structure

<!-- This section provides a description of the dataset fields, and additional information about the dataset structure such as criteria used to create the splits, relationships between data points, etc. -->

[More Information Needed]

## Dataset Creation

### Curation Rationale

<!-- Motivation for the creation of this dataset. -->

[More Information Needed]

### Source Data

<!-- This section describes the source data (e.g. news text and headlines, social media posts, translated sentences, ...). -->

#### Data Collection and Processing

<!-- This section describes the data collection and processing process such as data selection criteria, filtering and normalization methods, tools and libraries used, etc. -->

[More Information Needed]

#### Who are the source data producers?

<!-- This section describes the people or systems who originally created the data. It should also include self-reported demographic or identity information for the source data creators if this information is available. -->

[More Information Needed]

### Annotations [optional]

<!-- If the dataset contains annotations which are not part of the initial data collection, use this section to describe them. -->

#### Annotation process

<!-- This section describes the annotation process such as annotation tools used in the process, the amount of data annotated, annotation guidelines provided to the annotators, interannotator statistics, annotation validation, etc. -->

[More Information Needed]

#### Who are the annotators?

<!-- This section describes the people or systems who created the annotations. -->

[More Information Needed]

#### Personal and Sensitive Information

<!-- State whether the dataset contains data that might be considered personal, sensitive, or private (e.g., data that reveals addresses, uniquely identifiable names or aliases, racial or ethnic origins, sexual orientations, religious beliefs, political opinions, financial or health data, etc.). If efforts were made to anonymize the data, describe the anonymization process. -->

[More Information Needed]

## Bias, Risks, and Limitations

<!-- This section is meant to convey both technical and sociotechnical limitations. -->

[More Information Needed]

### Recommendations

<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->

Users should be made aware of the risks, biases and limitations of the dataset. More information needed for further recommendations.

## Citation [optional]

<!-- If there is a paper or blog post introducing the dataset, the APA and Bibtex information for that should go in this section. -->

**BibTeX:**

[More Information Needed]

**APA:**

[More Information Needed]

## Glossary [optional]

<!-- If relevant, include terms and calculations in this section that can help readers understand the dataset or dataset card. -->

[More Information Needed]

## More Information [optional]
### Evaluation
#### CAP
##### ChatGPT
Total: 199 / 389 (0.5116)
| Subject | Accuracy | correct / total |
|:------------- | -------- |:--------------- |
| chinese | 0.5179 | 29 / 56 |
| mathematics | 0.3273 | 36 / 110 |
| physics | 0.5000 | 5 / 10 |
| chemistry | 0.2727 | 6 / 22 |
| biology | 0.4545 | 10 / 22 |
| earth science | 0.4000 | 4 / 10 |
| geography | 0.5750 | 23 / 40 |
| history | 0.8235 | 42 / 51 |
| civics | 0.6471 | 44 / 68 |

##### GPT-4-turbo
Total: 289 / 389 (0.7429)
| Subject | Accuracy | correct / total |
|:------------- | -------- |:--------------- |
| chinese | 0.8571 | 48 / 56 |
| mathematics | 0.4000 | 44 / 110 |
| physics | 0.7000 | 7 / 10 |
| chemistry | 0.8182 | 18 / 22 |
| biology | 0.9091 | 20 / 22 |
| earth science | 0.8000 | 8 / 10 |
| geography | 0.9000 | 36 / 40 |
| history | 0.9608 | 49 / 51 |
| civics | 0.8676 | 59 / 68 |

##### Claude-Instant-1
Total: 214 / 389 (0.5501)
| Subject | Accuracy | correct / total |
|:------------- | -------- |:--------------- |
| chinese | 0.6071 | 34 / 56 |
| mathematics | 0.2636 | 29 / 110 |
| physics | 0.4000 | 4 / 10 |
| chemistry | 0.4545 | 10 / 22 |
| biology | 0.5909 | 13 / 22 |
| earth science | 0.4000 | 4 / 10 |
| geography | 0.6500 | 26 / 40 |
| history | 0.8431 | 43 / 51 |
| civics | 0.7500 | 51 / 68 |

##### Claude-2
Total: 213 / 389 (0.5476)
| Subject | Accuracy | correct / total |
|:------------- | -------- |:--------------- |
| chinese | 0.6071 | 34 / 56 |
| mathematics | 0.3727 | 41 / 110 |
| physics | 0.6000 | 6 / 10 |
| chemistry | 0.5000 | 11 / 22 |
| biology | 0.6364 | 14 / 22 |
| earth science | 0.7000 | 7 / 10 |
| geography | 0.7000 | 28 / 40 |
| history | 0.7255 | 37 / 51 |
| civics | 0.5147 | 35 / 68 |

#### GSAT
##### ChatGPT
Total: 180 / 387 (0.4651)
| Subject | Accuracy | correct / total |
|:------------- | -------- |:--------------- |
| chinese | 0.3587 | 33 / 92 |
| mathematics | 0.2083 | 5 / 24 |
| physics | 0.3684 | 7 / 19 |
| chemistry | 0.2917 | 7 / 24 |
| biology | 0.2500 | 4 / 16 |
| earth science | 0.4211 | 8 / 19 |
| geography | 0.5455 | 24 / 44 |
| history | 0.6049 | 49 / 81 |
| civics | 0.6324 | 43 / 68 |

##### GPT-4-turbo
Total: 293 / 387 (0.7571)
| Subject | Accuracy | correct / total |
|:------------- | -------- |:--------------- |
| chinese | 0.7826 | 72 / 92 |
| mathematics | 0.2500 | 6 / 24 |
| physics | 0.7368 | 14 / 19 |
| chemistry | 0.5417 | 13 / 24 |
| biology | 0.6875 | 11 / 16 |
| earth science | 0.8421 | 16 / 19 |
| geography | 0.8864 | 39 / 44 |
| history | 0.8519 | 69 / 81 |
| civics | 0.7794 | 53 / 68 |

##### Claude-instant-1
Total: 213 / 387 (0.5504)
| Subject | Accuracy | correct / total |
|:------------- | -------- |:--------------- |
| chinese | 0.4891 | 45 / 92 |
| mathematics | 0.2500 | 6 / 24 |
| physics | 0.3684 | 7 / 19 |
| chemistry | 0.3333 | 8 / 24 |
| biology | 0.5625 | 9 / 16 |
| earth science | 0.4211 | 8 / 19 |
| geography | 0.6818 | 30 / 44 |
| history | 0.7160 | 58 / 81 |
| civics | 0.6176 | 42 / 68 |

##### Claude-2
Total: 180 / 387 (0.4651)
| Subject | Accuracy | correct / total |
|:------------- | -------- |:--------------- |
| chinese | 0.3152 | 29 / 92 |
| mathematics | 0.2083 | 5 / 24 |
| physics | 0.3684 | 7 / 19 |
| chemistry | 0.2917 | 7 / 24 |
| biology | 0.1875 | 3 / 16 |
| earth science | 0.2632 | 5 / 19 |
| geography | 0.6818 | 30 / 44 |
| history | 0.6914 | 56 / 81 |
| civics | 0.5588 | 38 / 68 |

#### AST
##### ChatGPT
Total: 193 / 405 (0.4765)
| Subject | Accuracy | correct / total |
|:----------- | -------- |:--------------- |
| chinese | 0.4365 | 55 / 126 |
| mathematics | 0.1500 | 3 / 20 |
| physics | 0.2368 | 9 / 38 |
| chemistry | 0.2759 | 8 / 29 |
| biology | 0.7500 | 27 / 36 |
| geography | 0.5094 | 27 / 53 |
| history | 0.7843 | 40 / 51 |
| civics | 0.4615 | 24 / 52 |

##### GPT-4-turbo
Total: 280 / 405 (0.6914)
| Subject | Accuracy | correct / total |
|:----------- | -------- |:--------------- |
| chinese | 0.7302 | 92 / 126 |
| mathematics | 0.1500 | 3 / 20 |
| physics | 0.5263 | 20 / 38 |
| chemistry | 0.3103 | 9 / 29 |
| biology | 0.8889 | 32 / 36 |
| geography | 0.6981 | 37 / 53 |
| history | 0.9804 | 50 / 51 |
| civics | 0.7115 | 37 / 52 |

##### Claude-instant-1
Total: 219 / 405 (0.5407)
| Subject | Accuracy | correct / total |
|:----------- | -------- |:--------------- |
| chinese | 0.5635 | 71 / 126 |
| mathematics | 0.3500 | 7 / 20 |
| physics | 0.3947 | 15 / 38 |
| chemistry | 0.1724 | 5 / 29 |
| biology | 0.6389 | 23 / 36 |
| geography | 0.6038 | 32 / 53 |
| history | 0.6863 | 35 / 51 |
| civics | 0.5962 | 31 / 52 |

##### Claude-2
Total: 185 / 405 (0.4568)
| Subject | Accuracy | correct / total |
|:----------- | -------- |:--------------- |
| chinese | 0.4365 | 55 / 126 |
| mathematics | 0.0500 | 1 / 20 |
| physics | 0.3421 | 13 / 38 |
| chemistry | 0.1034 | 3 / 29 |
| biology | 0.4444 | 16 / 36 |
| geography | 0.6604 | 35 / 53 |
| history | 0.7255 | 37 / 51 |
| civics | 0.4808 | 25 / 52 |

## Dataset Card Authors [optional]

[More Information Needed]

## Dataset Card Contact

[More Information Needed]

原始信息汇总

数据集概述

数据集详情

任务类别

• 问答
• 文本分类

语言

• 中文

数据集名称

• TMLU

数据规模

• 1K<n<10K

配置详情

数据集包含多个配置，每个配置对应不同学科的测试和开发数据文件。以下是部分配置及其数据文件路径：

AST系列

• AST_chinese
  • 测试集: AST_chinese_test.jsonl
  • 开发集: AST_chinese_dev.jsonl
• AST_mathematics
  • 测试集: AST_mathematics_test.jsonl
  • 开发集: AST_mathematics_dev.jsonl
• AST_biology
  • 测试集: AST_biology_test.jsonl
  • 开发集: AST_biology_dev.jsonl
• AST_chemistry
  • 测试集: AST_chemistry_test.jsonl
  • 开发集: AST_chemistry_dev.jsonl
• AST_physics
  • 测试集: AST_physics_test.jsonl
  • 开发集: AST_physics_dev.jsonl
• AST_civics
  • 测试集: AST_civics_test.jsonl
  • 开发集: AST_civics_dev.jsonl
• AST_geography
  • 测试集: AST_geography_test.jsonl
  • 开发集: AST_geography_dev.jsonl
• AST_history
  • 测试集: AST_history_test.jsonl
  • 开发集: AST_history_dev.jsonl

GSAT系列

• GSAT_chinese
  • 测试集: GSAT_chinese_test.jsonl
  • 开发集: GSAT_chinese_dev.jsonl
• GSAT_chemistry
  • 测试集: GSAT_chemistry_test.jsonl
  • 开发集: GSAT_chemistry_dev.jsonl
• GSAT_biology
  • 测试集: GSAT_biology_test.jsonl
  • 开发集: GSAT_biology_dev.jsonl
• GSAT_physics
  • 测试集: GSAT_physics_test.jsonl
  • 开发集: GSAT_physics_dev.jsonl
• GSAT_earth_science
  • 测试集: GSAT_earth_science_test.jsonl
  • 开发集: GSAT_earth_science_dev.jsonl
• GSAT_mathematics
  • 测试集: GSAT_mathematics_test.jsonl
  • 开发集: GSAT_mathematics_dev.jsonl
• GSAT_geography
  • 测试集: GSAT_geography_test.jsonl
  • 开发集: GSAT_geography_dev.jsonl
• GSAT_history
  • 测试集: GSAT_history_test.jsonl
  • 开发集: GSAT_history_dev.jsonl
• GSAT_civics
  • 测试集: GSAT_civics_test.jsonl
  • 开发集: GSAT_civics_dev.jsonl

CAP系列

• CAP_mathematics
  • 测试集: CAP_mathematics_test.jsonl
  • 开发集: CAP_mathematics_dev.jsonl
• CAP_biology
  • 测试集: CAP_biology_test.jsonl
  • 开发集: CAP_biology_dev.jsonl
• CAP_physics
  • 测试集: CAP_physics_test.jsonl
  • 开发集: CAP_physics_dev.jsonl
• CAP_chemistry
  • 测试集: CAP_chemistry_test.jsonl
  • 开发集: CAP_chemistry_dev.jsonl
• CAP_earth_science
  • 测试集: CAP_earth_science_test.jsonl
  • 开发集: CAP_earth_science_dev.jsonl
• CAP_civics
  • 测试集: CAP_civics_test.jsonl
  • 开发集: CAP_civics_dev.jsonl
• CAP_history
  • 测试集: CAP_history_test.jsonl
  • 开发集: CAP_history_dev.jsonl
• CAP_geography
  • 测试集: CAP_geography_test.jsonl
  • 开发集: CAP_geography_dev.jsonl
• CAP_chinese
  • 测试集: CAP_chinese_test.jsonl
  • 开发集: CAP_chinese_dev.jsonl

其他系列

• driving_rule
  • 测试集: driving_rule_test.jsonl
  • 开发集: driving_rule_dev.jsonl
• basic_traditional_chinese_medicine
  • 测试集: basic_traditional_chinese_medicine_test.jsonl
  • 开发集: basic_traditional_chinese_medicine_dev.jsonl
• clinical_traditional_chinese_medicine
  • 测试集: clinical_traditional_chinese_medicine_test.jsonl
  • 开发集: clinical_traditional_chinese_medicine_dev.jsonl
• lawyer_qualification
  • 测试集: lawyer_qualification_test.jsonl
  • 开发集: lawyer_qualification_dev.jsonl
• nutritionist
  • 测试集: nutritionist_test.jsonl
  • 开发集: nutritionist_dev.jsonl
• tour_leader
  • 测试集: tour_leader_test.jsonl
  • 开发集: tour_leader_dev.jsonl
• tour_guide
  • 测试集: tour_guide_test.jsonl
  • 开发集: tour_guide_dev.jsonl
• taiwan_tourist_resources
  • 测试集: taiwan_tourist_resources_test.jsonl
  • 开发集: taiwan_tourist_resources_dev.jsonl
• clinical_psychologist
  • 测试集: clinical_psychologist_test.jsonl
  • 开发集: clinical_psychologist_dev.jsonl
• teacher_qualification
  • 测试集: teacher_qualification_test.jsonl
  • 开发集: teacher_qualification_dev.jsonl
• accountant
  • 测试集: accountant_test.jsonl
  • 开发集: accountant_dev.jsonl