mteb/AfriXNLI

--- annotations_creators: - human-annotated language: - amh - eng - ewe - fra - hau - ibo - kin - lin - lug - orm - sna - sot - swa - twi - wol - xho - yor - zul license: cc-by-4.0 multilinguality: multilingual source_datasets: - masakhane/afrixnli task_categories: - text-classification task_ids: - natural-language-inference dataset_info: - config_name: amh features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 82220.0 num_examples: 300 - name: test num_bytes: 108543.0 num_examples: 400 download_size: 74147 dataset_size: 190763.0 - config_name: eng features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 52146.0 num_examples: 300 - name: test num_bytes: 66098.0 num_examples: 400 download_size: 57599 dataset_size: 118244.0 - config_name: ewe features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 50618.0 num_examples: 300 - name: test num_bytes: 65034.0075 num_examples: 399 download_size: 54043 dataset_size: 115652.0075 - config_name: fra features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 61306.0 num_examples: 300 - name: test num_bytes: 77694.0 num_examples: 400 download_size: 65749 dataset_size: 139000.0 - config_name: gaz features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 60860.0 num_examples: 300 - name: test num_bytes: 77965.0 num_examples: 400 download_size: 62309 dataset_size: 138825.0 - config_name: hau features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 52198.0 num_examples: 300 - name: test num_bytes: 68341.0 num_examples: 400 download_size: 56601 dataset_size: 120539.0 - config_name: ibo features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 62998.0 num_examples: 300 - name: test num_bytes: 80525.0 num_examples: 400 download_size: 59789 dataset_size: 143523.0 - config_name: kin features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 50611.0 num_examples: 300 - name: test num_bytes: 64702.0 num_examples: 400 download_size: 56419 dataset_size: 115313.0 - config_name: lin features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 55425.0 num_examples: 300 - name: test num_bytes: 60450.495 num_examples: 399 download_size: 56219 dataset_size: 115875.495 - config_name: lug features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 59827.0 num_examples: 300 - name: test num_bytes: 78305.0 num_examples: 400 download_size: 64424 dataset_size: 138132.0 - config_name: sna features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 55677.0 num_examples: 300 - name: test num_bytes: 71533.0 num_examples: 400 download_size: 59859 dataset_size: 127210.0 - config_name: sot features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 55290.0 num_examples: 300 - name: test num_bytes: 70199.0 num_examples: 400 download_size: 56651 dataset_size: 125489.0 - config_name: swh features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 51679.0 num_examples: 300 - name: test num_bytes: 66611.0 num_examples: 400 download_size: 56866 dataset_size: 118290.0 - config_name: twi features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 56804.0 num_examples: 300 - name: test num_bytes: 69996.0 num_examples: 400 download_size: 56630 dataset_size: 126800.0 - config_name: wol features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 55466.0 num_examples: 300 - name: test num_bytes: 70362.0 num_examples: 400 download_size: 61561 dataset_size: 125828.0 - config_name: xho features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 55524.0 num_examples: 300 - name: test num_bytes: 71000.0 num_examples: 400 download_size: 60993 dataset_size: 126524.0 - config_name: yor features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 68968.0 num_examples: 300 - name: test num_bytes: 88506.0 num_examples: 400 download_size: 62544 dataset_size: 157474.0 - config_name: zul features: - name: sentence1 dtype: string - name: sentence2 dtype: string - name: labels dtype: int64 splits: - name: validation num_bytes: 54169.83 num_examples: 299 - name: test num_bytes: 68975.13 num_examples: 399 download_size: 59598 dataset_size: 123144.96 configs: - config_name: amh data_files: - split: validation path: amh/validation-* - split: test path: amh/test-* - config_name: eng data_files: - split: validation path: eng/validation-* - split: test path: eng/test-* - config_name: ewe data_files: - split: validation path: ewe/validation-* - split: test path: ewe/test-* - config_name: fra data_files: - split: validation path: fra/validation-* - split: test path: fra/test-* - config_name: gaz data_files: - split: validation path: gaz/validation-* - split: test path: gaz/test-* - config_name: hau data_files: - split: validation path: hau/validation-* - split: test path: hau/test-* - config_name: ibo data_files: - split: validation path: ibo/validation-* - split: test path: ibo/test-* - config_name: kin data_files: - split: validation path: kin/validation-* - split: test path: kin/test-* - config_name: lin data_files: - split: validation path: lin/validation-* - split: test path: lin/test-* - config_name: lug data_files: - split: validation path: lug/validation-* - split: test path: lug/test-* - config_name: sna data_files: - split: validation path: sna/validation-* - split: test path: sna/test-* - config_name: sot data_files: - split: validation path: sot/validation-* - split: test path: sot/test-* - config_name: swh data_files: - split: validation path: swh/validation-* - split: test path: swh/test-* - config_name: twi data_files: - split: validation path: twi/validation-* - split: test path: twi/test-* - config_name: wol data_files: - split: validation path: wol/validation-* - split: test path: wol/test-* - config_name: xho data_files: - split: validation path: xho/validation-* - split: test path: xho/test-* - config_name: yor data_files: - split: validation path: yor/validation-* - split: test path: yor/test-* - config_name: zul data_files: - split: validation path: zul/validation-* - split: test path: zul/test-* tags: - mteb - text --- <!-- adapted from https://github.com/huggingface/huggingface_hub/blob/v0.30.2/src/huggingface_hub/templates/datasetcard_template.md --> <div align="center" style="padding: 40px 20px; background-color: white; border-radius: 12px; box-shadow: 0 2px 10px rgba(0, 0, 0, 0.05); max-width: 600px; margin: 0 auto;"> <h1 style="font-size: 3.5rem; color: #1a1a1a; margin: 0 0 20px 0; letter-spacing: 2px; font-weight: 700;">AfriXNLI</h1> <div style="font-size: 1.5rem; color: #4a4a4a; margin-bottom: 5px; font-weight: 300;">An <a href="https://github.com/embeddings-benchmark/mteb" style="color: #2c5282; font-weight: 600; text-decoration: none;" onmouseover="this.style.textDecoration='underline'" onmouseout="this.style.textDecoration='none'">MTEB</a> dataset</div> <div style="font-size: 0.9rem; color: #2c5282; margin-top: 10px;">Massive Text Embedding Benchmark</div> </div> Cross-lingual natural language inference dataset focusing on African languages. | | | |---------------|---------------------------------------------| | Task category | t2t | | Domains | News, Written | | Reference | https://github.com/masakhane-io/afri-xnli | Source datasets: - [masakhane/afrixnli](https://huggingface.co/datasets/masakhane/afrixnli) ## Dataset Preparation in MTEB This repository is a staging copy of `masakhane/afrixnli` for MTEB. The intended long-term canonical benchmark copy is `mteb/AfriXNLI`. ### Transformations - Filtered the source labels to contradiction and entailment only (`label in {0, 2}`) - Renamed `premise` -> `sentence1` and `hypothesis` -> `sentence2` - Mapped labels to the binary pair-classification convention used by MTEB - Preserved the MTEB-facing subset names, including `gaz` and `swh`, while sourcing from the original Hub configs - Removed empty pairs, duplicate pairs, and pair-level label conflicts in the staging copy ### Label Schema - `0`: contradiction - `1`: entailment ### Splits and subsets - Language-specific configs are preserved from the benchmark task - Each config contains the transformed pair-classification validation/test splits used by MTEB ## How to evaluate on this task You can evaluate an embedding model on this dataset using the following code: ```python import mteb task = mteb.get_task("AfriXNLI") evaluator = mteb.MTEB([task]) model = mteb.get_model(YOUR_MODEL) evaluator.run(model) ``` <!-- Datasets want link to arxiv in readme to autolink dataset with paper --> To learn more about how to run models on `mteb` task check out the [GitHub repository](https://github.com/embeddings-benchmark/mteb). ## Citation If you use this dataset, please cite the dataset as well as [mteb](https://github.com/embeddings-benchmark/mteb), as this dataset likely includes additional processing as a part of the [MMTEB Contribution](https://github.com/embeddings-benchmark/mteb/tree/main/docs/mmteb). ```bibtex @article{enevoldsen2025mmtebmassivemultilingualtext, title={MMTEB: Massive Multilingual Text Embedding Benchmark}, author={Kenneth Enevoldsen and Isaac Chung and Imene Kerboua and Márton Kardos and Ashwin Mathur and David Stap and Jay Gala and Wissam Siblini and Dominik Krzemiński and Genta Indra Winata and Saba Sturua and Saiteja Utpala and Mathieu Ciancone and Marion Schaeffer and Gabriel Sequeira and Diganta Misra and Shreeya Dhakal and Jonathan Rystrøm and Roman Solomatin and Ömer Çağatan and Akash Kundu and Martin Bernstorff and Shitao Xiao and Akshita Sukhlecha and Bhavish Pahwa and Rafał Poświata and Kranthi Kiran GV and Shawon Ashraf and Daniel Auras and Björn Plüster and Jan Philipp Harries and Loïc Magne and Isabelle Mohr and Mariya Hendriksen and Dawei Zhu and Hippolyte Gisserot-Boukhlef and Tom Aarsen and Jan Kostkan and Konrad Wojtasik and Taemin Lee and Marek Šuppa and Crystina Zhang and Roberta Rocca and Mohammed Hamdy and Andrianos Michail and John Yang and Manuel Faysse and Aleksei Vatolin and Nandan Thakur and Manan Dey and Dipam Vasani and Pranjal Chitale and Simone Tedeschi and Nguyen Tai and Artem Snegirev and Michael Günther and Mengzhou Xia and Weijia Shi and Xing Han Lù and Jordan Clive and Gayatri Krishnakumar and Anna Maksimova and Silvan Wehrli and Maria Tikhonova and Henil Panchal and Aleksandr Abramov and Malte Ostendorff and Zheng Liu and Simon Clematide and Lester James Miranda and Alena Fenogenova and Guangyu Song and Ruqiya Bin Safi and Wen-Ding Li and Alessia Borghini and Federico Cassano and Hongjin Su and Jimmy Lin and Howard Yen and Lasse Hansen and Sara Hooker and Chenghao Xiao and Vaibhav Adlakha and Orion Weller and Siva Reddy and Niklas Muennighoff}, publisher = {arXiv}, journal={arXiv preprint arXiv:2502.13595}, year={2025}, url={https://arxiv.org/abs/2502.13595}, doi = {10.48550/arXiv.2502.13595}, } @article{muennighoff2022mteb, author = {Muennighoff, Niklas and Tazi, Nouamane and Magne, Loïc and Reimers, Nils}, title = {MTEB: Massive Text Embedding Benchmark}, publisher = {arXiv}, journal={arXiv preprint arXiv:2210.07316}, year = {2022} url = {https://arxiv.org/abs/2210.07316}, doi = {10.48550/ARXIV.2210.07316}, } ``` # Dataset Statistics <details> <summary> Dataset Statistics</summary> The following code contains the descriptive statistics from the task. These can also be obtained using: ```python import mteb task = mteb.get_task("AfriXNLI") desc_stats = task.metadata.descriptive_stats ``` ```json { "test": { "num_samples": 7200, "unique_pairs": 7197, "number_of_characters": 1092291, "text1_statistics": { "total_text_length": 707366, "min_text_length": 14, "average_text_length": 98.24527777777777, "max_text_length": 324, "unique_texts": 3600 }, "image1_statistics": null, "audio1_statistics": null, "text2_statistics": { "total_text_length": 384925, "min_text_length": 10, "average_text_length": 53.46180555555556, "max_text_length": 230, "unique_texts": 7196 }, "image2_statistics": null, "audio2_statistics": null, "labels_statistics": { "min_labels_per_text": 1, "average_label_per_text": 1.0, "max_labels_per_text": 1, "unique_labels": 2, "labels": { "0": { "count": 3600 }, "1": { "count": 3600 } } } } } ``` </details> --- *This dataset card was automatically generated using [MTEB](https://github.com/embeddings-benchmark/mteb)*注释创建者： - 人工标注（human-annotated） 语言： - 阿姆哈拉语（amh） - 英语（eng） - 埃维语（ewe） - 法语（fra） - 豪萨语（hau） - 伊博语（ibo） - 卢旺达语（kin） - 林加拉语（lin） - 卢干达语（lug） - 奥罗莫语（orm） - 绍纳语（sna） - 索托语（sot） - 斯瓦希里语（swa） - 特维语（twi） - 沃洛夫语（wol） - 科萨语（xho） - 约鲁巴语（yor） - 祖鲁语（zul） 许可协议：CC-BY-4.0 多语言属性：多语言 源数据集： - masakhane/afrixnli 任务类别： - 文本分类（text-classification） 任务子类型： - 自然语言推理（natural-language-inference） 数据集信息： - 配置名称：amh 特征： - 字段名：sentence1，数据类型：字符串（string） - 字段名：sentence2，数据类型：字符串（string） - 字段名：labels，数据类型：int64 数据集划分： - 划分名称：验证集（validation），占用字节数：82220.0，样本数：300 - 划分名称：测试集（test），占用字节数：108543.0，样本数：400 下载大小：74147 数据集总大小：190763.0 - 配置名称：eng 特征： - 字段名：sentence1，数据类型：字符串（string） - 字段名：sentence2，数据类型：字符串（string） - 字段名：labels，数据类型：int64 数据集划分： - 划分名称：验证集（validation），占用字节数：52146.0，样本数：300 - 划分名称：测试集（test），占用字节数：66098.0，样本数：400 下载大小：57599 数据集总大小：118244.0 （其余语言配置格式与上述一致，此处省略） 配置项： - 配置名称：amh 数据文件： - 划分：验证集（validation），路径：amh/validation-* - 划分：测试集（test），路径：amh/test-* （其余语言配置格式与上述一致，此处省略） 标签： - mteb - 文本 <div align="center" style="padding: 40px 20px; background-color: white; border-radius: 12px; box-shadow: 0 2px 10px rgba(0, 0, 0, 0.05); max-width: 600px; margin: 0 auto;"> <h1 style="font-size: 3.5rem; color: #1a1a1a; margin: 0 0 20px 0; letter-spacing: 2px; font-weight: 700;">AfriXNLI</h1> <div style="font-size: 1.5rem; color: #4a4a4a; margin-bottom: 5px; font-weight: 300;">一款<a href="https://github.com/embeddings-benchmark/mteb" style="color: #2c5282; font-weight: 600; text-decoration: none;" onmouseover="this.style.textDecoration='underline'" onmouseout="this.style.textDecoration='none'">大规模文本嵌入基准（Massive Text Embedding Benchmark, 简称MTEB）</a>数据集</div> <div style="font-size: 0.9rem; color: #2c5282; margin-top: 10px;">大规模文本嵌入基准</div> </div> 跨语言自然语言推理数据集，聚焦非洲语言。 | | | |---------------|---------------------------------------------| | 任务类别 | t2t | | 领域 | 新闻、书面文本 | | 参考文献 | https://github.com/masakhane-io/afri-xnli | 源数据集： - [masakhane/afrixnli](https://huggingface.co/datasets/masakhane/afrixnli) ## 大规模文本嵌入基准（MTEB）中的数据集预处理 本仓库为适配MTEB的`masakhane/afrixnli`临时副本，长期规范的基准副本存储于`mteb/AfriXNLI`。 ### 数据变换 - 仅保留源数据集中的矛盾（contradiction）与蕴含（entailment）两类标签（即`label ∈ {0, 2}`） - 将`premise`（前提句）重命名为`sentence1`，`hypothesis`（假设句）重命名为`sentence2` - 将标签映射至MTEB采用的二元句对分类规范 - 保留面向MTEB的子集名称（如`gaz`、`swh`），并从原始Hub配置中获取数据 - 移除临时副本中的空句对、重复句对以及句对级别的标签冲突样本 ### 标签规范 - `0`：矛盾样本 - `1`：蕴含样本 ### 划分与子集 - 保留基准任务中针对各语言的配置 - 每个配置均包含MTEB所用的经过变换的句对分类验证集与测试集划分 ## 该任务的评估方法 可通过以下代码在本数据集上评估嵌入模型（embedding model）： python import mteb task = mteb.get_task("AfriXNLI") evaluator = mteb.MTEB([task]) model = mteb.get_model(YOUR_MODEL) evaluator.run(model) 如需了解如何在MTEB任务中运行模型，请访问[GitHub仓库](https://github.com/embeddings-benchmark/mteb)。 ## 引用声明 若您使用本数据集，请同时引用本数据集与[MTEB](https://github.com/embeddings-benchmark/mteb)，因为本数据集作为[大规模多语言文本嵌入基准（MMTEB）贡献项](https://github.com/embeddings-benchmark/mteb/tree/main/docs/mmteb)的一部分，可能包含额外的预处理步骤。 bibtex @article{enevoldsen2025mmtebmassivemultilingualtext, title={MMTEB: Massive Multilingual Text Embedding Benchmark}, author={Kenneth Enevoldsen and Isaac Chung and Imene Kerboua and Márton Kardos and Ashwin Mathur and David Stap and Jay Gala and Wissam Siblini and Dominik Krzemiński and Genta Indra Winata and Saba Sturua and Saiteja Utpala and Mathieu Ciancone and Marion Schaeffer and Gabriel Sequeira and Diganta Misra and Shreeya Dhakal and Jonathan Rystrøm and Roman Solomatin and Ömer Çağatan and Akash Kundu and Martin Bernstorff and Shitao Xiao and Akshita Sukhlecha and Bhavish Pahwa and Rafał Poświata and Kranthi Kiran GV and Shawon Ashraf and Daniel Auras and Björn Plüster and Jan Philipp Harries and Loïc Magne and Isabelle Mohr and Mariya Hendriksen and Dawei Zhu and Hippolyte Gisserot-Boukhlef and Tom Aarsen and Jan Kostkan and Konrad Wojtasik and Taemin Lee and Marek Šuppa and Crystina Zhang and Roberta Rocca and Mohammed Hamdy and Andrianos Michail and John Yang and Manuel Faysse and Aleksei Vatolin and Nandan Thakur and Manan Dey and Dipam Vasani and Pranjal Chitale and Simone Tedeschi and Nguyen Tai and Artem Snegirev and Michael Günther and Mengzhou Xia and Weijia Shi and Xing Han Lù and Jordan Clive and Gayatri Krishnakumar and Anna Maksimova and Silvan Wehrli and Maria Tikhonova and Henil Panchal and Aleksandr Abramov and Malte Ostendorff and Zheng Liu and Simon Clematide and Lester James Miranda and Alena Fenogenova and Guangyu Song and Ruqiya Bin Safi and Wen-Ding Li and Alessia Borghini and Federico Cassano and Hongjin Su and Jimmy Lin and Howard Yen and Lasse Hansen and Sara Hooker and Chenghao Xiao and Vaibhav Adlakha and Orion Weller and Siva Reddy and Niklas Muennighoff}, publisher = {arXiv}, journal={arXiv preprint arXiv:2502.13595}, year={2025}, url={https://arxiv.org/abs/2502.13595}, doi = {10.48550/arXiv.2502.13595}, } @article{muennighoff2022mteb, author = {Muennighoff, Niklas and Tazi, Nouamane and Magne, Loïc and Reimers, Nils}, title = {MTEB: Massive Text Embedding Benchmark}, publisher = {arXiv}, journal={arXiv preprint arXiv:2210.07316}, year = {2022} url = {https://arxiv.org/abs/2210.07316}, doi = {10.48550/ARXIV.2210.07316}, } # 数据集统计信息 <details> <summary> 数据集统计信息</summary> 以下为该任务的描述性统计数据，也可通过以下代码获取： python import mteb task = mteb.get_task("AfriXNLI") desc_stats = task.metadata.descriptive_stats json { "test": { "num_samples": 7200, "unique_pairs": 7197, "number_of_characters": 1092291, "text1_statistics": { "total_text_length": 707366, "min_text_length": 14, "average_text_length": 98.24527777777777, "max_text_length": 324, "unique_texts": 3600 }, "image1_statistics": null, "audio1_statistics": null, "text2_statistics": { "total_text_length": 384925, "min_text_length": 10, "average_text_length": 53.46180555555556, "max_text_length": 230, "unique_texts": 7196 }, "image2_statistics": null, "audio2_statistics": null, "labels_statistics": { "min_labels_per_text": 1, "average_label_per_text": 1.0, "max_labels_per_text": 1, "unique_labels": 2, "labels": { "0": { "count": 3600 }, "1": { "count": 3600 } } } } } </details> --- *本数据集卡片由[MTEB](https://github.com/embeddings-benchmark/mteb)自动生成*