utter-project/EuroWeb-2512

--- configs: - config_name: "ar" data_files: - split: high path: "ar/high/*.parquet" - split: medium_high path: "ar/medium-high/*.parquet" - split: medium path: "ar/medium/*.parquet" - split: medium_low path: "ar/medium-low/*.parquet" - split: low path: "ar/low/*.parquet" - config_name: "bg" data_files: - split: high path: "bg/high/*.parquet" - split: medium_high path: "bg/medium-high/*.parquet" - split: medium path: "bg/medium/*.parquet" - split: medium_low path: "bg/medium-low/*.parquet" - split: low path: "bg/low/*.parquet" - config_name: "ca" data_files: - split: high path: "ca/high/*.parquet" - split: medium_high path: "ca/medium-high/*.parquet" - split: medium path: "ca/medium/*.parquet" - split: medium_low path: "ca/medium-low/*.parquet" - split: low path: "ca/low/*.parquet" - config_name: "cs" data_files: - split: high path: "cs/high/*.parquet" - split: medium_high path: "cs/medium-high/*.parquet" - split: medium path: "cs/medium/*.parquet" - split: medium_low path: "cs/medium-low/*.parquet" - split: low path: "cs/low/*.parquet" - config_name: "da" data_files: - split: high path: "da/high/*.parquet" - split: medium_high path: "da/medium-high/*.parquet" - split: medium path: "da/medium/*.parquet" - split: medium_low path: "da/medium-low/*.parquet" - split: low path: "da/low/*.parquet" - config_name: "de" data_files: - split: high path: "de/high/*.parquet" - split: medium_high path: "de/medium-high/*.parquet" - split: medium path: "de/medium/*.parquet" - split: medium_low path: "de/medium-low/*.parquet" - split: low path: "de/low/*.parquet" - config_name: "el" data_files: - split: high path: "el/high/*.parquet" - split: medium_high path: "el/medium-high/*.parquet" - split: medium path: "el/medium/*.parquet" - split: medium_low path: "el/medium-low/*.parquet" - split: low path: "el/low/*.parquet" - config_name: "es" data_files: - split: high path: "es/high/*.parquet" - split: medium_high path: "es/medium-high/*.parquet" - split: medium path: "es/medium/*.parquet" - split: medium_low path: "es/medium-low/*.parquet" - split: low path: "es/low/*.parquet" - config_name: "et" data_files: - split: high path: "et/high/*.parquet" - split: medium_high path: "et/medium-high/*.parquet" - split: medium path: "et/medium/*.parquet" - split: medium_low path: "et/medium-low/*.parquet" - split: low path: "et/low/*.parquet" - config_name: "fi" data_files: - split: high path: "fi/high/*.parquet" - split: medium_high path: "fi/medium-high/*.parquet" - split: medium path: "fi/medium/*.parquet" - split: medium_low path: "fi/medium-low/*.parquet" - split: low path: "fi/low/*.parquet" - config_name: "fr" data_files: - split: high path: "fr/high/*.parquet" - split: medium_high path: "fr/medium-high/*.parquet" - split: medium path: "fr/medium/*.parquet" - split: medium_low path: "fr/medium-low/*.parquet" - split: low path: "fr/low/*.parquet" - config_name: "ga" data_files: - split: high path: "ga/high/*.parquet" - split: medium_high path: "ga/medium-high/*.parquet" - split: medium path: "ga/medium/*.parquet" - split: medium_low path: "ga/medium-low/*.parquet" - split: low path: "ga/low/*.parquet" - config_name: "gl" data_files: - split: high path: "gl/high/*.parquet" - split: medium_high path: "gl/medium-high/*.parquet" - split: medium path: "gl/medium/*.parquet" - split: medium_low path: "gl/medium-low/*.parquet" - split: low path: "gl/low/*.parquet" - config_name: "hi" data_files: - split: high path: "hi/high/*.parquet" - split: medium_high path: "hi/medium-high/*.parquet" - split: medium path: "hi/medium/*.parquet" - split: medium_low path: "hi/medium-low/*.parquet" - split: low path: "hi/low/*.parquet" - config_name: "hr" data_files: - split: high path: "hr/high/*.parquet" - split: medium_high path: "hr/medium-high/*.parquet" - split: medium path: "hr/medium/*.parquet" - split: medium_low path: "hr/medium-low/*.parquet" - split: low path: "hr/low/*.parquet" - config_name: "hu" data_files: - split: high path: "hu/high/*.parquet" - split: medium_high path: "hu/medium-high/*.parquet" - split: medium path: "hu/medium/*.parquet" - split: medium_low path: "hu/medium-low/*.parquet" - split: low path: "hu/low/*.parquet" - config_name: "it" data_files: - split: high path: "it/high/*.parquet" - split: medium_high path: "it/medium-high/*.parquet" - split: medium path: "it/medium/*.parquet" - split: medium_low path: "it/medium-low/*.parquet" - split: low path: "it/low/*.parquet" - config_name: "ja" data_files: - split: high path: "ja/high/*.parquet" - split: medium_high path: "ja/medium-high/*.parquet" - split: medium path: "ja/medium/*.parquet" - split: medium_low path: "ja/medium-low/*.parquet" - split: low path: "ja/low/*.parquet" - config_name: "ko" data_files: - split: high path: "ko/high/*.parquet" - split: medium_high path: "ko/medium-high/*.parquet" - split: medium path: "ko/medium/*.parquet" - split: medium_low path: "ko/medium-low/*.parquet" - split: low path: "ko/low/*.parquet" - config_name: "lt" data_files: - split: high path: "lt/high/*.parquet" - split: medium_high path: "lt/medium-high/*.parquet" - split: medium path: "lt/medium/*.parquet" - split: medium_low path: "lt/medium-low/*.parquet" - split: low path: "lt/low/*.parquet" - config_name: "lv" data_files: - split: high path: "lv/high/*.parquet" - split: medium_high path: "lv/medium-high/*.parquet" - split: medium path: "lv/medium/*.parquet" - split: medium_low path: "lv/medium-low/*.parquet" - split: low path: "lv/low/*.parquet" - config_name: "mt" data_files: - split: high path: "mt/high/*.parquet" - split: medium_high path: "mt/medium-high/*.parquet" - split: medium path: "mt/medium/*.parquet" - split: medium_low path: "mt/medium-low/*.parquet" - split: low path: "mt/low/*.parquet" - config_name: "nl" data_files: - split: high path: "nl/high/*.parquet" - split: medium_high path: "nl/medium-high/*.parquet" - split: medium path: "nl/medium/*.parquet" - split: medium_low path: "nl/medium-low/*.parquet" - split: low path: "nl/low/*.parquet" - config_name: "no" data_files: - split: high path: "no/high/*.parquet" - split: medium_high path: "no/medium-high/*.parquet" - split: medium path: "no/medium/*.parquet" - split: medium_low path: "no/medium-low/*.parquet" - split: low path: "no/low/*.parquet" - config_name: "pl" data_files: - split: high path: "pl/high/*.parquet" - split: medium_high path: "pl/medium-high/*.parquet" - split: medium path: "pl/medium/*.parquet" - split: medium_low path: "pl/medium-low/*.parquet" - split: low path: "pl/low/*.parquet" - config_name: "pt" data_files: - split: high path: "pt/high/*.parquet" - split: medium_high path: "pt/medium-high/*.parquet" - split: medium path: "pt/medium/*.parquet" - split: medium_low path: "pt/medium-low/*.parquet" - split: low path: "pt/low/*.parquet" - config_name: "ro" data_files: - split: high path: "ro/high/*.parquet" - split: medium_high path: "ro/medium-high/*.parquet" - split: medium path: "ro/medium/*.parquet" - split: medium_low path: "ro/medium-low/*.parquet" - split: low path: "ro/low/*.parquet" - config_name: "ru" data_files: - split: high path: "ru/high/*.parquet" - split: medium_high path: "ru/medium-high/*.parquet" - split: medium path: "ru/medium/*.parquet" - split: medium_low path: "ru/medium-low/*.parquet" - split: low path: "ru/low/*.parquet" - config_name: "sk" data_files: - split: high path: "sk/high/*.parquet" - split: medium_high path: "sk/medium-high/*.parquet" - split: medium path: "sk/medium/*.parquet" - split: medium_low path: "sk/medium-low/*.parquet" - split: low path: "sk/low/*.parquet" - config_name: "sl" data_files: - split: high path: "sl/high/*.parquet" - split: medium_high path: "sl/medium-high/*.parquet" - split: medium path: "sl/medium/*.parquet" - split: medium_low path: "sl/medium-low/*.parquet" - split: low path: "sl/low/*.parquet" - config_name: "sv" data_files: - split: high path: "sv/high/*.parquet" - split: medium_high path: "sv/medium-high/*.parquet" - split: medium path: "sv/medium/*.parquet" - split: medium_low path: "sv/medium-low/*.parquet" - split: low path: "sv/low/*.parquet" - config_name: "tr" data_files: - split: high path: "tr/high/*.parquet" - split: medium_high path: "tr/medium-high/*.parquet" - split: medium path: "tr/medium/*.parquet" - split: medium_low path: "tr/medium-low/*.parquet" - split: low path: "tr/low/*.parquet" - config_name: "uk" data_files: - split: high path: "uk/high/*.parquet" - split: medium_high path: "uk/medium-high/*.parquet" - split: medium path: "uk/medium/*.parquet" - split: medium_low path: "uk/medium-low/*.parquet" - split: low path: "uk/low/*.parquet" - config_name: "zh" data_files: - split: high path: "zh/high/*.parquet" - split: medium_high path: "zh/medium-high/*.parquet" - split: medium path: "zh/medium/*.parquet" - split: medium_low path: "zh/medium-low/*.parquet" - split: low path: "zh/low/*.parquet" --- # EuroWeb-2512 EuroWeb is a dataset of collecting multilingual web data from various sources. It was processed with standard practices and then classified with [utter-project/EuroFilter-v1](https://huggingface.co/utter-project/EuroFilter-v1). For more information read the [EuroLLM-22B: Technical Report](https://arxiv.org/abs/2602.05879). ## Citation ```bibtex @misc{ramos2026eurollm22btechnicalreport, title={EuroLLM-22B: Technical Report}, author={Miguel Moura Ramos and Duarte M. Alves and Hippolyte Gisserot-Boukhlef and João Alves and Pedro Henrique Martins and Patrick Fernandes and José Pombal and Nuno M. Guerreiro and Ricardo Rei and Nicolas Boizard and Amin Farajian and Mateusz Klimaszewski and José G. C. de Souza and Barry Haddow and François Yvon and Pierre Colombo and Alexandra Birch and André F. T. Martins}, year={2026}, eprint={2602.05879}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2602.05879}, } ```配置项： - 配置名称："ar"（阿拉伯语，ISO 639-1语言代码） 数据文件： - 拆分集：high 文件路径："ar/high/*.parquet"（Parquet列式存储格式） - 拆分集：medium_high 文件路径："ar/medium-high/*.parquet" - 拆分集：medium 文件路径："ar/medium/*.parquet" - 拆分集：medium_low 文件路径："ar/medium-low/*.parquet" - 拆分集：low 文件路径："ar/low/*.parquet" 后续所有配置项的结构与上述阿拉伯语配置完全一致，仅配置名称与文件路径中的语言代码分别对应保加利亚语（bg）、加泰罗尼亚语（ca）、捷克语（cs）、丹麦语（da）、德语（de）、希腊语（el）、西班牙语（es）、爱沙尼亚语（et）、芬兰语（fi）、法语（fr）、爱尔兰语（ga）、加利西亚语（gl）、印地语（hi）、克罗地亚语（hr）、匈牙利语（hu）、意大利语（it）、日语（ja）、韩语（ko）、立陶宛语（lt）、拉脱维亚语（lv）、马耳他语（mt）、荷兰语（nl）、挪威语（no）、波兰语（pl）、葡萄牙语（pt）、罗马尼亚语（ro）、俄语（ru）、斯洛伐克语（sk）、斯洛文尼亚语（sl）、瑞典语（sv）、土耳其语（tr）、乌克兰语（uk）、中文（zh）。 # EuroWeb-2512 数据集 EuroWeb 是一款多语言网页数据采集数据集，其采用标准流程完成预处理，并通过 [utter-project/EuroFilter-v1](https://huggingface.co/utter-project/EuroFilter-v1) 完成数据分类标注。 如需了解更多详情，请参阅《EuroLLM-22B：技术报告》（https://arxiv.org/abs/2602.05879）。 ## 引用 bibtex @misc{ramos2026eurollm22btechnicalreport, title={EuroLLM-22B: Technical Report}, author={Miguel Moura Ramos and Duarte M. Alves and Hippolyte Gisserot-Boukhlef and João Alves and Pedro Henrique Martins and Patrick Fernandes and José Pombal and Nuno M. Guerreiro and Ricardo Rei and Nicolas Boizard and Amin Farajian and Mateusz Klimaszewski and José G. C. de Souza and Barry Haddow and François Yvon and Pierre Colombo and Alexandra Birch and André F. T. Martins}, year={2026}, eprint={2602.05879}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2602.05879}, }