Cohere/wikipedia-2023-11-embed-multilingual-v3-int8-binary
收藏Hugging Face2024-03-21 更新2024-05-25 收录
下载链接：
https://hf-mirror.com/datasets/Cohere/wikipedia-2023-11-embed-multilingual-v3-int8-binary
下载链接
链接失效反馈官方服务：
资源简介：
---
configs:
- config_name: "ab"
data_files:
- split: train
path: ab/*
- config_name: "ace"
data_files:
- split: train
path: ace/*
- config_name: "ady"
data_files:
- split: train
path: ady/*
- config_name: "af"
data_files:
- split: train
path: af/*
- config_name: "als"
data_files:
- split: train
path: als/*
- config_name: "alt"
data_files:
- split: train
path: alt/*
- config_name: "am"
data_files:
- split: train
path: am/*
- config_name: "ami"
data_files:
- split: train
path: ami/*
- config_name: "an"
data_files:
- split: train
path: an/*
- config_name: "ang"
data_files:
- split: train
path: ang/*
- config_name: "anp"
data_files:
- split: train
path: anp/*
- config_name: "ar"
data_files:
- split: train
path: ar/*
- config_name: "arc"
data_files:
- split: train
path: arc/*
- config_name: "ary"
data_files:
- split: train
path: ary/*
- config_name: "arz"
data_files:
- split: train
path: arz/*
- config_name: "as"
data_files:
- split: train
path: as/*
- config_name: "ast"
data_files:
- split: train
path: ast/*
- config_name: "atj"
data_files:
- split: train
path: atj/*
- config_name: "av"
data_files:
- split: train
path: av/*
- config_name: "avk"
data_files:
- split: train
path: avk/*
- config_name: "awa"
data_files:
- split: train
path: awa/*
- config_name: "ay"
data_files:
- split: train
path: ay/*
- config_name: "az"
data_files:
- split: train
path: az/*
- config_name: "azb"
data_files:
- split: train
path: azb/*
- config_name: "ba"
data_files:
- split: train
path: ba/*
- config_name: "ban"
data_files:
- split: train
path: ban/*
- config_name: "bar"
data_files:
- split: train
path: bar/*
- config_name: "bat-smg"
data_files:
- split: train
path: bat-smg/*
- config_name: "bcl"
data_files:
- split: train
path: bcl/*
- config_name: "be"
data_files:
- split: train
path: be/*
- config_name: "be-x-old"
data_files:
- split: train
path: be-x-old/*
- config_name: "bg"
data_files:
- split: train
path: bg/*
- config_name: "bh"
data_files:
- split: train
path: bh/*
- config_name: "bi"
data_files:
- split: train
path: bi/*
- config_name: "bjn"
data_files:
- split: train
path: bjn/*
- config_name: "blk"
data_files:
- split: train
path: blk/*
- config_name: "bm"
data_files:
- split: train
path: bm/*
- config_name: "bn"
data_files:
- split: train
path: bn/*
- config_name: "bo"
data_files:
- split: train
path: bo/*
- config_name: "bpy"
data_files:
- split: train
path: bpy/*
- config_name: "br"
data_files:
- split: train
path: br/*
- config_name: "bs"
data_files:
- split: train
path: bs/*
- config_name: "bug"
data_files:
- split: train
path: bug/*
- config_name: "bxr"
data_files:
- split: train
path: bxr/*
- config_name: "ca"
data_files:
- split: train
path: ca/*
- config_name: "cbk-zam"
data_files:
- split: train
path: cbk-zam/*
- config_name: "cdo"
data_files:
- split: train
path: cdo/*
- config_name: "ce"
data_files:
- split: train
path: ce/*
- config_name: "ceb"
data_files:
- split: train
path: ceb/*
- config_name: "ch"
data_files:
- split: train
path: ch/*
- config_name: "chr"
data_files:
- split: train
path: chr/*
- config_name: "chy"
data_files:
- split: train
path: chy/*
- config_name: "ckb"
data_files:
- split: train
path: ckb/*
- config_name: "co"
data_files:
- split: train
path: co/*
- config_name: "cr"
data_files:
- split: train
path: cr/*
- config_name: "crh"
data_files:
- split: train
path: crh/*
- config_name: "cs"
data_files:
- split: train
path: cs/*
- config_name: "csb"
data_files:
- split: train
path: csb/*
- config_name: "cu"
data_files:
- split: train
path: cu/*
- config_name: "cv"
data_files:
- split: train
path: cv/*
- config_name: "cy"
data_files:
- split: train
path: cy/*
- config_name: "da"
data_files:
- split: train
path: da/*
- config_name: "dag"
data_files:
- split: train
path: dag/*
- config_name: "de"
data_files:
- split: train
path: de/*
- config_name: "din"
data_files:
- split: train
path: din/*
- config_name: "diq"
data_files:
- split: train
path: diq/*
- config_name: "dsb"
data_files:
- split: train
path: dsb/*
- config_name: "dty"
data_files:
- split: train
path: dty/*
- config_name: "dv"
data_files:
- split: train
path: dv/*
- config_name: "dz"
data_files:
- split: train
path: dz/*
- config_name: "ee"
data_files:
- split: train
path: ee/*
- config_name: "el"
data_files:
- split: train
path: el/*
- config_name: "eml"
data_files:
- split: train
path: eml/*
- config_name: "en"
data_files:
- split: train
path: en/*
- config_name: "eo"
data_files:
- split: train
path: eo/*
- config_name: "es"
data_files:
- split: train
path: es/*
- config_name: "et"
data_files:
- split: train
path: et/*
- config_name: "eu"
data_files:
- split: train
path: eu/*
- config_name: "ext"
data_files:
- split: train
path: ext/*
- config_name: "fa"
data_files:
- split: train
path: fa/*
- config_name: "fat"
data_files:
- split: train
path: fat/*
- config_name: "ff"
data_files:
- split: train
path: ff/*
- config_name: "fi"
data_files:
- split: train
path: fi/*
- config_name: "fiu-vro"
data_files:
- split: train
path: fiu-vro/*
- config_name: "fj"
data_files:
- split: train
path: fj/*
- config_name: "fo"
data_files:
- split: train
path: fo/*
- config_name: "fon"
data_files:
- split: train
path: fon/*
- config_name: "fr"
data_files:
- split: train
path: fr/*
- config_name: "frp"
data_files:
- split: train
path: frp/*
- config_name: "frr"
data_files:
- split: train
path: frr/*
- config_name: "fur"
data_files:
- split: train
path: fur/*
- config_name: "fy"
data_files:
- split: train
path: fy/*
- config_name: "ga"
data_files:
- split: train
path: ga/*
- config_name: "gag"
data_files:
- split: train
path: gag/*
- config_name: "gan"
data_files:
- split: train
path: gan/*
- config_name: "gcr"
data_files:
- split: train
path: gcr/*
- config_name: "gd"
data_files:
- split: train
path: gd/*
- config_name: "gl"
data_files:
- split: train
path: gl/*
- config_name: "glk"
data_files:
- split: train
path: glk/*
- config_name: "gn"
data_files:
- split: train
path: gn/*
- config_name: "gom"
data_files:
- split: train
path: gom/*
- config_name: "gor"
data_files:
- split: train
path: gor/*
- config_name: "got"
data_files:
- split: train
path: got/*
- config_name: "gpe"
data_files:
- split: train
path: gpe/*
- config_name: "gu"
data_files:
- split: train
path: gu/*
- config_name: "guc"
data_files:
- split: train
path: guc/*
- config_name: "gur"
data_files:
- split: train
path: gur/*
- config_name: "guw"
data_files:
- split: train
path: guw/*
- config_name: "gv"
data_files:
- split: train
path: gv/*
- config_name: "ha"
data_files:
- split: train
path: ha/*
- config_name: "hak"
data_files:
- split: train
path: hak/*
- config_name: "haw"
data_files:
- split: train
path: haw/*
- config_name: "he"
data_files:
- split: train
path: he/*
- config_name: "hi"
data_files:
- split: train
path: hi/*
- config_name: "hif"
data_files:
- split: train
path: hif/*
- config_name: "hr"
data_files:
- split: train
path: hr/*
- config_name: "hsb"
data_files:
- split: train
path: hsb/*
- config_name: "ht"
data_files:
- split: train
path: ht/*
- config_name: "hu"
data_files:
- split: train
path: hu/*
- config_name: "hy"
data_files:
- split: train
path: hy/*
- config_name: "hyw"
data_files:
- split: train
path: hyw/*
- config_name: "ia"
data_files:
- split: train
path: ia/*
- config_name: "id"
data_files:
- split: train
path: id/*
- config_name: "ie"
data_files:
- split: train
path: ie/*
- config_name: "ig"
data_files:
- split: train
path: ig/*
- config_name: "ik"
data_files:
- split: train
path: ik/*
- config_name: "ilo"
data_files:
- split: train
path: ilo/*
- config_name: "inh"
data_files:
- split: train
path: inh/*
- config_name: "io"
data_files:
- split: train
path: io/*
- config_name: "is"
data_files:
- split: train
path: is/*
- config_name: "it"
data_files:
- split: train
path: it/*
- config_name: "iu"
data_files:
- split: train
path: iu/*
- config_name: "ja"
data_files:
- split: train
path: ja/*
- config_name: "jam"
data_files:
- split: train
path: jam/*
- config_name: "jbo"
data_files:
- split: train
path: jbo/*
- config_name: "jv"
data_files:
- split: train
path: jv/*
- config_name: "ka"
data_files:
- split: train
path: ka/*
- config_name: "kaa"
data_files:
- split: train
path: kaa/*
- config_name: "kab"
data_files:
- split: train
path: kab/*
- config_name: "kbd"
data_files:
- split: train
path: kbd/*
- config_name: "kbp"
data_files:
- split: train
path: kbp/*
- config_name: "kcg"
data_files:
- split: train
path: kcg/*
- config_name: "kg"
data_files:
- split: train
path: kg/*
- config_name: "ki"
data_files:
- split: train
path: ki/*
- config_name: "kk"
data_files:
- split: train
path: kk/*
- config_name: "kl"
data_files:
- split: train
path: kl/*
- config_name: "km"
data_files:
- split: train
path: km/*
- config_name: "kn"
data_files:
- split: train
path: kn/*
- config_name: "ko"
data_files:
- split: train
path: ko/*
- config_name: "koi"
data_files:
- split: train
path: koi/*
- config_name: "krc"
data_files:
- split: train
path: krc/*
- config_name: "ks"
data_files:
- split: train
path: ks/*
- config_name: "ksh"
data_files:
- split: train
path: ksh/*
- config_name: "ku"
data_files:
- split: train
path: ku/*
- config_name: "kv"
data_files:
- split: train
path: kv/*
- config_name: "kw"
data_files:
- split: train
path: kw/*
- config_name: "ky"
data_files:
- split: train
path: ky/*
- config_name: "la"
data_files:
- split: train
path: la/*
- config_name: "lad"
data_files:
- split: train
path: lad/*
- config_name: "lb"
data_files:
- split: train
path: lb/*
- config_name: "lbe"
data_files:
- split: train
path: lbe/*
- config_name: "lez"
data_files:
- split: train
path: lez/*
- config_name: "lfn"
data_files:
- split: train
path: lfn/*
- config_name: "lg"
data_files:
- split: train
path: lg/*
- config_name: "li"
data_files:
- split: train
path: li/*
- config_name: "lij"
data_files:
- split: train
path: lij/*
- config_name: "lld"
data_files:
- split: train
path: lld/*
- config_name: "lmo"
data_files:
- split: train
path: lmo/*
- config_name: "ln"
data_files:
- split: train
path: ln/*
- config_name: "lo"
data_files:
- split: train
path: lo/*
- config_name: "lt"
data_files:
- split: train
path: lt/*
- config_name: "ltg"
data_files:
- split: train
path: ltg/*
- config_name: "lv"
data_files:
- split: train
path: lv/*
- config_name: "mad"
data_files:
- split: train
path: mad/*
- config_name: "mai"
data_files:
- split: train
path: mai/*
- config_name: "map-bms"
data_files:
- split: train
path: map-bms/*
- config_name: "mdf"
data_files:
- split: train
path: mdf/*
- config_name: "mg"
data_files:
- split: train
path: mg/*
- config_name: "mhr"
data_files:
- split: train
path: mhr/*
- config_name: "mi"
data_files:
- split: train
path: mi/*
- config_name: "min"
data_files:
- split: train
path: min/*
- config_name: "mk"
data_files:
- split: train
path: mk/*
- config_name: "ml"
data_files:
- split: train
path: ml/*
- config_name: "mn"
data_files:
- split: train
path: mn/*
- config_name: "mni"
data_files:
- split: train
path: mni/*
- config_name: "mnw"
data_files:
- split: train
path: mnw/*
- config_name: "mr"
data_files:
- split: train
path: mr/*
- config_name: "mrj"
data_files:
- split: train
path: mrj/*
- config_name: "ms"
data_files:
- split: train
path: ms/*
- config_name: "mt"
data_files:
- split: train
path: mt/*
- config_name: "mwl"
data_files:
- split: train
path: mwl/*
- config_name: "my"
data_files:
- split: train
path: my/*
- config_name: "myv"
data_files:
- split: train
path: myv/*
- config_name: "mzn"
data_files:
- split: train
path: mzn/*
- config_name: "nah"
data_files:
- split: train
path: nah/*
- config_name: "nap"
data_files:
- split: train
path: nap/*
- config_name: "nds"
data_files:
- split: train
path: nds/*
- config_name: "nds-nl"
data_files:
- split: train
path: nds-nl/*
- config_name: "ne"
data_files:
- split: train
path: ne/*
- config_name: "new"
data_files:
- split: train
path: new/*
- config_name: "nia"
data_files:
- split: train
path: nia/*
- config_name: "nl"
data_files:
- split: train
path: nl/*
- config_name: "nn"
data_files:
- split: train
path: nn/*
- config_name: "no"
data_files:
- split: train
path: no/*
- config_name: "nov"
data_files:
- split: train
path: nov/*
- config_name: "nqo"
data_files:
- split: train
path: nqo/*
- config_name: "nrm"
data_files:
- split: train
path: nrm/*
- config_name: "nso"
data_files:
- split: train
path: nso/*
- config_name: "nv"
data_files:
- split: train
path: nv/*
- config_name: "ny"
data_files:
- split: train
path: ny/*
- config_name: "oc"
data_files:
- split: train
path: oc/*
- config_name: "olo"
data_files:
- split: train
path: olo/*
- config_name: "om"
data_files:
- split: train
path: om/*
- config_name: "or"
data_files:
- split: train
path: or/*
- config_name: "os"
data_files:
- split: train
path: os/*
- config_name: "pa"
data_files:
- split: train
path: pa/*
- config_name: "pag"
data_files:
- split: train
path: pag/*
- config_name: "pam"
data_files:
- split: train
path: pam/*
- config_name: "pap"
data_files:
- split: train
path: pap/*
- config_name: "pcd"
data_files:
- split: train
path: pcd/*
- config_name: "pcm"
data_files:
- split: train
path: pcm/*
- config_name: "pdc"
data_files:
- split: train
path: pdc/*
- config_name: "pfl"
data_files:
- split: train
path: pfl/*
- config_name: "pi"
data_files:
- split: train
path: pi/*
- config_name: "pih"
data_files:
- split: train
path: pih/*
- config_name: "pl"
data_files:
- split: train
path: pl/*
- config_name: "pms"
data_files:
- split: train
path: pms/*
- config_name: "pnb"
data_files:
- split: train
path: pnb/*
- config_name: "pnt"
data_files:
- split: train
path: pnt/*
- config_name: "ps"
data_files:
- split: train
path: ps/*
- config_name: "pt"
data_files:
- split: train
path: pt/*
- config_name: "pwn"
data_files:
- split: train
path: pwn/*
- config_name: "qu"
data_files:
- split: train
path: qu/*
- config_name: "rm"
data_files:
- split: train
path: rm/*
- config_name: "rmy"
data_files:
- split: train
path: rmy/*
- config_name: "rn"
data_files:
- split: train
path: rn/*
- config_name: "ro"
data_files:
- split: train
path: ro/*
- config_name: "roa-rup"
data_files:
- split: train
path: roa-rup/*
- config_name: "roa-tara"
data_files:
- split: train
path: roa-tara/*
- config_name: "ru"
data_files:
- split: train
path: ru/*
- config_name: "rue"
data_files:
- split: train
path: rue/*
- config_name: "rw"
data_files:
- split: train
path: rw/*
- config_name: "sa"
data_files:
- split: train
path: sa/*
- config_name: "sah"
data_files:
- split: train
path: sah/*
- config_name: "sat"
data_files:
- split: train
path: sat/*
- config_name: "sc"
data_files:
- split: train
path: sc/*
- config_name: "scn"
data_files:
- split: train
path: scn/*
- config_name: "sco"
data_files:
- split: train
path: sco/*
- config_name: "sd"
data_files:
- split: train
path: sd/*
- config_name: "se"
data_files:
- split: train
path: se/*
- config_name: "sg"
data_files:
- split: train
path: sg/*
- config_name: "sh"
data_files:
- split: train
path: sh/*
- config_name: "shi"
data_files:
- split: train
path: shi/*
- config_name: "shn"
data_files:
- split: train
path: shn/*
- config_name: "si"
data_files:
- split: train
path: si/*
- config_name: "simple"
data_files:
- split: train
path: simple/*
- config_name: "sk"
data_files:
- split: train
path: sk/*
- config_name: "skr"
data_files:
- split: train
path: skr/*
- config_name: "sl"
data_files:
- split: train
path: sl/*
- config_name: "sm"
data_files:
- split: train
path: sm/*
- config_name: "smn"
data_files:
- split: train
path: smn/*
- config_name: "sn"
data_files:
- split: train
path: sn/*
- config_name: "so"
data_files:
- split: train
path: so/*
- config_name: "sq"
data_files:
- split: train
path: sq/*
- config_name: "sr"
data_files:
- split: train
path: sr/*
- config_name: "srn"
data_files:
- split: train
path: srn/*
- config_name: "ss"
data_files:
- split: train
path: ss/*
- config_name: "st"
data_files:
- split: train
path: st/*
- config_name: "stq"
data_files:
- split: train
path: stq/*
- config_name: "su"
data_files:
- split: train
path: su/*
- config_name: "sv"
data_files:
- split: train
path: sv/*
- config_name: "sw"
data_files:
- split: train
path: sw/*
- config_name: "szl"
data_files:
- split: train
path: szl/*
- config_name: "szy"
data_files:
- split: train
path: szy/*
- config_name: "ta"
data_files:
- split: train
path: ta/*
- config_name: "tay"
data_files:
- split: train
path: tay/*
- config_name: "tcy"
data_files:
- split: train
path: tcy/*
- config_name: "te"
data_files:
- split: train
path: te/*
- config_name: "tet"
data_files:
- split: train
path: tet/*
- config_name: "tg"
data_files:
- split: train
path: tg/*
- config_name: "th"
data_files:
- split: train
path: th/*
- config_name: "ti"
data_files:
- split: train
path: ti/*
- config_name: "tk"
data_files:
- split: train
path: tk/*
- config_name: "tl"
data_files:
- split: train
path: tl/*
- config_name: "tly"
data_files:
- split: train
path: tly/*
- config_name: "tn"
data_files:
- split: train
path: tn/*
- config_name: "to"
data_files:
- split: train
path: to/*
- config_name: "tpi"
data_files:
- split: train
path: tpi/*
- config_name: "tr"
data_files:
- split: train
path: tr/*
- config_name: "trv"
data_files:
- split: train
path: trv/*
- config_name: "ts"
data_files:
- split: train
path: ts/*
- config_name: "tt"
data_files:
- split: train
path: tt/*
- config_name: "tum"
data_files:
- split: train
path: tum/*
- config_name: "tw"
data_files:
- split: train
path: tw/*
- config_name: "ty"
data_files:
- split: train
path: ty/*
- config_name: "tyv"
data_files:
- split: train
path: tyv/*
- config_name: "udm"
data_files:
- split: train
path: udm/*
- config_name: "ug"
data_files:
- split: train
path: ug/*
- config_name: "uk"
data_files:
- split: train
path: uk/*
- config_name: "ur"
data_files:
- split: train
path: ur/*
- config_name: "uz"
data_files:
- split: train
path: uz/*
- config_name: "ve"
data_files:
- split: train
path: ve/*
- config_name: "vec"
data_files:
- split: train
path: vec/*
- config_name: "vep"
data_files:
- split: train
path: vep/*
- config_name: "vi"
data_files:
- split: train
path: vi/*
- config_name: "vls"
data_files:
- split: train
path: vls/*
- config_name: "vo"
data_files:
- split: train
path: vo/*
- config_name: "wa"
data_files:
- split: train
path: wa/*
- config_name: "war"
data_files:
- split: train
path: war/*
- config_name: "wo"
data_files:
- split: train
path: wo/*
- config_name: "wuu"
data_files:
- split: train
path: wuu/*
- config_name: "xal"
data_files:
- split: train
path: xal/*
- config_name: "xh"
data_files:
- split: train
path: xh/*
- config_name: "xmf"
data_files:
- split: train
path: xmf/*
- config_name: "yi"
data_files:
- split: train
path: yi/*
- config_name: "yo"
data_files:
- split: train
path: yo/*
- config_name: "za"
data_files:
- split: train
path: za/*
- config_name: "zea"
data_files:
- split: train
path: zea/*
- config_name: "zh"
data_files:
- split: train
path: zh/*
- config_name: "zh-classical"
data_files:
- split: train
path: zh-classical/*
- config_name: "zh-min-nan"
data_files:
- split: train
path: zh-min-nan/*
- config_name: "zh-yue"
data_files:
- split: train
path: zh-yue/*
- config_name: "zu"
data_files:
- split: train
path: zu/*
---
# Multilingual Embeddings for Wikipedia in 300+ Languages (int8 & binary embeddings)
This dataset contains the [wikimedia/wikipedia](https://huggingface.co/datasets/wikimedia/wikipedia) dataset dump from 2023-11-01 from Wikipedia in all 300+ languages. The embeddings are provided as **int8** and **ubinary** that allow quick search and reduction of your vector index size up to 32. For more details, see [Cohere int8 & binary Embeddings](https://txt.cohere.com/int8-binary-embeddings/)
The individual articles have been chunked and embedded with the state-of-the-art multilingual [Cohere Embed V3 embedding model](https://txt.cohere.com/introducing-embed-v3/). This enables an easy way to semantically search across all of Wikipedia or to use it as a knowledge source for your RAG application. In total is it close to 250M paragraphs / embeddings.
You can also use the model to perform cross-lingual search: Enter your search query in any language and get the most relevant results back.
## Loading the dataset
### Loading the document embeddings
You can either load the dataset like this:
```python
from datasets import load_dataset
lang = "simple" #Use the Simple English Wikipedia subset
docs = load_dataset("Cohere/wikipedia-2023-11-embed-multilingual-v3-int8-binary", lang, split="train")
```
Or you can also stream it without downloading it before:
```python
from datasets import load_dataset
lang = "simple" #Use the Simple English Wikipedia subset
docs = load_dataset("Cohere/wikipedia-2023-11-embed-multilingual-v3-int8-binary", lang, split="train", streaming=True)
for doc in docs:
doc_id = doc['_id']
title = doc['title']
text = doc['text']
emb_int8 = doc['emb_int8'] #Int8 embeddings
emb_ubinary = doc['emb_ubinary'] #(unsigned) binary embeddings
```
Note, depending on the language, the download can be quite large.
## Search - int8 embeddings
int8 embeddings keep 99.7-100% of the search quality, while reducing your vector index 4x and speeding-up vector search up to 30%.
Below example using [usearch](https://github.com/unum-cloud/usearch) to search on int8 embeddings.
```python
from datasets import load_dataset
import numpy as np
import cohere
from usearch.index import Index
co = cohere.Client("<<YOUR_API_KEY>>") # Add your cohere API key from www.cohere.com
lang = "simple"
#Load at max 1000 chunks + embeddings
max_docs = 1000
docs_stream = load_dataset(f"Cohere/wikipedia-2023-11-embed-multilingual-v3-int8-binary", lang, split="train", streaming=True)
docs = []
doc_embeddings = []
for doc in docs_stream:
docs.append(doc)
doc_embeddings.append(doc['emb_int8'])
if len(docs) >= max_docs:
break
doc_embeddings = np.asarray(doc_embeddings, dtype='int8')
print(doc_embeddings.shape, doc_embeddings.dtype)
#Create the usearch HNSW index on the int8 embeddings
num_dim = 1024
index = Index(ndim=1024, metric='ip', dtype='i8')
index.add(np.arange(len(doc_embeddings)), doc_embeddings)
#Searching
query = 'Who was Alan Turing'
query_emb = co.embed(texts=[query], model="embed-multilingual-v3.0", input_type="search_query", embedding_types=["int8"]).embeddings
query_emb = np.asarray(query_emb.int8, dtype='int8')
#Search on the index and get top-10 results
matches = index.search(query_emb, 10)
# Print results
for match in matches:
doc_id = match.key
print(docs[doc_id]['title'])
print(docs[doc_id]['text'])
print("----")
```
## Search - Binary embeddings
Binary embeddings keep 95% of the search quality, while reducing your vector index 32x and speeding-up vector search up to 40x.
Below example using [faiss](https://github.com/facebookresearch/faiss) to search on binary embeddings.
```python
#Run: pip install cohere datasets numpy faiss-cpu
from datasets import load_dataset
import numpy as np
import cohere
import faiss
co = cohere.Client("<<YOUR_API_KEY>>") # Add your cohere API key from www.cohere.com
lang = "simple"
#Load at max 1000 chunks + embeddings
max_docs = 1000
docs_stream = load_dataset(f"Cohere/wikipedia-2023-11-embed-multilingual-v3-int8-binary", lang, split="train", streaming=True)
docs = []
doc_embeddings = []
for doc in docs_stream:
docs.append(doc)
doc_embeddings.append(doc['emb_ubinary'])
if len(docs) >= max_docs:
break
doc_embeddings = np.asarray(doc_embeddings, dtype='uint8')
#Create the faiss IndexBinaryFlat index
num_dim = 1024
index = faiss.IndexBinaryFlat(num_dim)
index.add(doc_embeddings)
#Search
def search(index, query, top_k=10):
# Make sure to set input_type="search_query"
query_emb = co.embed(texts=[query], model="embed-multilingual-v3.0", input_type="search_query", embedding_types=["ubinary", "float"]).embeddings
query_emb_bin = np.asarray(query_emb.ubinary, dtype='uint8')
query_emb_float = np.asarray(query_emb.float, dtype="float32")
# Phase I: Search on the index with a binary
hits_scores, hits_doc_ids = index.search(query_emb_bin, k=min(10*top_k, index.ntotal))
#Get the results in a list of hits
hits = [{'doc_id': doc_id.item(), 'score_bin': score_bin} for doc_id, score_bin in zip(hits_doc_ids[0], hits_scores[0])]
# Phase II: Do a re-scoring with the float query embedding
binary_doc_emb = np.asarray([index.reconstruct(hit['doc_id']) for hit in hits])
binary_doc_emb_unpacked = np.unpackbits(binary_doc_emb, axis=-1).astype("int")
binary_doc_emb_unpacked = 2*binary_doc_emb_unpacked-1
scores_cont = (query_emb_float[0] @ binary_doc_emb_unpacked.T)
for idx in range(len(scores_cont)):
hits[idx]['score_cont'] = scores_cont[idx]
#Sort by largest score_cont
hits.sort(key=lambda x: x['score_cont'], reverse=True)
return hits[0:top_k]
query = 'Who was Alan Turing'
hits = search(index, query)
# Print results
print("Query:", query)
for hit in hits:
doc_id = hit['doc_id']
print(docs[doc_id]['title'])
print(docs[doc_id]['text'])
print(docs[doc_id]['url'], "\n")
```
## Overview
The following table contains all language codes together with the total numbers of passages.
| Language | #Docs |
|---|:---:|
| en | 41,488,110 |
| de | 20,772,081 |
| fr | 17,813,768 |
| ru | 13,734,543 |
| es | 12,905,284 |
| it | 10,462,162 |
| ceb | 9,818,657 |
| uk | 6,901,192 |
| ja | 6,626,537 |
| nl | 6,101,353 |
| pl | 5,973,650 |
| pt | 5,637,930 |
| sv | 4,911,480 |
| ca | 4,156,889 |
| ar | 3,691,451 |
| cs | 3,118,524 |
| he | 2,948,882 |
| hu | 2,924,609 |
| vi | 2,835,049 |
| zh | 2,775,260 |
| fi | 2,427,097 |
| id | 2,358,335 |
| no | 2,211,270 |
| sr | 2,154,631 |
| fa | 2,073,154 |
| tr | 1,775,036 |
| ro | 1,770,527 |
| el | 1,599,770 |
| ko | 1,513,291 |
| bg | 1,455,765 |
| hy | 1,386,140 |
| eu | 1,327,579 |
| da | 1,224,982 |
| eo | 1,216,706 |
| war | 1,185,097 |
| sh | 1,139,309 |
| tt | 1,119,546 |
| arz | 1,089,164 |
| gl | 1,056,990 |
| et | 1,054,770 |
| ce | 1,013,217 |
| ast | 1,010,445 |
| sl | 984,855 |
| hr | 910,923 |
| sk | 874,014 |
| ms | 869,579 |
| be | 857,033 |
| th | 839,712 |
| az | 817,631 |
| uz | 811,028 |
| mk | 784,576 |
| lt | 770,069 |
| bn | 767,965 |
| cy | 762,338 |
| ta | 685,975 |
| simple | 646,424 |
| te | 634,778 |
| kk | 627,085 |
| ka | 595,401 |
| hi | 541,822 |
| nn | 530,590 |
| lv | 484,957 |
| af | 461,674 |
| ba | 434,939 |
| ur | 434,269 |
| bs | 396,692 |
| sq | 388,788 |
| ml | 384,795 |
| min | 373,156 |
| la | 340,521 |
| pnb | 335,958 |
| be-x-old | 314,600 |
| kn | 309,085 |
| azb | 294,248 |
| oc | 283,297 |
| zh-min-nan | 278,547 |
| fy | 248,075 |
| my | 241,025 |
| lb | 216,558 |
| ky | 216,344 |
| als | 206,387 |
| mr | 203,479 |
| br | 200,486 |
| pa | 188,091 |
| is | 177,272 |
| mg | 171,947 |
| sw | 171,650 |
| ha | 167,807 |
| tl | 166,907 |
| nds | 166,019 |
| an | 143,163 |
| jv | 142,104 |
| ps | 138,240 |
| ig | 132,250 |
| new | 128,696 |
| tg | 128,237 |
| ga | 125,456 |
| lld | 125,094 |
| su | 124,390 |
| cv | 122,671 |
| ckb | 120,886 |
| si | 119,223 |
| mn | 114,878 |
| lmo | 103,836 |
| io | 101,713 |
| gu | 99,450 |
| vec | 95,072 |
| zh-yue | 89,145 |
| bar | 88,238 |
| sco | 83,906 |
| ne | 83,598 |
| ku | 82,935 |
| hyw | 82,343 |
| pms | 77,834 |
| as | 76,093 |
| km | 74,177 |
| sah | 71,599 |
| li | 69,267 |
| or | 65,510 |
| mt | 64,038 |
| szl | 56,836 |
| yi | 55,375 |
| ht | 55,079 |
| dag | 53,343 |
| sa | 51,735 |
| nv | 49,355 |
| bpy | 47,757 |
| vo | 47,375 |
| ug | 44,764 |
| sat | 43,500 |
| ia | 42,012 |
| bo | 41,438 |
| mwl | 41,273 |
| sd | 40,395 |
| bcl | 39,967 |
| mnw | 39,578 |
| hsb | 39,560 |
| avk | 39,001 |
| scn | 38,359 |
| rm | 37,436 |
| diq | 34,743 |
| vep | 33,654 |
| xmf | 33,238 |
| ban | 32,320 |
| wa | 32,132 |
| ilo | 31,046 |
| nds-nl | 30,918 |
| qu | 30,529 |
| so | 29,936 |
| mhr | 29,619 |
| vls | 29,227 |
| sc | 28,977 |
| fo | 28,809 |
| gd | 28,149 |
| rw | 28,037 |
| gom | 27,792 |
| yo | 27,789 |
| tum | 26,743 |
| wuu | 26,532 |
| frr | 26,010 |
| sn | 25,941 |
| tk | 24,269 |
| blk | 24,194 |
| mzn | 23,837 |
| co | 23,065 |
| szy | 22,854 |
| am | 22,467 |
| shn | 22,432 |
| skr | 21,081 |
| lfn | 20,781 |
| tyv | 20,762 |
| lij | 20,553 |
| ie | 19,994 |
| rue | 19,916 |
| crh | 19,016 |
| gor | 18,146 |
| ary | 17,463 |
| dv | 16,941 |
| lg | 16,751 |
| roa-tara | 16,572 |
| bjn | 16,429 |
| tw | 16,304 |
| bh | 15,938 |
| pam | 15,134 |
| os | 15,096 |
| myv | 15,062 |
| gn | 14,983 |
| lez | 14,152 |
| mai | 13,806 |
| kv | 13,534 |
| pcd | 13,057 |
| zh-classical | 12,791 |
| zea | 12,528 |
| lo | 12,525 |
| gv | 12,074 |
| stq | 11,890 |
| zu | 11,680 |
| smn | 11,672 |
| kw | 11,539 |
| bat-smg | 11,240 |
| hif | 11,215 |
| ext | 10,967 |
| ace | 10,821 |
| trv | 10,546 |
| ami | 10,538 |
| tcy | 10,531 |
| lad | 10,386 |
| alt | 10,256 |
| pap | 10,187 |
| kab | 10,179 |
| fur | 10,148 |
| nap | 10,079 |
| mrj | 9,771 |
| kaa | 9,548 |
| nqo | 9,153 |
| glk | 9,120 |
| pfl | 8,790 |
| fiu-vro | 8,757 |
| nso | 8,635 |
| jbo | 8,577 |
| bxr | 8,549 |
| wo | 8,549 |
| olo | 8,530 |
| map-bms | 8,393 |
| ksh | 8,226 |
| csb | 8,085 |
| av | 7,873 |
| mni | 7,740 |
| udm | 7,730 |
| mi | 7,643 |
| kbp | 7,616 |
| dsb | 7,536 |
| frp | 7,294 |
| om | 7,045 |
| ang | 7,023 |
| hak | 6,866 |
| gur | 6,761 |
| se | 6,733 |
| anp | 6,704 |
| tay | 6,434 |
| mdf | 6,351 |
| gcr | 6,347 |
| koi | 6,300 |
| krc | 6,293 |
| ay | 5,985 |
| cdo | 5,917 |
| nrm | 5,786 |
| xh | 5,756 |
| tn | 5,712 |
| tly | 5,598 |
| shi | 5,179 |
| pcm | 5,076 |
| fat | 4,968 |
| nia | 4,795 |
| dty | 4,728 |
| kbd | 4,667 |
| gpe | 4,289 |
| cbk-zam | 4,224 |
| ff | 4,166 |
| dz | 4,117 |
| guw | 3,982 |
| eml | 3,979 |
| ln | 3,774 |
| inh | 3,768 |
| nah | 3,720 |
| ab | 3,465 |
| ks | 3,255 |
| mad | 3,236 |
| haw | 3,227 |
| gag | 3,076 |
| tet | 3,030 |
| ny | 2,933 |
| pag | 2,727 |
| guc | 2,454 |
| roa-rup | 2,409 |
| jam | 2,387 |
| awa | 2,242 |
| pdc | 2,239 |
| to | 2,165 |
| za | 2,132 |
| st | 2,051 |
| ltg | 2,005 |
| atj | 1,967 |
| nov | 1,916 |
| ss | 1,904 |
| pwn | 1,881 |
| ee | 1,819 |
| sm | 1,659 |
| ts | 1,645 |
| gan | 1,626 |
| xal | 1,619 |
| kcg | 1,555 |
| cu | 1,477 |
| srn | 1,395 |
| got | 1,280 |
| fon | 1,247 |
| din | 1,214 |
| arc | 1,167 |
| fj | 1,164 |
| rmy | 1,113 |
| ady | 1,040 |
| rn | 1,033 |
| bm | 1,017 |
| tpi | 957 |
| ve | 919 |
| ki | 798 |
| pnt | 796 |
| chr | 788 |
| kl | 770 |
| lbe | 766 |
| bi | 718 |
| ti | 706 |
| kg | 609 |
| pih | 606 |
| ch | 513 |
| bug | 429 |emb
| ty | 297 |
| ik | 275 |
| iu | 263 |
| pi | 260 |
| sg | 204 |
| chy | 57 |
| cr | 41 |
| Total | 247,154,006 |
提供机构：
Cohere
原始信息汇总
数据集概述
本数据集包含多个配置，每个配置对应不同语言的数据文件。所有数据文件均为训练集（split: train），路径格式统一为语言代码后跟通配符（例如 ab/*）。以下是各配置的详细信息：
配置列表
-
config_name: ab
- data_files:
- split: train
- path: ab/*
- data_files:
-
config_name: ace
- data_files:
- split: train
- path: ace/*
- data_files:
-
config_name: ady
- data_files:
- split: train
- path: ady/*
- data_files:
-
config_name: af
- data_files:
- split: train
- path: af/*
- data_files:
-
config_name: als
- data_files:
- split: train
- path: als/*
- data_files:
-
config_name: alt
- data_files:
- split: train
- path: alt/*
- data_files:
-
config_name: am
- data_files:
- split: train
- path: am/*
- data_files:
-
config_name: ami
- data_files:
- split: train
- path: ami/*
- data_files:
-
config_name: an
- data_files:
- split: train
- path: an/*
- data_files:
-
config_name: ang
- data_files:
- split: train
- path: ang/*
- data_files:
-
config_name: anp
- data_files:
- split: train
- path: anp/*
- data_files:
-
config_name: ar
- data_files:
- split: train
- path: ar/*
- data_files:
-
config_name: arc
- data_files:
- split: train
- path: arc/*
- data_files:
-
config_name: ary
- data_files:
- split: train
- path: ary/*
- data_files:
-
config_name: arz
- data_files:
- split: train
- path: arz/*
- data_files:
-
config_name: as
- data_files:
- split: train
- path: as/*
- data_files:
-
config_name: ast
- data_files:
- split: train
- path: ast/*
- data_files:
-
config_name: atj
- data_files:
- split: train
- path: atj/*
- data_files:
-
config_name: av
- data_files:
- split: train
- path: av/*
- data_files:
-
config_name: avk
- data_files:
- split: train
- path: avk/*
- data_files:
-
config_name: awa
- data_files:
- split: train
- path: awa/*
- data_files:
-
config_name: ay
- data_files:
- split: train
- path: ay/*
- data_files:
-
config_name: az
- data_files:
- split: train
- path: az/*
- data_files:
-
config_name: azb
- data_files:
- split: train
- path: azb/*
- data_files:
-
config_name: ba
- data_files:
- split: train
- path: ba/*
- data_files:
-
config_name: ban
- data_files:
- split: train
- path: ban/*
- data_files:
-
config_name: bar
- data_files:
- split: train
- path: bar/*
- data_files:
-
config_name: bat-smg
- data_files:
- split: train
- path: bat-smg/*
- data_files:
-
config_name: bcl
- data_files:
- split: train
- path: bcl/*
- data_files:
-
config_name: be
- data_files:
- split: train
- path: be/*
- data_files:
-
config_name: be-x-old
- data_files:
- split: train
- path: be-x-old/*
- data_files:
-
config_name: bg
- data_files:
- split: train
- path: bg/*
- data_files:
-
config_name: bh
- data_files:
- split: train
- path: bh/*
- data_files:
-
config_name: bi
- data_files:
- split: train
- path: bi/*
- data_files:
-
config_name: bjn
- data_files:
- split: train
- path: bjn/*
- data_files:
-
config_name: blk
- data_files:
- split: train
- path: blk/*
- data_files:
-
config_name: bm
- data_files:
- split: train
- path: bm/*
- data_files:
-
config_name: bn
- data_files:
- split: train
- path: bn/*
- data_files:
-
config_name: bo
- data_files:
- split: train
- path: bo/*
- data_files:
-
config_name: bpy
- data_files:
- split: train
- path: bpy/*
- data_files:
-
config_name: br
- data_files:
- split: train
- path: br/*
- data_files:
-
config_name: bs
- data_files:
- split: train
- path: bs/*
- data_files:
-
config_name: bug
- data_files:
- split: train
- path: bug/*
- data_files:
-
config_name: bxr
- data_files:
- split: train
- path: bxr/*
- data_files:
-
config_name: ca
- data_files:
- split: train
- path: ca/*
- data_files:
-
config_name: cbk-zam
- data_files:
- split: train
- path: cbk-zam/*
- data_files:
-
config_name: cdo
- data_files:
- split: train
- path: cdo/*
- data_files:
-
config_name: ce
- data_files:
- split: train
- path: ce/*
- data_files:
-
config_name: ceb
- data_files:
- split: train
- path: ceb/*
- data_files:
-
config_name: ch
- data_files:
- split: train
- path: ch/*
- data_files:
-
config_name: chr
- data_files:
- split: train
- path: chr/*
- data_files:
-
config_name: chy
- data_files:
- split: train
- path: chy/*
- data_files:
-
config_name: ckb
- data_files:
- split: train
- path: ckb/*
- data_files:
-
config_name: co
- data_files:
- split: train
- path: co/*
- data_files:
-
config_name: cr
- data_files:
- split: train
- path: cr/*
- data_files:
-
config_name: crh
- data_files:
- split: train
- path: crh/*
- data_files:
-
config_name: cs
- data_files:
- split: train
- path: cs/*
- data_files:
-
config_name: csb
- data_files:
- split: train
- path: csb/*
- data_files:
-
config_name: cu
- data_files:
- split: train
- path: cu/*
- data_files:
-
config_name: cv
- data_files:
- split: train
- path: cv/*
- data_files:
-
config_name: cy
- data_files:
- split: train
- path: cy/*
- data_files:
-
config_name: da
- data_files:
- split: train
- path: da/*
- data_files:
-
config_name: dag
- data_files:
- split: train
- path: dag/*
- data_files:
-
config_name: de
- data_files:
- split: train
- path: de/*
- data_files:
-
config_name: din
- data_files:
- split: train
- path: din/*
- data_files:
-
config_name: diq
- data_files:
- split: train
- path: diq/*
- data_files:
-
config_name: dsb
- data_files:
- split: train
- path: dsb/*
- data_files:
-
config_name: dty
- data_files:
- split: train
- path: dty/*
- data_files:
-
config_name: dv
- data_files:
- split: train
- path: dv/*
- data_files:
-
config_name: dz
- data_files:
- split: train
- path: dz/*
- data_files:
-
config_name: ee
- data_files:
- split: train
- path: ee/*
- data_files:
-
config_name: el
- data_files:
- split: train
- path: el/*
- data_files:
-
config_name: eml
- data_files:
- split: train
- path: eml/*
- data_files:
-
config_name: en
- data_files:
- split: train
- path: en/*
- data_files:
-
config_name: eo
- data_files:
- split: train
- path: eo/*
- data_files:
-
config_name: es
- data_files:
- split: train
- path: es/*
- data_files:
-
config_name: et
- data_files:
- split: train
- path: et/*
- data_files:
-
config_name: eu
- data_files:
- split: train
- path: eu/*
- data_files:
-
config_name: ext
- data_files:
- split: train
- path: ext/*
- data_files:
-
config_name: fa
- data_files:
- split: train
- path: fa/*
- data_files:
-
config_name: fat
- data_files:
- split: train
- path: fat/*
- data_files:
-
config_name: ff
- data_files:
- split: train
- path: ff/*
- data_files:
-
config_name: fi
- data_files:
- split: train
- path: fi/*
- data_files:
-
config_name: fiu-vro
- data_files:
- split: train
- path: fiu-vro/*
- data_files:
-
config_name: fj
- data_files:
- split: train
- path: fj/*
- data_files:
-
config_name: fo
- data_files:
- split: train
- path: fo/*
- data_files:
-
config_name: fon
- data_files:
- split: train
- path: fon/*
- data_files:
-
config_name: fr
- data_files:
- split: train
- path: fr/*
- data_files:
-
config_name: frp
- data_files:
- split: train
- path: frp/*
- data_files:
-
config_name: frr
- data_files:
- split: train
- path: frr/*
- data_files:
-
config_name: fur
- data_files:
- split: train
- path: fur/*
- data_files:
-
config_name: fy
- data_files:
- split: train
- path: fy/*
- data_files:
-
config_name: ga
- data_files:
- split: train
- path: ga/*
- data_files:
-
config_name: gag
- data_files:
- split: train
- path: gag/*
- data_files:
-
config_name: gan
- data_files:
- split: train
- path: gan/*
- data_files:
-
config_name: gcr
- data_files:
- split: train
- path: gcr/*
- data_files:
-
config_name: gd
- data_files:
- split: train
- path: gd/*
- data_files:
-
config_name: gl
- data_files:
- split: train
- path: gl/*
- data_files:
-
config_name: glk
- data_files:
- split: train
- path: glk/*
- data_files:
-
config_name: gn
- data_files:
- split: train
- path: gn/*
- data_files:
-
config_name: gom
- data_files:
- split: train
- path: gom/*
- data_files:
-
config_name: gor
- data_files:
- split: train
- path: gor/*
- data_files:
-
config_name: got
- data_files:
- split: train
- path: got/*
- data_files:
-
config_name: gpe
- data_files:
- split: train
- path: gpe/*
- data_files:
-
config_name: gu
- data_files:
- split: train
- path: gu/*
- data_files:
-
config_name: guc
- data_files:
- split: train
- path: guc/*
- data_files:
-
config_name: gur
- data_files:
- split: train
- path: gur/*
- data_files:
-
config_name: guw
- data_files:
- split: train
- path: guw/*
- data_files:
-
config_name: gv
- data_files:
- split: train
- path: gv/*
- data_files:
-
config_name: ha
- data_files:
- split: train
- path: ha/*
- data_files:
-
config_name: hak
- data_files:
- split: train
- path: hak/*
- data_files:
-
config_name: haw
- data_files:
- split: train
- path: haw/*
- data_files:
-
config_name: he
- data_files:
- split: train
- path: he/*
- data_files:
-
config_name: hi
- data_files:
- split: train
- path: hi/*
- data_files:
-
config_name: hif
- data_files:
- split: train
- path: hif/*
- data_files:
-
config_name: hr
- data_files:
- split: train
- path: hr/*
- data_files:
-
config_name: hsb
- data_files:
- split: train
- path: hsb/*
- data_files:
-
config_name: ht
- data_files:
- split: train
- path: ht/*
- data_files:
-
config_name: hu
- data_files:
- split: train
- path: hu/*
- data_files:
-
config_name: hy
- data_files:
- split: train
- path: hy/*
- data_files:
-
config_name: hyw
- data_files:
- split: train
- path: hyw/*
- data_files:
-
config_name: ia
- data_files:
- split: train
- path: ia/*
- data_files:
-
config_name: id
- **
