Ba2han/tokenized-corpus-0903

---
dataset_info:
features:
- name: input_ids
sequence: int32
configs:
- config_name: default
data_files:
- split: train
path: data/train-*
---
# Tokenized Corpus Statistics

| Shard | Total Tokens | Avg Tokens | Median Tokens |
|-------|--------------|------------|---------------|
| train-00000-of-00081 | 65,149,125 | 260.60 | 215.0 |
| train-00001-of-00081 | 65,155,277 | 260.62 | 214.0 |
| train-00002-of-00081 | 65,176,189 | 260.70 | 215.0 |
| train-00003-of-00081 | 64,925,319 | 259.70 | 214.0 |
| train-00004-of-00081 | 65,202,114 | 260.81 | 215.0 |
| train-00005-of-00081 | 65,195,301 | 260.78 | 215.0 |
| train-00006-of-00081 | 65,124,294 | 260.50 | 215.0 |
| train-00007-of-00081 | 65,212,450 | 260.85 | 215.0 |
| train-00008-of-00081 | 65,085,854 | 260.34 | 216.0 |
| train-00009-of-00081 | 65,262,915 | 261.05 | 215.0 |
| train-00010-of-00081 | 65,160,847 | 260.64 | 214.0 |
| train-00011-of-00081 | 65,083,349 | 260.33 | 214.0 |
| train-00012-of-00081 | 65,180,783 | 260.72 | 214.0 |
| train-00013-of-00081 | 65,145,549 | 260.58 | 215.0 |
| train-00014-of-00081 | 65,143,472 | 260.57 | 214.0 |
| train-00015-of-00081 | 65,236,842 | 260.95 | 215.0 |
| train-00016-of-00081 | 64,994,889 | 259.98 | 214.0 |
| train-00017-of-00081 | 65,208,457 | 260.83 | 215.0 |
| train-00018-of-00081 | 65,131,612 | 260.53 | 215.0 |
| train-00019-of-00081 | 65,290,738 | 261.16 | 215.0 |
| train-00020-of-00081 | 65,229,252 | 260.92 | 216.0 |
| train-00021-of-00081 | 65,102,800 | 260.41 | 215.0 |
| train-00022-of-00081 | 65,087,885 | 260.35 | 214.0 |
| train-00023-of-00081 | 65,128,525 | 260.51 | 214.0 |
| train-00024-of-00081 | 65,117,461 | 260.47 | 214.0 |
| train-00025-of-00081 | 65,013,884 | 260.06 | 214.0 |
| train-00026-of-00081 | 65,262,375 | 261.05 | 215.0 |
| train-00027-of-00081 | 65,108,885 | 260.44 | 215.0 |
| train-00028-of-00081 | 65,155,292 | 260.62 | 214.0 |
| train-00029-of-00081 | 65,176,150 | 260.70 | 215.0 |
| train-00030-of-00081 | 65,408,725 | 261.63 | 216.0 |
| train-00031-of-00081 | 65,157,271 | 260.63 | 214.0 |
| train-00032-of-00081 | 65,055,801 | 260.22 | 213.0 |
| train-00033-of-00081 | 65,195,690 | 260.78 | 214.0 |
| train-00034-of-00081 | 65,259,397 | 261.04 | 215.0 |
| train-00035-of-00081 | 65,161,691 | 260.65 | 214.0 |
| train-00036-of-00081 | 65,092,191 | 260.37 | 214.0 |
| train-00037-of-00081 | 65,052,808 | 260.21 | 214.0 |
| train-00038-of-00081 | 65,279,508 | 261.12 | 216.0 |
| train-00039-of-00081 | 65,150,109 | 260.60 | 215.0 |
| train-00040-of-00081 | 65,050,940 | 260.20 | 214.0 |
| train-00041-of-00081 | 65,300,486 | 261.20 | 215.0 |
| train-00042-of-00081 | 65,195,632 | 260.78 | 214.0 |
| train-00043-of-00081 | 65,111,425 | 260.45 | 214.0 |
| train-00044-of-00081 | 65,276,592 | 261.11 | 215.0 |
| train-00045-of-00081 | 65,093,170 | 260.37 | 214.0 |
| train-00046-of-00081 | 65,135,055 | 260.54 | 215.0 |
| train-00047-of-00081 | 65,111,389 | 260.45 | 215.0 |
| train-00048-of-00081 | 65,136,576 | 260.55 | 215.0 |
| train-00049-of-00081 | 65,215,493 | 260.86 | 215.0 |
| train-00050-of-00081 | 65,063,484 | 260.25 | 214.0 |
| train-00051-of-00081 | 65,165,732 | 260.66 | 215.0 |
| train-00052-of-00081 | 65,060,103 | 260.24 | 214.0 |
| train-00053-of-00081 | 65,152,927 | 260.61 | 215.0 |
| train-00054-of-00081 | 65,229,197 | 260.92 | 215.0 |
| train-00055-of-00081 | 65,034,655 | 260.14 | 214.0 |
| train-00056-of-00081 | 65,156,166 | 260.62 | 215.0 |
| train-00057-of-00081 | 65,159,496 | 260.64 | 214.0 |
| train-00058-of-00081 | 65,421,015 | 261.68 | 216.0 |
| train-00059-of-00081 | 65,183,688 | 260.73 | 214.0 |
| train-00060-of-00081 | 65,230,891 | 260.92 | 215.0 |
| train-00061-of-00081 | 65,109,420 | 260.44 | 214.0 |
| train-00062-of-00081 | 64,867,204 | 259.47 | 214.0 |
| train-00063-of-00081 | 65,228,736 | 260.91 | 215.0 |
| train-00064-of-00081 | 65,070,913 | 260.28 | 214.0 |
| train-00065-of-00081 | 65,109,111 | 260.44 | 215.0 |
| train-00066-of-00081 | 64,889,563 | 259.56 | 214.0 |
| train-00067-of-00081 | 65,025,475 | 260.10 | 214.0 |
| train-00068-of-00081 | 65,070,167 | 260.28 | 215.0 |
| train-00069-of-00081 | 65,156,197 | 260.62 | 214.0 |
| train-00070-of-00081 | 65,097,990 | 260.39 | 215.0 |
| train-00071-of-00081 | 65,198,912 | 260.80 | 215.0 |
| train-00072-of-00081 | 65,092,415 | 260.37 | 214.0 |
| train-00073-of-00081 | 65,194,841 | 260.78 | 214.0 |
| train-00074-of-00081 | 65,078,030 | 260.31 | 214.0 |
| train-00075-of-00081 | 65,185,924 | 260.74 | 215.0 |
| train-00076-of-00081 | 65,027,719 | 260.11 | 214.0 |
| train-00077-of-00081 | 65,351,327 | 261.41 | 216.0 |
| train-00078-of-00081 | 65,025,952 | 260.10 | 214.0 |
| train-00079-of-00081 | 65,190,932 | 260.76 | 215.0 |
| train-00080-of-00081 | 5,133,157 | 261.03 | 216 |