mmcarpi/corpus-carolina-dedup

--- dataset_info: features: - name: carolina_title dtype: large_string - name: source_title dtype: large_string - name: mime_type dtype: large_string - name: source_typology dtype: large_string - name: carolina_typology dtype: large_string - name: tokens dtype: int64 - name: originality dtype: int64 - name: text dtype: large_string - name: __INDEX__ dtype: int64 - name: __CLUSTER__ dtype: int64 - name: __duplicate__ dtype: bool splits: - name: test num_bytes: 35336036 num_examples: 23503 - name: validation num_bytes: 35336036 num_examples: 23503 - name: train num_bytes: 2227000806 num_examples: 1476497 - name: tokenizer_train num_bytes: 353429122 num_examples: 234652 download_size: 1589190245 dataset_size: 2651102000 configs: - config_name: default data_files: - split: test path: data/test-* - split: validation path: data/validation-* - split: train path: data/train-* - split: tokenizer_train path: data/tokenizer_train-* ---