clembench-playpen/lm-pragmatics

---
---
task_categories:
- question-answering
language:
- en
pretty_name: LM-Pragmatics
size_categories:
- 1K<n<10K
configs:
- config_name: coherence
data_files:
- split: seed_0
path: CoherenceInference_prompts_seed0_examples0.csv
- split: seed_1
path: CoherenceInference_prompts_seed1_examples0.csv
- split: seed_2
path: CoherenceInference_prompts_seed2_examples0.csv
- split: seed_3
path: CoherenceInference_prompts_seed3_examples0.csv
- split: seed_4
path: CoherenceInference_prompts_seed4_examples0.csv
- config_name: deceits
data_files:
- split: seed_0
path: Deceits_prompts_seed0_examples0.csv
- split: seed_1
path: Deceits_prompts_seed1_examples0.csv
- split: seed_2
path: Deceits_prompts_seed2_examples0.csv
- split: seed_3
path: Deceits_prompts_seed3_examples0.csv
- split: seed_4
path: Deceits_prompts_seed4_examples0.csv
- config_name: deceits_no_story
data_files:
- split: seed_0
path: Deceits_no-story_prompts_seed0_examples0.csv
- split: seed_1
path: Deceits_no-story_prompts_seed1_examples0.csv
- split: seed_2
path: Deceits_no-story_prompts_seed2_examples0.csv
- split: seed_3
path: Deceits_no-story_prompts_seed3_examples0.csv
- split: seed_4
path: Deceits_no-story_prompts_seed4_examples0.csv
- config_name: humour
data_files:
- split: seed_0
path: Humour_prompts_seed0_examples0.csv
- split: seed_1
path: Humour_prompts_seed1_examples0.csv
- split: seed_2
path: Humour_prompts_seed2_examples0.csv
- split: seed_3
path: Humour_prompts_seed3_examples0.csv
- split: seed_4
path: Humour_prompts_seed4_examples0.csv
- config_name: indirect_speech
data_files:
- split: seed_0
path: IndirectSpeech_prompts_seed0_examples0.csv
- split: seed_1
path: IndirectSpeech_prompts_seed1_examples0.csv
- split: seed_2
path: IndirectSpeech_prompts_seed2_examples0.csv
- split: seed_3
path: IndirectSpeech_prompts_seed3_examples0.csv
- split: seed_4
path: IndirectSpeech_prompts_seed4_examples0.csv
- config_name: indirect_speech_no_story
data_files:
- split: seed_0
path: IndirectSpeech_no-story_prompts_seed0_examples0.csv
- split: seed_1
path: IndirectSpeech_no-story_prompts_seed1_examples0.csv
- split: seed_2
path: IndirectSpeech_no-story_prompts_seed2_examples0.csv
- split: seed_3
path: IndirectSpeech_no-story_prompts_seed3_examples0.csv
- split: seed_4
path: IndirectSpeech_no-story_prompts_seed4_examples0.csv
- config_name: irony
data_files:
- split: seed_0
path: Irony_prompts_seed0_examples0.csv
- split: seed_1
path: Irony_prompts_seed1_examples0.csv
- split: seed_2
path: Irony_prompts_seed2_examples0.csv
- split: seed_3
path: Irony_prompts_seed3_examples0.csv
- split: seed_4
path: Irony_prompts_seed4_examples0.csv
- config_name: irony_no_story
data_files:
- split: seed_0
path: Irony_no-story_prompts_seed0_examples0.csv
- split: seed_1
path: Irony_no-story_prompts_seed1_examples0.csv
- split: seed_2
path: Irony_no-story_prompts_seed2_examples0.csv
- split: seed_3
path: Irony_no-story_prompts_seed3_examples0.csv
- split: seed_4
path: Irony_no-story_prompts_seed4_examples0.csv
- config_name: maxims
data_files:
- split: seed_0
path: Maxims_prompts_seed0_examples0.csv
- split: seed_1
path: Maxims_prompts_seed1_examples0.csv
- split: seed_2
path: Maxims_prompts_seed2_examples0.csv
- split: seed_3
path: Maxims_prompts_seed3_examples0.csv
- split: seed_4
path: Maxims_prompts_seed4_examples0.csv
- config_name: maxims_no_story
data_files:
- split: seed_0
path: Maxims_no-story_prompts_seed0_examples0.csv
- split: seed_1
path: Maxims_no-story_prompts_seed1_examples0.csv
- split: seed_2
path: Maxims_no-story_prompts_seed2_examples0.csv
- split: seed_3
path: Maxims_no-story_prompts_seed3_examples0.csv
- split: seed_4
path: Maxims_no-story_prompts_seed4_examples0.csv
- config_name: metaphor
data_files:
- split: seed_0
path: Metaphor_prompts_seed0_examples0.csv
- split: seed_1
path: Metaphor_prompts_seed1_examples0.csv
- split: seed_2
path: Metaphor_prompts_seed2_examples0.csv
- split: seed_3
path: Metaphor_prompts_seed3_examples0.csv
- split: seed_4
path: Metaphor_prompts_seed4_examples0.csv
- config_name: metaphor_no_story
data_files:
- split: seed_0
path: Metaphor_no-story_prompts_seed0_examples0.csv
- split: seed_1
path: Metaphor_no-story_prompts_seed1_examples0.csv
- split: seed_2
path: Metaphor_no-story_prompts_seed2_examples0.csv
- split: seed_3
path: Metaphor_no-story_prompts_seed3_examples0.csv
- split: seed_4
path: Metaphor_no-story_prompts_seed4_examples0.csv
---

## Citation
> @misc{hu2023finegrainedcomparisonpragmaticlanguage,
> title={A fine-grained comparison of pragmatic language understanding in humans and language models},
> author={Jennifer Hu and Sammy Floyd and Olessia Jouravlev and Evelina Fedorenko and Edward Gibson},
> year={2023},
> eprint={2212.06801},
> archivePrefix={arXiv},
> primaryClass={cs.CL},
> url={https://arxiv.org/abs/2212.06801},
>}