Advocate512/legalbench

---
license: cc-by-4.0
task_categories:
- text-classification
- question-answering
language:
- en
pretty_name: LegalBench (Staging)
tags:
- legal
- law
- benchmark
configs:
- config_name: abercrombie
data_files:
- split: train
path: data/abercrombie/train.tsv
- split: test
path: data/abercrombie/test.tsv
default: true
- config_name: canada_tax_court_outcomes
data_files:
- split: train
path: data/canada_tax_court_outcomes/train.tsv
- split: test
path: data/canada_tax_court_outcomes/test.tsv
default: false
- config_name: citation_prediction_classification
data_files:
- split: train
path: data/citation_prediction_classification/train.tsv
- split: test
path: data/citation_prediction_classification/test.tsv
default: false
- config_name: citation_prediction_open
data_files:
- split: train
path: data/citation_prediction_open/train.tsv
- split: test
path: data/citation_prediction_open/test.tsv
default: false
- config_name: consumer_contracts_qa
data_files:
- split: train
path: data/consumer_contracts_qa/train.tsv
- split: test
path: data/consumer_contracts_qa/test.tsv
default: false
- config_name: contract_nli_confidentiality_of_agreement
data_files:
- split: train
path: data/contract_nli_confidentiality_of_agreement/train.tsv
- split: test
path: data/contract_nli_confidentiality_of_agreement/test.tsv
default: false
- config_name: contract_nli_explicit_identification
data_files:
- split: train
path: data/contract_nli_explicit_identification/train.tsv
- split: test
path: data/contract_nli_explicit_identification/test.tsv
default: false
- config_name: contract_nli_inclusion_of_verbally_conveyed_information
data_files:
- split: train
path: data/contract_nli_inclusion_of_verbally_conveyed_information/train.tsv
- split: test
path: data/contract_nli_inclusion_of_verbally_conveyed_information/test.tsv
default: false
- config_name: contract_nli_limited_use
data_files:
- split: train
path: data/contract_nli_limited_use/train.tsv
- split: test
path: data/contract_nli_limited_use/test.tsv
default: false
- config_name: contract_nli_no_licensing
data_files:
- split: train
path: data/contract_nli_no_licensing/train.tsv
- split: test
path: data/contract_nli_no_licensing/test.tsv
default: false
- config_name: contract_nli_notice_on_compelled_disclosure
data_files:
- split: train
path: data/contract_nli_notice_on_compelled_disclosure/train.tsv
- split: test
path: data/contract_nli_notice_on_compelled_disclosure/test.tsv
default: false
- config_name: contract_nli_permissible_acquirement_of_similar_information
data_files:
- split: train
path: data/contract_nli_permissible_acquirement_of_similar_information/train.tsv
- split: test
path: data/contract_nli_permissible_acquirement_of_similar_information/test.tsv
default: false
- config_name: contract_nli_permissible_copy
data_files:
- split: train
path: data/contract_nli_permissible_copy/train.tsv
- split: test
path: data/contract_nli_permissible_copy/test.tsv
default: false
- config_name: contract_nli_permissible_development_of_similar_information
data_files:
- split: train
path: data/contract_nli_permissible_development_of_similar_information/train.tsv
- split: test
path: data/contract_nli_permissible_development_of_similar_information/test.tsv
default: false
- config_name: contract_nli_permissible_post-agreement_possession
data_files:
- split: train
path: data/contract_nli_permissible_post-agreement_possession/train.tsv
- split: test
path: data/contract_nli_permissible_post-agreement_possession/test.tsv
default: false
- config_name: contract_nli_return_of_confidential_information
data_files:
- split: train
path: data/contract_nli_return_of_confidential_information/train.tsv
- split: test
path: data/contract_nli_return_of_confidential_information/test.tsv
default: false
- config_name: contract_nli_sharing_with_employees
data_files:
- split: train
path: data/contract_nli_sharing_with_employees/train.tsv
- split: test
path: data/contract_nli_sharing_with_employees/test.tsv
default: false
- config_name: contract_nli_sharing_with_third-parties
data_files:
- split: train
path: data/contract_nli_sharing_with_third-parties/train.tsv
- split: test
path: data/contract_nli_sharing_with_third-parties/test.tsv
default: false
- config_name: contract_nli_survival_of_obligations
data_files:
- split: train
path: data/contract_nli_survival_of_obligations/train.tsv
- split: test
path: data/contract_nli_survival_of_obligations/test.tsv
default: false
- config_name: contract_qa
data_files:
- split: train
path: data/contract_qa/train.tsv
- split: test
path: data/contract_qa/test.tsv
default: false
- config_name: corporate_lobbying
data_files:
- split: train
path: data/corporate_lobbying/train.tsv
- split: test
path: data/corporate_lobbying/test.tsv
default: false
- config_name: cuad_affiliate_license-licensee
data_files:
- split: train
path: data/cuad_affiliate_license-licensee/train.tsv
- split: test
path: data/cuad_affiliate_license-licensee/test.tsv
default: false
- config_name: cuad_affiliate_license-licensor
data_files:
- split: train
path: data/cuad_affiliate_license-licensor/train.tsv
- split: test
path: data/cuad_affiliate_license-licensor/test.tsv
default: false
- config_name: cuad_anti-assignment
data_files:
- split: train
path: data/cuad_anti-assignment/train.tsv
- split: test
path: data/cuad_anti-assignment/test.tsv
default: false
- config_name: cuad_audit_rights
data_files:
- split: train
path: data/cuad_audit_rights/train.tsv
- split: test
path: data/cuad_audit_rights/test.tsv
default: false
- config_name: cuad_cap_on_liability
data_files:
- split: train
path: data/cuad_cap_on_liability/train.tsv
- split: test
path: data/cuad_cap_on_liability/test.tsv
default: false
- config_name: cuad_change_of_control
data_files:
- split: train
path: data/cuad_change_of_control/train.tsv
- split: test
path: data/cuad_change_of_control/test.tsv
default: false
- config_name: cuad_competitive_restriction_exception
data_files:
- split: train
path: data/cuad_competitive_restriction_exception/train.tsv
- split: test
path: data/cuad_competitive_restriction_exception/test.tsv
default: false
- config_name: cuad_covenant_not_to_sue
data_files:
- split: train
path: data/cuad_covenant_not_to_sue/train.tsv
- split: test
path: data/cuad_covenant_not_to_sue/test.tsv
default: false
- config_name: cuad_effective_date
data_files:
- split: train
path: data/cuad_effective_date/train.tsv
- split: test
path: data/cuad_effective_date/test.tsv
default: false
- config_name: cuad_exclusivity
data_files:
- split: train
path: data/cuad_exclusivity/train.tsv
- split: test
path: data/cuad_exclusivity/test.tsv
default: false
- config_name: cuad_expiration_date
data_files:
- split: train
path: data/cuad_expiration_date/train.tsv
- split: test
path: data/cuad_expiration_date/test.tsv
default: false
- config_name: cuad_governing_law
data_files:
- split: train
path: data/cuad_governing_law/train.tsv
- split: test
path: data/cuad_governing_law/test.tsv
default: false
- config_name: cuad_insurance
data_files:
- split: train
path: data/cuad_insurance/train.tsv
- split: test
path: data/cuad_insurance/test.tsv
default: false
- config_name: cuad_ip_ownership_assignment
data_files:
- split: train
path: data/cuad_ip_ownership_assignment/train.tsv
- split: test
path: data/cuad_ip_ownership_assignment/test.tsv
default: false
- config_name: cuad_irrevocable_or_perpetual_license
data_files:
- split: train
path: data/cuad_irrevocable_or_perpetual_license/train.tsv
- split: test
path: data/cuad_irrevocable_or_perpetual_license/test.tsv
default: false
- config_name: cuad_joint_ip_ownership
data_files:
- split: train
path: data/cuad_joint_ip_ownership/train.tsv
- split: test
path: data/cuad_joint_ip_ownership/test.tsv
default: false
- config_name: cuad_license_grant
data_files:
- split: train
path: data/cuad_license_grant/train.tsv
- split: test
path: data/cuad_license_grant/test.tsv
default: false
- config_name: cuad_liquidated_damages
data_files:
- split: train
path: data/cuad_liquidated_damages/train.tsv
- split: test
path: data/cuad_liquidated_damages/test.tsv
default: false
- config_name: cuad_minimum_commitment
data_files:
- split: train
path: data/cuad_minimum_commitment/train.tsv
- split: test
path: data/cuad_minimum_commitment/test.tsv
default: false
- config_name: cuad_most_favored_nation
data_files:
- split: train
path: data/cuad_most_favored_nation/train.tsv
- split: test
path: data/cuad_most_favored_nation/test.tsv
default: false
- config_name: cuad_no-solicit_of_customers
data_files:
- split: train
path: data/cuad_no-solicit_of_customers/train.tsv
- split: test
path: data/cuad_no-solicit_of_customers/test.tsv
default: false
- config_name: cuad_no-solicit_of_employees
data_files:
- split: train
path: data/cuad_no-solicit_of_employees/train.tsv
- split: test
path: data/cuad_no-solicit_of_employees/test.tsv
default: false
- config_name: cuad_non-compete
data_files:
- split: train
path: data/cuad_non-compete/train.tsv
- split: test
path: data/cuad_non-compete/test.tsv
default: false
- config_name: cuad_non-disparagement
data_files:
- split: train
path: data/cuad_non-disparagement/train.tsv
- split: test
path: data/cuad_non-disparagement/test.tsv
default: false
- config_name: cuad_non-transferable_license
data_files:
- split: train
path: data/cuad_non-transferable_license/train.tsv
- split: test
path: data/cuad_non-transferable_license/test.tsv
default: false
- config_name: cuad_notice_period_to_terminate_renewal
data_files:
- split: train
path: data/cuad_notice_period_to_terminate_renewal/train.tsv
- split: test
path: data/cuad_notice_period_to_terminate_renewal/test.tsv
default: false
- config_name: cuad_post-termination_services
data_files:
- split: train
path: data/cuad_post-termination_services/train.tsv
- split: test
path: data/cuad_post-termination_services/test.tsv
default: false
- config_name: cuad_price_restrictions
data_files:
- split: train
path: data/cuad_price_restrictions/train.tsv
- split: test
path: data/cuad_price_restrictions/test.tsv
default: false
- config_name: cuad_renewal_term
data_files:
- split: train
path: data/cuad_renewal_term/train.tsv
- split: test
path: data/cuad_renewal_term/test.tsv
default: false
- config_name: cuad_revenue-profit_sharing
data_files:
- split: train
path: data/cuad_revenue-profit_sharing/train.tsv
- split: test
path: data/cuad_revenue-profit_sharing/test.tsv
default: false
- config_name: cuad_rofr-rofo-rofn
data_files:
- split: train
path: data/cuad_rofr-rofo-rofn/train.tsv
- split: test
path: data/cuad_rofr-rofo-rofn/test.tsv
default: false
- config_name: cuad_source_code_escrow
data_files:
- split: train
path: data/cuad_source_code_escrow/train.tsv
- split: test
path: data/cuad_source_code_escrow/test.tsv
default: false
- config_name: cuad_termination_for_convenience
data_files:
- split: train
path: data/cuad_termination_for_convenience/train.tsv
- split: test
path: data/cuad_termination_for_convenience/test.tsv
default: false
- config_name: cuad_third_party_beneficiary
data_files:
- split: train
path: data/cuad_third_party_beneficiary/train.tsv
- split: test
path: data/cuad_third_party_beneficiary/test.tsv
default: false
- config_name: cuad_uncapped_liability
data_files:
- split: train
path: data/cuad_uncapped_liability/train.tsv
- split: test
path: data/cuad_uncapped_liability/test.tsv
default: false
- config_name: cuad_unlimited-all-you-can-eat-license
data_files:
- split: train
path: data/cuad_unlimited-all-you-can-eat-license/train.tsv
- split: test
path: data/cuad_unlimited-all-you-can-eat-license/test.tsv
default: false
- config_name: cuad_volume_restriction
data_files:
- split: train
path: data/cuad_volume_restriction/train.tsv
- split: test
path: data/cuad_volume_restriction/test.tsv
default: false
- config_name: cuad_warranty_duration
data_files:
- split: train
path: data/cuad_warranty_duration/train.tsv
- split: test
path: data/cuad_warranty_duration/test.tsv
default: false
- config_name: definition_classification
data_files:
- split: train
path: data/definition_classification/train.tsv
- split: test
path: data/definition_classification/test.tsv
default: false
- config_name: definition_extraction
data_files:
- split: train
path: data/definition_extraction/train.tsv
- split: test
path: data/definition_extraction/test.tsv
default: false
- config_name: diversity_1
data_files:
- split: train
path: data/diversity_1/train.tsv
- split: test
path: data/diversity_1/test.tsv
default: false
- config_name: diversity_2
data_files:
- split: train
path: data/diversity_2/train.tsv
- split: test
path: data/diversity_2/test.tsv
default: false
- config_name: diversity_3
data_files:
- split: train
path: data/diversity_3/train.tsv
- split: test
path: data/diversity_3/test.tsv
default: false
- config_name: diversity_4
data_files:
- split: train
path: data/diversity_4/train.tsv
- split: test
path: data/diversity_4/test.tsv
default: false
- config_name: diversity_5
data_files:
- split: train
path: data/diversity_5/train.tsv
- split: test
path: data/diversity_5/test.tsv
default: false
- config_name: diversity_6
data_files:
- split: train
path: data/diversity_6/train.tsv
- split: test
path: data/diversity_6/test.tsv
default: false
- config_name: function_of_decision_section
data_files:
- split: train
path: data/function_of_decision_section/train.tsv
- split: test
path: data/function_of_decision_section/test.tsv
default: false
- config_name: hearsay
data_files:
- split: train
path: data/hearsay/train.tsv
- split: test
path: data/hearsay/test.tsv
default: false
- config_name: insurance_policy_interpretation
data_files:
- split: train
path: data/insurance_policy_interpretation/train.tsv
- split: test
path: data/insurance_policy_interpretation/test.tsv
default: false
- config_name: international_citizenship_questions
data_files:
- split: train
path: data/international_citizenship_questions/train.tsv
- split: test
path: data/international_citizenship_questions/test.tsv
default: false
- config_name: jcrew_blocker
data_files:
- split: train
path: data/jcrew_blocker/train.tsv
- split: test
path: data/jcrew_blocker/test.tsv
default: false
- config_name: learned_hands_benefits
data_files:
- split: train
path: data/learned_hands_benefits/train.tsv
- split: test
path: data/learned_hands_benefits/test.tsv
default: false
- config_name: learned_hands_business
data_files:
- split: train
path: data/learned_hands_business/train.tsv
- split: test
path: data/learned_hands_business/test.tsv
default: false
- config_name: learned_hands_consumer
data_files:
- split: train
path: data/learned_hands_consumer/train.tsv
- split: test
path: data/learned_hands_consumer/test.tsv
default: false
- config_name: learned_hands_courts
data_files:
- split: train
path: data/learned_hands_courts/train.tsv
- split: test
path: data/learned_hands_courts/test.tsv
default: false
- config_name: learned_hands_crime
data_files:
- split: train
path: data/learned_hands_crime/train.tsv
- split: test
path: data/learned_hands_crime/test.tsv
default: false
- config_name: learned_hands_divorce
data_files:
- split: train
path: data/learned_hands_divorce/train.tsv
- split: test
path: data/learned_hands_divorce/test.tsv
default: false
- config_name: learned_hands_domestic_violence
data_files:
- split: train
path: data/learned_hands_domestic_violence/train.tsv
- split: test
path: data/learned_hands_domestic_violence/test.tsv
default: false
- config_name: learned_hands_education
data_files:
- split: train
path: data/learned_hands_education/train.tsv
- split: test
path: data/learned_hands_education/test.tsv
default: false
- config_name: learned_hands_employment
data_files:
- split: train
path: data/learned_hands_employment/train.tsv
- split: test
path: data/learned_hands_employment/test.tsv
default: false
- config_name: learned_hands_estates
data_files:
- split: train
path: data/learned_hands_estates/train.tsv
- split: test
path: data/learned_hands_estates/test.tsv
default: false
- config_name: learned_hands_family
data_files:
- split: train
path: data/learned_hands_family/train.tsv
- split: test
path: data/learned_hands_family/test.tsv
default: false
- config_name: learned_hands_health
data_files:
- split: train
path: data/learned_hands_health/train.tsv
- split: test
path: data/learned_hands_health/test.tsv
default: false
- config_name: learned_hands_housing
data_files:
- split: train
path: data/learned_hands_housing/train.tsv
- split: test
path: data/learned_hands_housing/test.tsv
default: false
- config_name: learned_hands_immigration
data_files:
- split: train
path: data/learned_hands_immigration/train.tsv
- split: test
path: data/learned_hands_immigration/test.tsv
default: false
- config_name: learned_hands_torts
data_files:
- split: train
path: data/learned_hands_torts/train.tsv
- split: test
path: data/learned_hands_torts/test.tsv
default: false
- config_name: learned_hands_traffic
data_files:
- split: train
path: data/learned_hands_traffic/train.tsv
- split: test
path: data/learned_hands_traffic/test.tsv
default: false
- config_name: legal_reasoning_causality
data_files:
- split: train
path: data/legal_reasoning_causality/train.tsv
- split: test
path: data/legal_reasoning_causality/test.tsv
default: false
- config_name: maud_ability_to_consummate_concept_is_subject_to_mae_carveouts
data_files:
- split: train
path: data/maud_ability_to_consummate_concept_is_subject_to_mae_carveouts/train.tsv
- split: test
path: data/maud_ability_to_consummate_concept_is_subject_to_mae_carveouts/test.tsv
default: false
- config_name: maud_accuracy_of_fundamental_target_rws_bringdown_standard
data_files:
- split: train
path: data/maud_accuracy_of_fundamental_target_rws_bringdown_standard/train.tsv
- split: test
path: data/maud_accuracy_of_fundamental_target_rws_bringdown_standard/test.tsv
default: false
- config_name: maud_accuracy_of_target_capitalization_rw_(outstanding_shares)_bringdown_standard_answer
data_files:
- split: train
path: data/maud_accuracy_of_target_capitalization_rw_(outstanding_shares)_bringdown_standard_answer/train.tsv
- split: test
path: data/maud_accuracy_of_target_capitalization_rw_(outstanding_shares)_bringdown_standard_answer/test.tsv
default: false
- config_name: maud_accuracy_of_target_general_rw_bringdown_timing_answer
data_files:
- split: train
path: data/maud_accuracy_of_target_general_rw_bringdown_timing_answer/train.tsv
- split: test
path: data/maud_accuracy_of_target_general_rw_bringdown_timing_answer/test.tsv
default: false
- config_name: maud_additional_matching_rights_period_for_modifications_(cor)
data_files:
- split: train
path: data/maud_additional_matching_rights_period_for_modifications_(cor)/train.tsv
- split: test
path: data/maud_additional_matching_rights_period_for_modifications_(cor)/test.tsv
default: false
- config_name: maud_application_of_buyer_consent_requirement_(negative_interim_covenant)
data_files:
- split: train
path: data/maud_application_of_buyer_consent_requirement_(negative_interim_covenant)/train.tsv
- split: test
path: data/maud_application_of_buyer_consent_requirement_(negative_interim_covenant)/test.tsv
default: false
- config_name: maud_buyer_consent_requirement_(ordinary_course)
data_files:
- split: train
path: data/maud_buyer_consent_requirement_(ordinary_course)/train.tsv
- split: test
path: data/maud_buyer_consent_requirement_(ordinary_course)/test.tsv
default: false
- config_name: maud_change_in_law__subject_to_disproportionate_impact_modifier
data_files:
- split: train
path: data/maud_change_in_law__subject_to_disproportionate_impact_modifier/train.tsv
- split: test
path: data/maud_change_in_law__subject_to_disproportionate_impact_modifier/test.tsv
default: false
- config_name: maud_changes_in_gaap_or_other_accounting_principles__subject_to_disproportionate_impact_modifier
data_files:
- split: train
path: data/maud_changes_in_gaap_or_other_accounting_principles__subject_to_disproportionate_impact_modifier/train.tsv
- split: test
path: data/maud_changes_in_gaap_or_other_accounting_principles__subject_to_disproportionate_impact_modifier/test.tsv
default: false
- config_name: maud_cor_permitted_in_response_to_intervening_event
data_files:
- split: train
path: data/maud_cor_permitted_in_response_to_intervening_event/train.tsv
- split: test
path: data/maud_cor_permitted_in_response_to_intervening_event/test.tsv
default: false
- config_name: maud_cor_permitted_with_board_fiduciary_determination_only
data_files:
- split: train
path: data/maud_cor_permitted_with_board_fiduciary_determination_only/train.tsv
- split: test
path: data/maud_cor_permitted_with_board_fiduciary_determination_only/test.tsv
default: false
- config_name: maud_cor_standard_(intervening_event)
data_files:
- split: train
path: data/maud_cor_standard_(intervening_event)/train.tsv
- split: test
path: data/maud_cor_standard_(intervening_event)/test.tsv
default: false
- config_name: maud_cor_standard_(superior_offer)
data_files:
- split: train
path: data/maud_cor_standard_(superior_offer)/train.tsv
- split: test
path: data/maud_cor_standard_(superior_offer)/test.tsv
default: false
- config_name: maud_definition_contains_knowledge_requirement_-_answer
data_files:
- split: train
path: data/maud_definition_contains_knowledge_requirement_-_answer/train.tsv
- split: test
path: data/maud_definition_contains_knowledge_requirement_-_answer/test.tsv
default: false
- config_name: maud_definition_includes_asset_deals
data_files:
- split: train
path: data/maud_definition_includes_asset_deals/train.tsv
- split: test
path: data/maud_definition_includes_asset_deals/test.tsv
default: false
- config_name: maud_definition_includes_stock_deals
data_files:
- split: train
path: data/maud_definition_includes_stock_deals/train.tsv
- split: test
path: data/maud_definition_includes_stock_deals/test.tsv
default: false
- config_name: maud_fiduciary_exception__board_determination_standard
data_files:
- split: train
path: data/maud_fiduciary_exception__board_determination_standard/train.tsv
- split: test
path: data/maud_fiduciary_exception__board_determination_standard/test.tsv
default: false
- config_name: maud_fiduciary_exception_board_determination_trigger_(no_shop)
data_files:
- split: train
path: data/maud_fiduciary_exception_board_determination_trigger_(no_shop)/train.tsv
- split: test
path: data/maud_fiduciary_exception_board_determination_trigger_(no_shop)/test.tsv
default: false
- config_name: maud_financial_point_of_view_is_the_sole_consideration
data_files:
- split: train
path: data/maud_financial_point_of_view_is_the_sole_consideration/train.tsv
- split: test
path: data/maud_financial_point_of_view_is_the_sole_consideration/test.tsv
default: false
- config_name: maud_fls_(mae)_standard
data_files:
- split: train
path: data/maud_fls_(mae)_standard/train.tsv
- split: test
path: data/maud_fls_(mae)_standard/test.tsv
default: false
- config_name: maud_general_economic_and_financial_conditions_subject_to_disproportionate_impact_modifier
data_files:
- split: train
path: data/maud_general_economic_and_financial_conditions_subject_to_disproportionate_impact_modifier/train.tsv
- split: test
path: data/maud_general_economic_and_financial_conditions_subject_to_disproportionate_impact_modifier/test.tsv
default: false
- config_name: maud_includes_consistent_with_past_practice
data_files:
- split: train
path: data/maud_includes_consistent_with_past_practice/train.tsv
- split: test
path: data/maud_includes_consistent_with_past_practice/test.tsv
default: false
- config_name: maud_initial_matching_rights_period_(cor)
data_files:
- split: train
path: data/maud_initial_matching_rights_period_(cor)/train.tsv
- split: test
path: data/maud_initial_matching_rights_period_(cor)/test.tsv
default: false
- config_name: maud_initial_matching_rights_period_(ftr)
data_files:
- split: train
path: data/maud_initial_matching_rights_period_(ftr)/train.tsv
- split: test
path: data/maud_initial_matching_rights_period_(ftr)/test.tsv
default: false
- config_name: maud_intervening_event_-_required_to_occur_after_signing_-_answer
data_files:
- split: train
path: data/maud_intervening_event_-_required_to_occur_after_signing_-_answer/train.tsv
- split: test
path: data/maud_intervening_event_-_required_to_occur_after_signing_-_answer/test.tsv
default: false
- config_name: maud_knowledge_definition
data_files:
- split: train
path: data/maud_knowledge_definition/train.tsv
- split: test
path: data/maud_knowledge_definition/test.tsv
default: false
- config_name: maud_liability_standard_for_no-shop_breach_by_target_non-do_representatives
data_files:
- split: train
path: data/maud_liability_standard_for_no-shop_breach_by_target_non-do_representatives/train.tsv
- split: test
path: data/maud_liability_standard_for_no-shop_breach_by_target_non-do_representatives/test.tsv
default: false
- config_name: maud_ordinary_course_efforts_standard
data_files:
- split: train
path: data/maud_ordinary_course_efforts_standard/train.tsv
- split: test
path: data/maud_ordinary_course_efforts_standard/test.tsv
default: false
- config_name: maud_pandemic_or_other_public_health_event__subject_to_disproportionate_impact_modifier
data_files:
- split: train
path: data/maud_pandemic_or_other_public_health_event__subject_to_disproportionate_impact_modifier/train.tsv
- split: test
path: data/maud_pandemic_or_other_public_health_event__subject_to_disproportionate_impact_modifier/test.tsv
default: false
- config_name: maud_pandemic_or_other_public_health_event_specific_reference_to_pandemic-related_governmental_responses_or_measures
data_files:
- split: train
path: data/maud_pandemic_or_other_public_health_event_specific_reference_to_pandemic-related_governmental_responses_or_measures/train.tsv
- split: test
path: data/maud_pandemic_or_other_public_health_event_specific_reference_to_pandemic-related_governmental_responses_or_measures/test.tsv
default: false
- config_name: maud_relational_language_(mae)_applies_to
data_files:
- split: train
path: data/maud_relational_language_(mae)_applies_to/train.tsv
- split: test
path: data/maud_relational_language_(mae)_applies_to/test.tsv
default: false
- config_name: maud_specific_performance
data_files:
- split: train
path: data/maud_specific_performance/train.tsv
- split: test
path: data/maud_specific_performance/test.tsv
default: false
- config_name: maud_tail_period_length
data_files:
- split: train
path: data/maud_tail_period_length/train.tsv
- split: test
path: data/maud_tail_period_length/test.tsv
default: false
- config_name: maud_type_of_consideration
data_files:
- split: train
path: data/maud_type_of_consideration/train.tsv
- split: test
path: data/maud_type_of_consideration/test.tsv
default: false
- config_name: nys_judicial_ethics
data_files:
- split: train
path: data/nys_judicial_ethics/train.tsv
- split: test
path: data/nys_judicial_ethics/test.tsv
default: false
- config_name: opp115_data_retention
data_files:
- split: train
path: data/opp115_data_retention/train.tsv
- split: test
path: data/opp115_data_retention/test.tsv
default: false
- config_name: opp115_data_security
data_files:
- split: train
path: data/opp115_data_security/train.tsv
- split: test
path: data/opp115_data_security/test.tsv
default: false
- config_name: opp115_do_not_track
data_files:
- split: train
path: data/opp115_do_not_track/train.tsv
- split: test
path: data/opp115_do_not_track/test.tsv
default: false
- config_name: opp115_first_party_collection_use
data_files:
- split: train
path: data/opp115_first_party_collection_use/train.tsv
- split: test
path: data/opp115_first_party_collection_use/test.tsv
default: false
- config_name: opp115_international_and_specific_audiences
data_files:
- split: train
path: data/opp115_international_and_specific_audiences/train.tsv
- split: test
path: data/opp115_international_and_specific_audiences/test.tsv
default: false
- config_name: opp115_policy_change
data_files:
- split: train
path: data/opp115_policy_change/train.tsv
- split: test
path: data/opp115_policy_change/test.tsv
default: false
- config_name: opp115_third_party_sharing_collection
data_files:
- split: train
path: data/opp115_third_party_sharing_collection/train.tsv
- split: test
path: data/opp115_third_party_sharing_collection/test.tsv
default: false
- config_name: opp115_user_access,_edit_and_deletion
data_files:
- split: train
path: data/opp115_user_access,_edit_and_deletion/train.tsv
- split: test
path: data/opp115_user_access,_edit_and_deletion/test.tsv
default: false
- config_name: opp115_user_choice_control
data_files:
- split: train
path: data/opp115_user_choice_control/train.tsv
- split: test
path: data/opp115_user_choice_control/test.tsv
default: false
- config_name: oral_argument_question_purpose
data_files:
- split: train
path: data/oral_argument_question_purpose/train.tsv
- split: test
path: data/oral_argument_question_purpose/test.tsv
default: false
- config_name: overruling
data_files:
- split: train
path: data/overruling/train.tsv
- split: test
path: data/overruling/test.tsv
default: false
- config_name: personal_jurisdiction
data_files:
- split: train
path: data/personal_jurisdiction/train.tsv
- split: test
path: data/personal_jurisdiction/test.tsv
default: false
- config_name: privacy_policy_entailment
data_files:
- split: train
path: data/privacy_policy_entailment/train.tsv
- split: test
path: data/privacy_policy_entailment/test.tsv
default: false
- config_name: privacy_policy_qa
data_files:
- split: train
path: data/privacy_policy_qa/train.tsv
- split: test
path: data/privacy_policy_qa/test.tsv
default: false
- config_name: proa
data_files:
- split: train
path: data/proa/train.tsv
- split: test
path: data/proa/test.tsv
default: false
- config_name: rule_qa
data_files:
- split: test
path: data/rule_qa/test.tsv
default: false
- config_name: sara_entailment
data_files:
- split: train
path: data/sara_entailment/train.tsv
- split: test
path: data/sara_entailment/test.tsv
default: false
- config_name: sara_numeric
data_files:
- split: train
path: data/sara_numeric/train.tsv
- split: test
path: data/sara_numeric/test.tsv
default: false
- config_name: scalr
data_files:
- split: test
path: data/scalr/test.tsv
default: false
- config_name: ssla_company_defendants
data_files:
- split: train
path: data/ssla_company_defendants/train.tsv
- split: test
path: data/ssla_company_defendants/test.tsv
default: false
- config_name: ssla_individual_defendants
data_files:
- split: train
path: data/ssla_individual_defendants/train.tsv
- split: test
path: data/ssla_individual_defendants/test.tsv
default: false
- config_name: ssla_plaintiff
data_files:
- split: train
path: data/ssla_plaintiff/train.tsv
- split: test
path: data/ssla_plaintiff/test.tsv
default: false
- config_name: successor_liability
data_files:
- split: train
path: data/successor_liability/train.tsv
- split: test
path: data/successor_liability/test.tsv
default: false
- config_name: supply_chain_disclosure_best_practice_accountability
data_files:
- split: train
path: data/supply_chain_disclosure_best_practice_accountability/train.tsv
- split: test
path: data/supply_chain_disclosure_best_practice_accountability/test.tsv
default: false
- config_name: supply_chain_disclosure_best_practice_audits
data_files:
- split: train
path: data/supply_chain_disclosure_best_practice_audits/train.tsv
- split: test
path: data/supply_chain_disclosure_best_practice_audits/test.tsv
default: false
- config_name: supply_chain_disclosure_best_practice_certification
data_files:
- split: train
path: data/supply_chain_disclosure_best_practice_certification/train.tsv
- split: test
path: data/supply_chain_disclosure_best_practice_certification/test.tsv
default: false
- config_name: supply_chain_disclosure_best_practice_training
data_files:
- split: train
path: data/supply_chain_disclosure_best_practice_training/train.tsv
- split: test
path: data/supply_chain_disclosure_best_practice_training/test.tsv
default: false
- config_name: supply_chain_disclosure_best_practice_verification
data_files:
- split: train
path: data/supply_chain_disclosure_best_practice_verification/train.tsv
- split: test
path: data/supply_chain_disclosure_best_practice_verification/test.tsv
default: false
- config_name: supply_chain_disclosure_disclosed_accountability
data_files:
- split: train
path: data/supply_chain_disclosure_disclosed_accountability/train.tsv
- split: test
path: data/supply_chain_disclosure_disclosed_accountability/test.tsv
default: false
- config_name: supply_chain_disclosure_disclosed_audits
data_files:
- split: train
path: data/supply_chain_disclosure_disclosed_audits/train.tsv
- split: test
path: data/supply_chain_disclosure_disclosed_audits/test.tsv
default: false
- config_name: supply_chain_disclosure_disclosed_certification
data_files:
- split: train
path: data/supply_chain_disclosure_disclosed_certification/train.tsv
- split: test
path: data/supply_chain_disclosure_disclosed_certification/test.tsv
default: false
- config_name: supply_chain_disclosure_disclosed_training
data_files:
- split: train
path: data/supply_chain_disclosure_disclosed_training/train.tsv
- split: test
path: data/supply_chain_disclosure_disclosed_training/test.tsv
default: false
- config_name: supply_chain_disclosure_disclosed_verification
data_files:
- split: train
path: data/supply_chain_disclosure_disclosed_verification/train.tsv
- split: test
path: data/supply_chain_disclosure_disclosed_verification/test.tsv
default: false
- config_name: telemarketing_sales_rule
data_files:
- split: train
path: data/telemarketing_sales_rule/train.tsv
- split: test
path: data/telemarketing_sales_rule/test.tsv
default: false
- config_name: textualism_tool_dictionaries
data_files:
- split: train
path: data/textualism_tool_dictionaries/train.tsv
- split: test
path: data/textualism_tool_dictionaries/test.tsv
default: false
- config_name: textualism_tool_plain
data_files:
- split: train
path: data/textualism_tool_plain/train.tsv
- split: test
path: data/textualism_tool_plain/test.tsv
default: false
- config_name: ucc_v_common_law
data_files:
- split: train
path: data/ucc_v_common_law/train.tsv
- split: test
path: data/ucc_v_common_law/test.tsv
default: false
- config_name: unfair_tos
data_files:
- split: train
path: data/unfair_tos/train.tsv
- split: test
path: data/unfair_tos/test.tsv
default: false
---
# Dataset Card for Dataset Name

- **Homepage: https://hazyresearch.stanford.edu/legalbench/**
- **Repository: https://github.com/HazyResearch/legalbench/**
- **Paper: https://arxiv.org/abs/2308.11462**

## Dataset Description

### Dataset Summary

The LegalBench project is an ongoing open science effort to collaboratively curate tasks for evaluating legal reasoning in English large language models (LLMs). The benchmark currently consists of 162 tasks gathered from 40 contributors.

If you have questions about the project or would like to get involved, please see the website for more information.

### Supported Tasks and Leaderboards

LegalBench tasks span multiple types (binary classification, multi-class classification, extraction, generation, entailment), multiple types of text (statutes, judicial opinions, contracts, etc.), and multiple areas of law (evidence, contracts, civil procedure, etc.). For more information on tasks, we recommend visiting the website, where you can search through task descriptions, or the Github repository, which contains more granular task descriptions. We also recommend reading the paper, which provides more background on task significance and construction process.

### Languages

All LegalBench tasks are in English.

## Dataset Structure

### Data Instances

Detailed descriptions of the instances for each task can be found on the Github. An example of an instance, for the `abercrombie` task, is provided below:

```
{
"text": "The mark "Ivory" for a product made of elephant tusks.",
"label": "generic"
"idx": 0
}
```

A substantial number of LegalBench tasks are binary classification tasks, which require the LLM to determine if a piece of text has some legal attribute. Because these are framed as Yes/No questions, the label space is "Yes" or "No".

### Data Fields

Detailed descriptions of the instances for each task can be found on the Github.

### Data Splits

Each task has a training and evaluation split. Following [RAFT](https://huggingface.co/datasets/ought/raft), train splits only consists of a few-labeled instances, reflecting the few-shot nature of most LLMs.

## Dataset Creation

### Curation Rationale

LegalBench was created to enable researchers to better benchmark the legal reasoning capabilities of LLMs.

### Source Data

#### Initial Data Collection and Normalization

Broadly, LegalBench tasks are drawn from three sources. The first source of tasks are existing available datasets and corpora. Most of these were originally released for non-LLM evaluation settings. In creating tasks for LegalBench from these sources, we often significantly reformatted data and restructured the prediction objective. For instance, the original [CUAD dataset](https://github.com/TheAtticusProject/cuad) contains annotations on long-documents and is intended for evaluating extraction with span-prediction models. We restructure this corpora to generate a binary classification task for each type of contractual clause. While the original corpus emphasized the long-document aspects of contracts, our restructured tasks emphasize whether LLMs can identify the distinguishing features of different types of clauses. The second source of tasks are datasets that were previously constructed by legal professionals but never released. This primarily includes datasets hand-coded by legal scholars as part of prior empirical legal projects. The last category of tasks are those that were developed specifically for \name, by the authors of this paper. Overall, tasks are drawn from 36 distinct corpora. Please see the Appendix of the paper for more details.

#### Who are the source language producers?

LegalBench data was created by humans. Demographic information for these individuals is not available.

### Annotations

#### Annotation process

Please see the paper for more information on the annotation process used in the creation of each task.

#### Who are the annotators?

Please see the paper for more information on the identity of annotators for each task.

### Personal and Sensitive Information

Data in this benchmark has either been synthetically generated, or derived from an already public source (e.g., contracts from the EDGAR database).

Several tasks have been derived from the LearnedHands corpus, which consists of public posts on /r/LegalAdvice. Some posts may discuss sensitive issues.

## Considerations for Using the Data

### Social Impact of Dataset

Please see the original paper for a discussion of social impact.

### Discussion of Biases

Please see the original paper for a discussion of social impact.

### Other Known Limitations

LegalBench primarily contains tasks corresponding to American law.

## Additional Information

### Dataset Curators

Please see the website for a full list of participants in the LegalBench project.

### Licensing Information

LegalBench tasks are subject to different licenses. Please see the paper for a description of the licenses.

### Citation Information

If you intend to reference LegalBench broadly, please use the citation below. If you are working with a particular task, please use the citation below in addition to the task specific citation (which can be found on the task page on the website or Github).

```
@misc{guha2023legalbench,
title={LegalBench: A Collaboratively Built Benchmark for Measuring Legal Reasoning in Large Language Models},
author={Neel Guha and Julian Nyarko and Daniel E. Ho and Christopher Ré and Adam Chilton and Aditya Narayana and Alex Chohlas-Wood and Austin Peters and Brandon Waldon and Daniel N. Rockmore and Diego Zambrano and Dmitry Talisman and Enam Hoque and Faiz Surani and Frank Fagan and Galit Sarfaty and Gregory M. Dickinson and Haggai Porat and Jason Hegland and Jessica Wu and Joe Nudell and Joel Niklaus and John Nay and Jonathan H. Choi and Kevin Tobia and Margaret Hagan and Megan Ma and Michael Livermore and Nikon Rasumov-Rahe and Nils Holzenberger and Noam Kolt and Peter Henderson and Sean Rehaag and Sharad Goel and Shang Gao and Spencer Williams and Sunny Gandhi and Tom Zur and Varun Iyer and Zehua Li},
year={2023},
eprint={2308.11462},
archivePrefix={arXiv},
primaryClass={cs.CL}
}
@article{koreeda2021contractnli,
title={ContractNLI: A dataset for document-level natural language inference for contracts},
author={Koreeda, Yuta and Manning, Christopher D},
journal={arXiv preprint arXiv:2110.01799},
year={2021}
}
@article{hendrycks2021cuad,
title={Cuad: An expert-annotated nlp dataset for legal contract review},
author={Hendrycks, Dan and Burns, Collin and Chen, Anya and Ball, Spencer},
journal={arXiv preprint arXiv:2103.06268},
year={2021}
}
@article{wang2023maud,
title={MAUD: An Expert-Annotated Legal NLP Dataset for Merger Agreement Understanding},
author={Wang, Steven H and Scardigli, Antoine and Tang, Leonard and Chen, Wei and Levkin, Dimitry and Chen, Anya and Ball, Spencer and Woodside, Thomas and Zhang, Oliver and Hendrycks, Dan},
journal={arXiv preprint arXiv:2301.00876},
year={2023}
}
@inproceedings{wilson2016creation,
title={The creation and analysis of a website privacy policy corpus},
author={Wilson, Shomir and Schaub, Florian and Dara, Aswarth Abhilash and Liu, Frederick and Cherivirala, Sushain and Leon, Pedro Giovanni and Andersen, Mads Schaarup and Zimmeck, Sebastian and Sathyendra, Kanthashree Mysore and Russell, N Cameron and others},
booktitle={Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)},
pages={1330--1340},
year={2016}
}
@inproceedings{zheng2021does,
title={When does pretraining help? assessing self-supervised learning for law and the casehold dataset of 53,000+ legal holdings},
author={Zheng, Lucia and Guha, Neel and Anderson, Brandon R and Henderson, Peter and Ho, Daniel E},
booktitle={Proceedings of the eighteenth international conference on artificial intelligence and law},
pages={159--168},
year={2021}
}
@article{zimmeck2019maps,
title={Maps: Scaling privacy compliance analysis to a million apps},
author={Zimmeck, Sebastian and Story, Peter and Smullen, Daniel and Ravichander, Abhilasha and Wang, Ziqi and Reidenberg, Joel R and Russell, N Cameron and Sadeh, Norman},
journal={Proc. Priv. Enhancing Tech.},
volume={2019},
pages={66},
year={2019}
}
@article{ravichander2019question,
title={Question answering for privacy policies: Combining computational and legal perspectives},
author={Ravichander, Abhilasha and Black, Alan W and Wilson, Shomir and Norton, Thomas and Sadeh, Norman},
journal={arXiv preprint arXiv:1911.00841},
year={2019}
}
@article{holzenberger2021factoring,
title={Factoring statutory reasoning as language understanding challenges},
author={Holzenberger, Nils and Van Durme, Benjamin},
journal={arXiv preprint arXiv:2105.07903},
year={2021}
}
@article{lippi2019claudette,
title={CLAUDETTE: an automated detector of potentially unfair clauses in online terms of service},
author={Lippi, Marco and Pa{\l}ka, Przemys{\l}aw and Contissa, Giuseppe and Lagioia, Francesca and Micklitz, Hans-Wolfgang and Sartor, Giovanni and Torroni, Paolo},
journal={Artificial Intelligence and Law},
volume={27},
pages={117--139},
year={2019},
publisher={Springer}
}
```