@inproceedings{ando-etal-2023-evaluating,
title = "Evaluating Large Language Models with {N}eu{BAROCO}: Syllogistic Reasoning Ability and Human-like Biases",
author = "Ando, Risako and
Morishita, Takanobu and
Abe, Hirohiko and
Mineshima, Koji and
Okada, Mitsuhiro",
editor = "Chatzikyriakidis, Stergios and
de Paiva, Valeria",
booktitle = "Proceedings of the 4th Natural Logic Meets Machine Learning Workshop",
month = jun,
year = "2023",
address = "Nancy, France",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.naloma-1.1",
pages = "1--11",
abstract = "This paper investigates whether current large language models exhibit biases in logical reasoning, similar to humans. Specifically, we focus on syllogistic reasoning, a well-studied form of inference in the cognitive science of human deduction. To facilitate our analysis, we introduce a dataset called NeuBAROCO, originally designed for psychological experiments that assess human logical abilities in syllogistic reasoning. The dataset consists of syllogistic inferences in both English and Japanese. We examine three types of biases observed in human syllogistic reasoning: belief biases, conversion errors, and atmosphere effects. Our findings demonstrate that current large language models struggle more with problems involving these three types of biases.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="ando-etal-2023-evaluating">
<titleInfo>
<title>Evaluating Large Language Models with NeuBAROCO: Syllogistic Reasoning Ability and Human-like Biases</title>
</titleInfo>
<name type="personal">
<namePart type="given">Risako</namePart>
<namePart type="family">Ando</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Takanobu</namePart>
<namePart type="family">Morishita</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Hirohiko</namePart>
<namePart type="family">Abe</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Koji</namePart>
<namePart type="family">Mineshima</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Mitsuhiro</namePart>
<namePart type="family">Okada</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-06</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 4th Natural Logic Meets Machine Learning Workshop</title>
</titleInfo>
<name type="personal">
<namePart type="given">Stergios</namePart>
<namePart type="family">Chatzikyriakidis</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Valeria</namePart>
<namePart type="family">de Paiva</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Nancy, France</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>This paper investigates whether current large language models exhibit biases in logical reasoning, similar to humans. Specifically, we focus on syllogistic reasoning, a well-studied form of inference in the cognitive science of human deduction. To facilitate our analysis, we introduce a dataset called NeuBAROCO, originally designed for psychological experiments that assess human logical abilities in syllogistic reasoning. The dataset consists of syllogistic inferences in both English and Japanese. We examine three types of biases observed in human syllogistic reasoning: belief biases, conversion errors, and atmosphere effects. Our findings demonstrate that current large language models struggle more with problems involving these three types of biases.</abstract>
<identifier type="citekey">ando-etal-2023-evaluating</identifier>
<location>
<url>https://aclanthology.org/2023.naloma-1.1</url>
</location>
<part>
<date>2023-06</date>
<extent unit="page">
<start>1</start>
<end>11</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Evaluating Large Language Models with NeuBAROCO: Syllogistic Reasoning Ability and Human-like Biases
%A Ando, Risako
%A Morishita, Takanobu
%A Abe, Hirohiko
%A Mineshima, Koji
%A Okada, Mitsuhiro
%Y Chatzikyriakidis, Stergios
%Y de Paiva, Valeria
%S Proceedings of the 4th Natural Logic Meets Machine Learning Workshop
%D 2023
%8 June
%I Association for Computational Linguistics
%C Nancy, France
%F ando-etal-2023-evaluating
%X This paper investigates whether current large language models exhibit biases in logical reasoning, similar to humans. Specifically, we focus on syllogistic reasoning, a well-studied form of inference in the cognitive science of human deduction. To facilitate our analysis, we introduce a dataset called NeuBAROCO, originally designed for psychological experiments that assess human logical abilities in syllogistic reasoning. The dataset consists of syllogistic inferences in both English and Japanese. We examine three types of biases observed in human syllogistic reasoning: belief biases, conversion errors, and atmosphere effects. Our findings demonstrate that current large language models struggle more with problems involving these three types of biases.
%U https://aclanthology.org/2023.naloma-1.1
%P 1-11
Markdown (Informal)
[Evaluating Large Language Models with NeuBAROCO: Syllogistic Reasoning Ability and Human-like Biases](https://aclanthology.org/2023.naloma-1.1) (Ando et al., NALOMA-WS 2023)
ACL