@inproceedings{hlavnova-ruder-2023-empowering,
title = "Empowering Cross-lingual Behavioral Testing of {NLP} Models with Typological Features",
author = "Hlavnova, Ester and
Ruder, Sebastian",
editor = "Rogers, Anna and
Boyd-Graber, Jordan and
Okazaki, Naoaki",
booktitle = "Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",
month = jul,
year = "2023",
address = "Toronto, Canada",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.acl-long.396",
doi = "10.18653/v1/2023.acl-long.396",
pages = "7181--7198",
abstract = "A challenge towards developing NLP systems for the world{'}s languages is understanding how they generalize to typological differences relevant for real-world applications. To this end, we propose M2C, a morphologically-aware framework for behavioral testing of NLP models. We use M2C to generate tests that probe models{'} behavior in light of specific linguistic features in 12 typologically diverse languages. We evaluate state-of-the-art language models on the generated tests. While models excel at most tests in English, we highlight generalization failures to specific typological characteristics such as temporal expressions in Swahili and compounding possessives in Finish. Our findings motivate the development of models that address these blind spots.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="hlavnova-ruder-2023-empowering">
<titleInfo>
<title>Empowering Cross-lingual Behavioral Testing of NLP Models with Typological Features</title>
</titleInfo>
<name type="personal">
<namePart type="given">Ester</namePart>
<namePart type="family">Hlavnova</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sebastian</namePart>
<namePart type="family">Ruder</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)</title>
</titleInfo>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Rogers</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jordan</namePart>
<namePart type="family">Boyd-Graber</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Naoaki</namePart>
<namePart type="family">Okazaki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Toronto, Canada</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>A challenge towards developing NLP systems for the world’s languages is understanding how they generalize to typological differences relevant for real-world applications. To this end, we propose M2C, a morphologically-aware framework for behavioral testing of NLP models. We use M2C to generate tests that probe models’ behavior in light of specific linguistic features in 12 typologically diverse languages. We evaluate state-of-the-art language models on the generated tests. While models excel at most tests in English, we highlight generalization failures to specific typological characteristics such as temporal expressions in Swahili and compounding possessives in Finish. Our findings motivate the development of models that address these blind spots.</abstract>
<identifier type="citekey">hlavnova-ruder-2023-empowering</identifier>
<identifier type="doi">10.18653/v1/2023.acl-long.396</identifier>
<location>
<url>https://aclanthology.org/2023.acl-long.396</url>
</location>
<part>
<date>2023-07</date>
<extent unit="page">
<start>7181</start>
<end>7198</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Empowering Cross-lingual Behavioral Testing of NLP Models with Typological Features
%A Hlavnova, Ester
%A Ruder, Sebastian
%Y Rogers, Anna
%Y Boyd-Graber, Jordan
%Y Okazaki, Naoaki
%S Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)
%D 2023
%8 July
%I Association for Computational Linguistics
%C Toronto, Canada
%F hlavnova-ruder-2023-empowering
%X A challenge towards developing NLP systems for the world’s languages is understanding how they generalize to typological differences relevant for real-world applications. To this end, we propose M2C, a morphologically-aware framework for behavioral testing of NLP models. We use M2C to generate tests that probe models’ behavior in light of specific linguistic features in 12 typologically diverse languages. We evaluate state-of-the-art language models on the generated tests. While models excel at most tests in English, we highlight generalization failures to specific typological characteristics such as temporal expressions in Swahili and compounding possessives in Finish. Our findings motivate the development of models that address these blind spots.
%R 10.18653/v1/2023.acl-long.396
%U https://aclanthology.org/2023.acl-long.396
%U https://doi.org/10.18653/v1/2023.acl-long.396
%P 7181-7198
Markdown (Informal)
[Empowering Cross-lingual Behavioral Testing of NLP Models with Typological Features](https://aclanthology.org/2023.acl-long.396) (Hlavnova & Ruder, ACL 2023)
ACL