@inproceedings{nikolaev-pado-2023-universe,
title = "The Universe of Utterances According to {BERT}",
author = "Nikolaev, Dmitry and
Pad{\'o}, Sebastian",
editor = "Amblard, Maxime and
Breitholtz, Ellen",
booktitle = "Proceedings of the 15th International Conference on Computational Semantics",
month = jun,
year = "2023",
address = "Nancy, France",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.iwcs-1.12",
pages = "99--105",
abstract = "It has been argued that BERT {``}rediscovers the traditional NLP pipeline{''}, with lower layers extracting morphosyntactic features and higher layers creating holistic sentence-level representations. In this paper, we critically examine this assumption through a principle-component-guided analysis, extracing sets of inputs that correspond to specific activation patterns in BERT sentence representations. We find that even in higher layers, the model mostly picks up on a variegated bunch of low-level features, many related to sentence complexity, that presumably arise from its specific pre-training objectives.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="nikolaev-pado-2023-universe">
<titleInfo>
<title>The Universe of Utterances According to BERT</title>
</titleInfo>
<name type="personal">
<namePart type="given">Dmitry</namePart>
<namePart type="family">Nikolaev</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Sebastian</namePart>
<namePart type="family">Padó</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-06</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Proceedings of the 15th International Conference on Computational Semantics</title>
</titleInfo>
<name type="personal">
<namePart type="given">Maxime</namePart>
<namePart type="family">Amblard</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ellen</namePart>
<namePart type="family">Breitholtz</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Nancy, France</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>It has been argued that BERT “rediscovers the traditional NLP pipeline”, with lower layers extracting morphosyntactic features and higher layers creating holistic sentence-level representations. In this paper, we critically examine this assumption through a principle-component-guided analysis, extracing sets of inputs that correspond to specific activation patterns in BERT sentence representations. We find that even in higher layers, the model mostly picks up on a variegated bunch of low-level features, many related to sentence complexity, that presumably arise from its specific pre-training objectives.</abstract>
<identifier type="citekey">nikolaev-pado-2023-universe</identifier>
<location>
<url>https://aclanthology.org/2023.iwcs-1.12</url>
</location>
<part>
<date>2023-06</date>
<extent unit="page">
<start>99</start>
<end>105</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T The Universe of Utterances According to BERT
%A Nikolaev, Dmitry
%A Padó, Sebastian
%Y Amblard, Maxime
%Y Breitholtz, Ellen
%S Proceedings of the 15th International Conference on Computational Semantics
%D 2023
%8 June
%I Association for Computational Linguistics
%C Nancy, France
%F nikolaev-pado-2023-universe
%X It has been argued that BERT “rediscovers the traditional NLP pipeline”, with lower layers extracting morphosyntactic features and higher layers creating holistic sentence-level representations. In this paper, we critically examine this assumption through a principle-component-guided analysis, extracing sets of inputs that correspond to specific activation patterns in BERT sentence representations. We find that even in higher layers, the model mostly picks up on a variegated bunch of low-level features, many related to sentence complexity, that presumably arise from its specific pre-training objectives.
%U https://aclanthology.org/2023.iwcs-1.12
%P 99-105
Markdown (Informal)
[The Universe of Utterances According to BERT](https://aclanthology.org/2023.iwcs-1.12) (Nikolaev & Padó, IWCS 2023)
ACL
- Dmitry Nikolaev and Sebastian Padó. 2023. The Universe of Utterances According to BERT. In Proceedings of the 15th International Conference on Computational Semantics, pages 99–105, Nancy, France. Association for Computational Linguistics.