@inproceedings{tang-etal-2023-diffusion,
title = "Can Diffusion Model Achieve Better Performance in Text Generation ? Bridging the Gap between Training and Inference !",
author = "Tang, Zecheng and
Wang, Pinzheng and
Zhou, Keyan and
Li, Juntao and
Cao, Ziqiang and
Zhang, Min",
editor = "Rogers, Anna and
Boyd-Graber, Jordan and
Okazaki, Naoaki",
booktitle = "Findings of the Association for Computational Linguistics: ACL 2023",
month = jul,
year = "2023",
address = "Toronto, Canada",
publisher = "Association for Computational Linguistics",
url = "https://aclanthology.org/2023.findings-acl.721",
doi = "10.18653/v1/2023.findings-acl.721",
pages = "11359--11386",
abstract = "Diffusion models have been successfully adapted to text generation tasks by mapping the discrete text into the continuous space. However, there exist nonnegligible gaps between training and inference, owing to the absence of the forward process during inference. Thus, the model only predicts based on the previously generated reverse noise rather than the noise computed by the forward process. Besides, the widely-used downsampling strategy in speeding up the inference will cause the mismatch of diffusion trajectories between training and inference. To understand and mitigate the above two types of training-inference discrepancies, we launch a thorough preliminary study. Based on our observations, we propose two simple yet effective methods to bridge the gaps mentioned above, named Distance Penalty and Adaptive Decay Sampling. Extensive experiments on \textbf{6} generation tasks confirm the superiority of our methods, which can achieve $\mathbf{100}\times \rightarrow \mathbf{200}\times$ speedup with better performance. Our code will be released at \url{https://github.com/CODINNLG/Bridge_Gap_Diffusion}.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="tang-etal-2023-diffusion">
<titleInfo>
<title>Can Diffusion Model Achieve Better Performance in Text Generation ? Bridging the Gap between Training and Inference !</title>
</titleInfo>
<name type="personal">
<namePart type="given">Zecheng</namePart>
<namePart type="family">Tang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Pinzheng</namePart>
<namePart type="family">Wang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Keyan</namePart>
<namePart type="family">Zhou</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Juntao</namePart>
<namePart type="family">Li</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Ziqiang</namePart>
<namePart type="family">Cao</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Min</namePart>
<namePart type="family">Zhang</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2023-07</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<relatedItem type="host">
<titleInfo>
<title>Findings of the Association for Computational Linguistics: ACL 2023</title>
</titleInfo>
<name type="personal">
<namePart type="given">Anna</namePart>
<namePart type="family">Rogers</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Jordan</namePart>
<namePart type="family">Boyd-Graber</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Naoaki</namePart>
<namePart type="family">Okazaki</namePart>
<role>
<roleTerm authority="marcrelator" type="text">editor</roleTerm>
</role>
</name>
<originInfo>
<publisher>Association for Computational Linguistics</publisher>
<place>
<placeTerm type="text">Toronto, Canada</placeTerm>
</place>
</originInfo>
<genre authority="marcgt">conference publication</genre>
</relatedItem>
<abstract>Diffusion models have been successfully adapted to text generation tasks by mapping the discrete text into the continuous space. However, there exist nonnegligible gaps between training and inference, owing to the absence of the forward process during inference. Thus, the model only predicts based on the previously generated reverse noise rather than the noise computed by the forward process. Besides, the widely-used downsampling strategy in speeding up the inference will cause the mismatch of diffusion trajectories between training and inference. To understand and mitigate the above two types of training-inference discrepancies, we launch a thorough preliminary study. Based on our observations, we propose two simple yet effective methods to bridge the gaps mentioned above, named Distance Penalty and Adaptive Decay Sampling. Extensive experiments on 6 generation tasks confirm the superiority of our methods, which can achieve \mathbf100\times \rightarrow \mathbf200\times speedup with better performance. Our code will be released at https://github.com/CODINNLG/Bridge_Gap_Diffusion.</abstract>
<identifier type="citekey">tang-etal-2023-diffusion</identifier>
<identifier type="doi">10.18653/v1/2023.findings-acl.721</identifier>
<location>
<url>https://aclanthology.org/2023.findings-acl.721</url>
</location>
<part>
<date>2023-07</date>
<extent unit="page">
<start>11359</start>
<end>11386</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Conference Proceedings
%T Can Diffusion Model Achieve Better Performance in Text Generation ? Bridging the Gap between Training and Inference !
%A Tang, Zecheng
%A Wang, Pinzheng
%A Zhou, Keyan
%A Li, Juntao
%A Cao, Ziqiang
%A Zhang, Min
%Y Rogers, Anna
%Y Boyd-Graber, Jordan
%Y Okazaki, Naoaki
%S Findings of the Association for Computational Linguistics: ACL 2023
%D 2023
%8 July
%I Association for Computational Linguistics
%C Toronto, Canada
%F tang-etal-2023-diffusion
%X Diffusion models have been successfully adapted to text generation tasks by mapping the discrete text into the continuous space. However, there exist nonnegligible gaps between training and inference, owing to the absence of the forward process during inference. Thus, the model only predicts based on the previously generated reverse noise rather than the noise computed by the forward process. Besides, the widely-used downsampling strategy in speeding up the inference will cause the mismatch of diffusion trajectories between training and inference. To understand and mitigate the above two types of training-inference discrepancies, we launch a thorough preliminary study. Based on our observations, we propose two simple yet effective methods to bridge the gaps mentioned above, named Distance Penalty and Adaptive Decay Sampling. Extensive experiments on 6 generation tasks confirm the superiority of our methods, which can achieve \mathbf100\times \rightarrow \mathbf200\times speedup with better performance. Our code will be released at https://github.com/CODINNLG/Bridge_Gap_Diffusion.
%R 10.18653/v1/2023.findings-acl.721
%U https://aclanthology.org/2023.findings-acl.721
%U https://doi.org/10.18653/v1/2023.findings-acl.721
%P 11359-11386
Markdown (Informal)
[Can Diffusion Model Achieve Better Performance in Text Generation ? Bridging the Gap between Training and Inference !](https://aclanthology.org/2023.findings-acl.721) (Tang et al., Findings 2023)
ACL