title = {{``}You are an expert annotator{''}: Automatic Best{--}Worst-Scaling Annotations for Emotion Intensity Modeling},
  author = {Bagdon, Christopher  and
      Karmalkar, Prathamesh  and
      Gurulingappa, Harsha  and
      Klinger, Roman},
  editor = {Duh, Kevin  and
      Gomez, Helena  and
      Bethard, Steven},
  booktitle = {Proceedings of the 2024 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies (Volume 1: Long Papers)},
  month = jun,
  year = {2024},
  address = {Mexico City, Mexico},
  publisher = {Association for Computational Linguistics},
  url = {https://aclanthology.org/2024.naacl-long.439},
  pages = {7917--7929},
  abstract = {Labeling corpora constitutes a bottleneck to create models for new tasks or domains. Large language models mitigate the issue with automatic corpus labeling methods, particularly for categorical annotations. Some NLP tasks such as emotion intensity prediction, however, require text regression, but there is no work on automating annotations for continuous label assignments. Regression is considered more challenging than classification: The fact that humans perform worse when tasked to choose values from a rating scale lead to comparative annotation methods, including best{--}worst scaling. This raises the question if large language model-based annotation methods show similar patterns, namely that they perform worse on rating scale annotation tasks than on comparative annotation tasks. To study this, we automate emotion intensity predictions and compare direct rating scale predictions, pairwise comparisons and best{--}worst scaling. We find that the latter shows the highest reliability. A transformer regressor fine-tuned on these data performs nearly on par with a model trained on the original manual annotations.},
  internaltype = {conferenceproc},
  url = {https://www.romanklinger.de/publications/BagdonNAACL2024.pdf}
  title = {Understanding Fine-grained Distortions in Reports of
                  Scientific Findings},
  author = {Amelie W\"uhrl and Dustin Wright and Roman Klinger
                  and Isabelle Augenstein},
  booktitle = {Findings of the Association for Computational
                  Linguistics: ACL 2024},
  month = {August},
  year = {2024},
  address = {Bangkok, Thailand},
  publisher = {Association for Computational Linguistics},
  pdf = {https://www.romanklinger.de/publications/WuehrlEtAlACLFindings2024.pdf},
  archiveprefix = {arXiv},
  eprint = {2402.12431},
  internaltype = {conferenceproc}
  title = {{E}mo{P}rogress: Cumulated Emotion Progression
                  Analysis in Dreams and Customer Service Dialogues},
  author = {Wemmer, Eileen and Labat, Sofie and Klinger, Roman},
  editor = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste,
                  Veronique and Lenci, Alessandro and Sakti, Sakriani
                  and Xue, Nianwen},
  booktitle = {Proceedings of the 2024 Joint International
                  Conference on Computational Linguistics, Language
                  Resources and Evaluation (LREC-COLING 2024)},
  month = may,
  year = {2024},
  address = {Torino, Italy},
  publisher = {ELRA and ICCL},
  url = {https://aclanthology.org/2024.lrec-main.503},
  pages = {5660--5677},
  pdf = {https://www.romanklinger.de/publications/WemmerLabatKlingerLRECCOLING2024.pdf},
  internaltype = {conferenceproc}
  title = {Can Factual Statements Be Deceptive? The
                  {D}e{F}a{B}el Corpus of Belief-based Deception},
  author = {Velutharambath, Aswathy and W{\"u}hrl, Amelie and
                  Klinger, Roman},
  editor = {Calzolari, Nicoletta and Kan, Min-Yen and Hoste,
                  Veronique and Lenci, Alessandro and Sakti, Sakriani
                  and Xue, Nianwen},
  booktitle = {Proceedings of the 2024 Joint International
                  Conference on Computational Linguistics, Language
                  Resources and Evaluation (LREC-COLING 2024)},
  month = may,
  year = {2024},
  address = {Torino, Italy},
  publisher = {ELRA and ICCL},
  url = {https://aclanthology.org/2024.lrec-main.243},
  pages = {2708--2723},
  internaltype = {conferenceproc},
  pdf = {https://www.romanklinger.de/publications/VelutharambathWuehrlKlinger-LREC-COLING2024.pdf},
  eprint = {2403.10185},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL}
  author = {Barei\ss{}, Patrick and Klinger, Roman and Barnes,
  title = {English Prompts are Better for {NLI}-based Zero-Shot
                  Emotion Classification than Target-Language Prompts},
  year = {2024},
  isbn = {9798400701726},
  publisher = {Association for Computing Machinery},
  address = {New York, NY, USA},
  url = {https://doi.org/10.1145/3589335.3651902},
  doi = {10.1145/3589335.3651902},
  abstract = {Emotion classification in text is a challenging task
                  due to the processes involved when interpreting a
                  textual description of a potential emotion
                  stimulus. In addition, the set of emotion categories
                  is highly domain-specific. For instance, literature
                  analysis might require the use of aesthetic emotions
                  (e.g., finding something beautiful), and social
                  media analysis could benefit from fine-grained sets
                  (e.g., separating anger from annoyance) than only
                  those that represent basic categories as they have
                  been proposed by Paul Ekman (anger, disgust, fear,
                  joy, surprise, sadness). This renders the task an
                  interesting field for zero-shot classifications, in
                  which the label set is not known at model
                  development time. Unfortunately, most resources for
                  emotion analysis are English, and therefore, most
                  studies on emotion analysis have been performed in
                  English, including those that involve prompting
                  language models for text labels. This leaves us with
                  a research gap that we address in this paper: In
                  which language should we prompt for emotion labels
                  on non-English texts? This is particularly of
                  interest when we have access to a multilingual large
                  language model, because we could request labels with
                  English prompts even for non-English data. Our
                  experiments with natural language inference-based
                  language models show that it is consistently better
                  to use English prompts even if the data is in a
                  different language.},
  booktitle = {Companion Proceedings of the ACM on Web Conference
  pages = {1318–1326},
  numpages = {9},
  location = {Singapore, Singapore},
  series = {WWW '24},
  internaltype = {workshop}
  title = {Topic Bias in Emotion Classification},
  author = {Wegge, Maximilian and Klinger, Roman},
  editor = {van der Goot, Rob and Bak, JinYeong and
                  M{\"u}ller-Eberstein, Max and Xu, Wei and Ritter,
                  Alan and Baldwin, Tim},
  booktitle = {Proceedings of the Ninth Workshop on Noisy and
                  User-generated Text (W-NUT 2024)},
  month = mar,
  year = {2024},
  address = {San {\.G}iljan, Malta},
  publisher = {Association for Computational Linguistics},
  url = {https://aclanthology.org/2024.wnut-1.9},
  pages = {89--103},
  abstract = {Emotion corpora are typically sampled based on
                  keyword/hashtag search or by asking study
                  participants to generate textual instances. In any
                  case, these corpora are not uniform samples
                  representing the entirety of a domain. We
                  hypothesize that this practice of data acquision
                  leads to unrealistic correlations between
                  overrepresented topics in these corpora that harm
                  the generalizability of models. Such topic bias
                  could lead to wrong predictions for instances like
                  {``}I organized the service for my aunt{'}s
                  funeral.{''} when funeral events are overpresented
                  for instances labeled with sadness, despite the
                  emotion of pride being more appropriate here. In
                  this paper, we study this topic bias both from the
                  data and the modeling perspective. We first label a
                  set of emotion corpora automatically via topic
                  modeling and show that emotions in fact correlate
                  with specific topics. Further, we see that emotion
                  classifiers are confounded by such topics. Finally,
                  we show that the established debiasing method of
                  adversarial correction via gradient reversal
                  mitigates the issue. Our work points out issues with
                  existing emotion corpora and that more
                  representative resources are required for fair
                  evaluation of models predicting affective concepts
                  from text.},
  internaltype = {workshop},
  pdf = {https://www.romanklinger.de/publications/WeggeKlinger2024.pdf},
  eprint = {2312.09043},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL}
  title = {What Makes Medical Claims (Un)Verifiable? Analyzing
                  Entity and Relation Properties for Fact
  author = {Wührl, Amelie and Menchaca Resendiz, Yarik and
                  Grimminger, Lara and Klinger, Roman},
  editor = {Graham, Yvette and Purver, Matthew},
  booktitle = {Proceedings of the 18th Conference of the European
                  Chapter of the Association for Computational
                  Linguistics (Volume 1: Long Papers)},
  month = mar,
  year = {2024},
  address = {St. Julian{'}s, Malta},
  publisher = {Association for Computational Linguistics},
  url = {https://aclanthology.org/2024.eacl-long.124},
  pages = {2046--2058},
  abstract = {Verifying biomedical claims fails if no evidence can
                  be discovered. In these cases, the fact-checking
                  verdict remains unknown and the claim is
                  unverifiable. To improve this situation, we have to
                  understand if there are any claim properties that
                  impact its verifiability. In this work we assume
                  that entities and relations define the core
                  variables in a biomedical claim{'}s anatomy and
                  analyze if their properties help us to differentiate
                  verifiable from unverifiable claims. In a study with
                  trained annotation experts we prompt them to find
                  evidence for biomedical claims, and observe how they
                  refine search queries for their evidence
                  search. This leads to the first corpus for
                  scientific fact verification annotated with
                  subject{--}relation{--}object triplets, evidence
                  documents, and fact-checking verdicts (the BEAR-FACT
                  corpus). We find (1) that discovering evidence for
                  negated claims (e.g., X{--}does-not-cause{--}Y) is
                  particularly challenging. Further, we see that
                  annotators process queries mostly by adding
                  constraints to the search and by normalizing
                  entities to canonical names. (2) We compare our
                  in-house annotations with a small crowdsourcing
                  setting where we employ both medical experts and
                  laypeople. We find that domain expertise does not
                  have a substantial effect on the reliability of
                  annotations. Finally, (3), we demonstrate that it is
                  possible to reliably estimate the success of
                  evidence retrieval purely from the claim text
                  (.82F$_1$), whereas identifying unverifiable claims
                  proves more challenging (.27F$_1$)},
  pdf = {https://www.romanklinger.de/publications/Wuehrl-etal-2024-EACL.pdf},
  eprint = {2402.01360},
  archiveprefix = {arXiv},
  primaryclass = {cs.CL},
  internaltype = {conferenceproc}