Alam, Firoj; Hasnat, Abul; Ahmad, Fatema; Hasan, Md. Arid; Hasanain, Maram
ÄrMeme: Propagandistic Content in Arabic Memes” Proceedings Article
In: Al-Onaizan, Yaser; Bansal, Mohit; Chen, Yun-Nung (Ed.): Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing, pp. 21071–21090, Association for Computational Linguistics, Miami, Florida, USA, 2024.
@inproceedings{alam-etal-2024-armeme,
title = {ÄrMeme: Propagandistic Content in Arabic Memes"},
author = {Firoj Alam and Abul Hasnat and Fatema Ahmad and Md. Arid Hasan and Maram Hasanain},
editor = {Yaser Al-Onaizan and Mohit Bansal and Yun-Nung Chen},
url = {https://aclanthology.org/2024.emnlp-main.1173},
year = {2024},
date = {2024-11-01},
booktitle = {Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing},
pages = {21071–21090},
publisher = {Association for Computational Linguistics},
address = {Miami, Florida, USA},
abstract = {With the rise of digital communication memes have become a significant medium for cultural and political expression that is often used to mislead audience. Identification of such misleading and persuasive multimodal content become more important among various stakeholders, including social media platforms, policymakers, and the broader society as they often cause harm to the individuals, organizations and/or society. While there has been effort to develop AI based automatic system for resource rich languages (e.g., English), it is relatively little to none for medium to low resource languages. In this study, we focused on developing an Arabic memes dataset with manual annotations of propagandistic content. We annotated $sim6K$ Arabic memes collected from various social media platforms, which is a first resource for Arabic multimodal research. We provide a comprehensive analysis aiming to develop computational tools for their detection. We made the dataset publicly available for the community.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Hasanain, Maram; Hasan, Md. Arid; Ahmad, Fatema; Suwaileh, Reem; Biswas, Md. Rafiul; Zaghouani, Wajdi; Alam, Firoj
ÄrAIEval Shared Task: Propagandistic Techniques Detection in Unimodal and Multimodal Arabic Content” Proceedings Article
In: Habash, Nizar; Bouamor, Houda; Eskander, Ramy; Tomeh, Nadi; Farha, Ibrahim Abu; Abdelali, Ahmed; Touileb, Samia; Hamed, Injy; Onaizan, Yaser; Alhafni, Bashar; Antoun, Wissam; Khalifa, Salam; Haddad, Hatem; Zitouni, Imed; AlKhamissi, Badr; Almatham, Rawan; Mrini, Khalil (Ed.): Proceedings of The Second Arabic Natural Language Processing Conference, pp. 456–466, Association for Computational Linguistics, Bangkok, Thailand, 2024.
@inproceedings{hasanain-etal-2024-araieval,
title = {ÄrAIEval Shared Task: Propagandistic Techniques Detection in Unimodal and Multimodal Arabic Content"},
author = {Maram Hasanain and Md. Arid Hasan and Fatema Ahmad and Reem Suwaileh and Md. Rafiul Biswas and Wajdi Zaghouani and Firoj Alam},
editor = {Nizar Habash and Houda Bouamor and Ramy Eskander and Nadi Tomeh and Ibrahim Abu Farha and Ahmed Abdelali and Samia Touileb and Injy Hamed and Yaser Onaizan and Bashar Alhafni and Wissam Antoun and Salam Khalifa and Hatem Haddad and Imed Zitouni and Badr AlKhamissi and Rawan Almatham and Khalil Mrini},
url = {https://aclanthology.org/2024.arabicnlp-1.44},
year = {2024},
date = {2024-08-01},
booktitle = {Proceedings of The Second Arabic Natural Language Processing Conference},
pages = {456–466},
publisher = {Association for Computational Linguistics},
address = {Bangkok, Thailand},
abstract = {We present an overview of the second edition of the ArAIEval shared task, organized as part of the ArabicNLP 2024 conference co-located with ACL 2024. In this edition, ArAIEval offers two tasks: (i) detection of propagandistic textual spans with persuasion techniques identification in tweets and news articles, and (ii) distinguishing between propagandistic and non-propagandistic memes. A total of 14 teams participated in the final evaluation phase, with 6 and 9 teams participating in Tasks 1 and 2, respectively. Finally, 11 teams submitted system description papers. Across both tasks, we observed that fine-tuning transformer models such as AraBERT was at the core of the majority of the participating systems. We provide a description of the task setup, including a description of the dataset construction and the evaluation setup. We further provide a brief overview of the participating systems. All datasets and evaluation scripts are released to the research community. We hope this will enable further research on these important tasks in Arabic.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Hasanain, Maram; Hasan, Md. Arid; Ahmad, Fatema; Suwaileh, Reem; Biswas, Md. Rafiul; Zaghouani, Wajdi; Alam, Firoj
ÄrAIEval Shared Task: Propagandistic Techniques Detection in Unimodal and Multimodal Arabic Content Proceedings Article
In: Habash, Nizar; Bouamor, Houda; Eskander, Ramy; Tomeh, Nadi; Farha, Ibrahim Abu; Abdelali, Ahmed; Touileb, Samia; Hamed, Injy; Onaizan, Yaser; Alhafni, Bashar; Antoun, Wissam; Khalifa, Salam; Haddad, Hatem; Zitouni, Imed; AlKhamissi, Badr; Almatham, Rawan; Mrini, Khalil (Ed.): Proceedings of The Second Arabic Natural Language Processing Conference, pp. 456–466, Association for Computational Linguistics, Bangkok, Thailand, 2024.
@inproceedings{hasanain-etal-2024-araievalb,
title = {ÄrAIEval Shared Task: Propagandistic Techniques Detection in Unimodal and Multimodal Arabic Content},
author = {Maram Hasanain and Md. Arid Hasan and Fatema Ahmad and Reem Suwaileh and Md. Rafiul Biswas and Wajdi Zaghouani and Firoj Alam},
editor = {Nizar Habash and Houda Bouamor and Ramy Eskander and Nadi Tomeh and Ibrahim Abu Farha and Ahmed Abdelali and Samia Touileb and Injy Hamed and Yaser Onaizan and Bashar Alhafni and Wissam Antoun and Salam Khalifa and Hatem Haddad and Imed Zitouni and Badr AlKhamissi and Rawan Almatham and Khalil Mrini},
url = {https://aclanthology.org/2024.arabicnlp-1.44},
year = {2024},
date = {2024-08-01},
urldate = {2024-08-01},
booktitle = {Proceedings of The Second Arabic Natural Language Processing Conference},
pages = {456–466},
publisher = {Association for Computational Linguistics},
address = {Bangkok, Thailand},
abstract = {We present an overview of the second edition of the ArAIEval shared task, organized as part of the ArabicNLP 2024 conference co-located with ACL 2024. In this edition, ArAIEval offers two tasks: (i) detection of propagandistic textual spans with persuasion techniques identification in tweets and news articles, and (ii) distinguishing between propagandistic and non-propagandistic memes. A total of 14 teams participated in the final evaluation phase, with 6 and 9 teams participating in Tasks 1 and 2, respectively. Finally, 11 teams submitted system description papers. Across both tasks, we observed that fine-tuning transformer models such as AraBERT was at the core of the majority of the participating systems. We provide a description of the task setup, including a description of the dataset construction and the evaluation setup. We further provide a brief overview of the participating systems. All datasets and evaluation scripts are released to the research community. We hope this will enable further research on these important tasks in Arabic.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Dimitrov, Dimitar; Alam, Firoj; Hasanain, Maram; Hasnat, Abul; Silvestri, Fabrizio; Nakov, Preslav; Martino, Giovanni Da San
SemEval-2024 Task 4: Multilingual Detection of Persuasion Techniques in Memes Proceedings Article
In: Ojha, Atul Kr.; Doğruöz, A. Seza; Madabushi, Harish Tayyar; Martino, Giovanni Da San; Rosenthal, Sara; Rosá, Aiala (Ed.): Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024), pp. 2009–2026, Association for Computational Linguistics, Mexico City, Mexico, 2024.
@inproceedings{dimitrov-etal-2024-semevalb,
title = {SemEval-2024 Task 4: Multilingual Detection of Persuasion Techniques in Memes},
author = {Dimitar Dimitrov and Firoj Alam and Maram Hasanain and Abul Hasnat and Fabrizio Silvestri and Preslav Nakov and Giovanni Da San Martino},
editor = {Atul Kr. Ojha and A. Seza Doğruöz and Harish Tayyar Madabushi and Giovanni Da San Martino and Sara Rosenthal and Aiala Rosá},
url = {https://aclanthology.org/2024.semeval-1.275},
doi = {https://doi.org/10.18653/v1/2024.semeval-1.275},
year = {2024},
date = {2024-06-01},
booktitle = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
pages = {2009–2026},
publisher = {Association for Computational Linguistics},
address = {Mexico City, Mexico},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Dimitrov, Dimitar; Alam, Firoj; Hasanain, Maram; Hasnat, Abul; Silvestri, Fabrizio; Nakov, Preslav; Martino, Giovanni Da San
SemEval-2024 Task 4: Multilingual Detection of Persuasion Techniques in Memes Proceedings Article
In: Ojha, Atul Kr.; Doğruöz, A. Seza; Madabushi, Harish Tayyar; Martino, Giovanni Da San; Rosenthal, Sara; Rosá, Aiala (Ed.): Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024), pp. 2009–2026, Association for Computational Linguistics, Mexico City, Mexico, 2024.
@inproceedings{dimitrov-etal-2024-semevalbb,
title = {SemEval-2024 Task 4: Multilingual Detection of Persuasion Techniques in Memes},
author = {Dimitar Dimitrov and Firoj Alam and Maram Hasanain and Abul Hasnat and Fabrizio Silvestri and Preslav Nakov and Giovanni Da San Martino},
editor = {Atul Kr. Ojha and A. Seza Doğruöz and Harish Tayyar Madabushi and Giovanni Da San Martino and Sara Rosenthal and Aiala Rosá},
url = {https://aclanthology.org/2024.semeval-1.275},
doi = {https://doi.org/10.18653/v1/2024.semeval-1.275},
year = {2024},
date = {2024-06-01},
urldate = {2024-06-01},
booktitle = {Proceedings of the 18th International Workshop on Semantic Evaluation (SemEval-2024)},
pages = {2009–2026},
publisher = {Association for Computational Linguistics},
address = {Mexico City, Mexico},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Hasanain, Maram; Ahmad, Fatema; Alam, Firoj
Can GPT-4 Identify Propaganda? Annotation and Detection of Propaganda Spans in News Articles Proceedings Article
In: Calzolari, Nicoletta; Kan, Min-Yen; Hoste, Veronique; Lenci, Alessandro; Sakti, Sakriani; Xue, Nianwen (Ed.): Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024), pp. 2724–2744, ELRA and ICCL, Torino, Italia, 2024.
@inproceedings{hasanain-etal-2024-gpt,
title = {Can GPT-4 Identify Propaganda? Annotation and Detection of Propaganda Spans in News Articles},
author = {Maram Hasanain and Fatema Ahmad and Firoj Alam},
editor = {Nicoletta Calzolari and Min-Yen Kan and Veronique Hoste and Alessandro Lenci and Sakriani Sakti and Nianwen Xue},
url = {https://aclanthology.org/2024.lrec-main.244},
year = {2024},
date = {2024-05-01},
booktitle = {Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)},
pages = {2724–2744},
publisher = {ELRA and ICCL},
address = {Torino, Italia},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Hasanain, Maram; Ahmad, Fatema; Alam, Firoj
Can GPT-4 Identify Propaganda? Annotation and Detection of Propaganda Spans in News Articles Proceedings Article
In: Calzolari, Nicoletta; Kan, Min-Yen; Hoste, Veronique; Lenci, Alessandro; Sakti, Sakriani; Xue, Nianwen (Ed.): Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024), pp. 2724–2744, ELRA and ICCL, Torino, Italia, 2024.
@inproceedings{hasanain-etal-2024-gptb,
title = {Can GPT-4 Identify Propaganda? Annotation and Detection of Propaganda Spans in News Articles},
author = {Maram Hasanain and Fatema Ahmad and Firoj Alam},
editor = {Nicoletta Calzolari and Min-Yen Kan and Veronique Hoste and Alessandro Lenci and Sakriani Sakti and Nianwen Xue},
url = {https://aclanthology.org/2024.lrec-main.244},
year = {2024},
date = {2024-05-01},
urldate = {2024-05-01},
booktitle = {Proceedings of the 2024 Joint International Conference on Computational Linguistics, Language Resources and Evaluation (LREC-COLING 2024)},
pages = {2724–2744},
publisher = {ELRA and ICCL},
address = {Torino, Italia},
abstract = {The use of propaganda has spiked on mainstream and social media, aiming to manipulate or mislead users. While efforts to automatically detect propaganda techniques in textual, visual, or multimodal content have increased, most of them primarily focus on English content. The majority of the recent initiatives targeting medium to low-resource languages produced relatively small annotated datasets, with a skewed distribution, posing challenges for the development of sophisticated propaganda detection models. To address this challenge, we carefully develop the largest propaganda dataset to date, ArPro, comprised of 8K paragraphs from newspaper articles, labeled at the text span level following a taxonomy of 23 propagandistic techniques. Furthermore, our work offers the first attempt to understand the performance of large language models (LLMs), using GPT-4, for fine-grained propaganda detection from text. Results showed that GPT-4's performance degrades as the task moves from simply classifying a paragraph as propagandistic or not, to the fine-grained task of detecting propaganda techniques and their manifestation in text. Compared to models fine-tuned on the dataset for propaganda detection at different classification granularities, GPT-4 is still far behind. Finally, we evaluate GPT-4 on a dataset consisting of six other languages for span detection, and results suggest that the model struggles with the task across languages. We made the dataset publicly available for the community.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Barrón-Cedeño, Alberto; Alam, Firoj; Chakraborty, Tanmoy; Elsayed, Tamer; Nakov, Preslav; Przybyła, Piotr; Struß, Julia Maria; Haouari, Fatima; Hasanain, Maram; Ruggeri, Federico; Song, Xingyi; Suwaileh, Reem
The CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles, Authorities, and Adversarial Robustness Proceedings Article
In: Goharian, Nazli; Tonellotto, Nicola; He, Yulan; Lipani, Aldo; McDonald, Graham; Macdonald, Craig; Ounis, Iadh (Ed.): Advances in Information Retrieval, pp. 449–458, Springer Nature Switzerland, 2024, ISBN: 978-3-031-56069-9.
@inproceedings{CheckThat:ECIR2024,
title = {The CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles, Authorities, and Adversarial Robustness},
author = {Alberto Barrón-Cedeño and Firoj Alam and Tanmoy Chakraborty and Tamer Elsayed and Preslav Nakov and Piotr Przybyła and Julia Maria Struß and Fatima Haouari and Maram Hasanain and Federico Ruggeri and Xingyi Song and Reem Suwaileh},
editor = {Nazli Goharian and Nicola Tonellotto and Yulan He and Aldo Lipani and Graham McDonald and Craig Macdonald and Iadh Ounis},
isbn = {978-3-031-56069-9},
year = {2024},
date = {2024-01-01},
booktitle = {Advances in Information Retrieval},
pages = {449–458},
publisher = {Springer Nature Switzerland},
abstract = {The first five editions of the CheckThat! lab focused on the main tasks of the information verification pipeline: check-worthiness, evidence retrieval and pairing, and verification. Since the 2023 edition, it has been focusing on new problems that can support the research and decision making during the verification process. In this new edition, we focus on new problems and —for the first time— we propose six tasks in fifteen languages (Arabic, Bulgarian, English, Dutch, French, Georgian, German, Greek, Italian, Polish, Portuguese, Russian, Slovene, Spanish, and code-mixed Hindi-English): Task 1 estimation of check-worthiness (the only task that has been present in all CheckThat! editions), Task 2 identification of subjectivity (a follow up of CheckThat! 2023 edition), Task 3 identification of persuasion (a follow up of SemEval 2023), Task 4 detection of hero, villain, and victim from memes (a follow up of CONSTRAINT 2022), Task 5 Rumor Verification using Evidence from Authorities (a first), and Task 6 robustness of credibility assessment with adversarial examples (a first). These tasks represent challenging classification and retrieval problems at the document and at the span level, including multilingual and multimodal settings.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Barrón-Cedeño, Alberto; Alam, Firoj; Struß, Julia Maria; Nakov, Preslav; Chakraborty, Tanmoy; Elsayed, Tamer; Przybyła, Piotr; Caselli, Tommaso; Martino, Giovanni Da San; Haouari, Fatima; Li, Chengkai; Piskorski, Jakub; Ruggeri, Federico; Song, Xingyi; Suwaileh, Reem
Overview of the CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles, Authorities and Adversarial Robustness Proceedings Article
In: Goeuriot, Lorraine; Mulhem, Philippe; Quénot, Georges; Schwab, Didier; Soulier, Laure; Nunzio, Giorgio Maria Di; Galuščáková, Petra; de Herrera, Alba García Seco; Faggioli, Guglielmo; Ferro, Nicola (Ed.): Experimental IR Meets Multilinguality, Multimodality, and Interaction. Proceedings of the Fifteenth International Conference of the CLEF Association (CLEF 2024), 2024.
@inproceedings{clef-checkthat:2024-lncs,
title = {Overview of the CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles,
Authorities and Adversarial Robustness},
author = {Alberto Barrón-Cedeño and Firoj Alam and Julia Maria Struß and Preslav Nakov and Tanmoy Chakraborty and Tamer Elsayed and Piotr Przybyła and Tommaso Caselli and Giovanni Da San Martino and Fatima Haouari and Chengkai Li and Jakub Piskorski and Federico Ruggeri and Xingyi Song and Reem Suwaileh},
editor = {Lorraine Goeuriot and Philippe Mulhem and Georges Quénot and Didier Schwab and Laure Soulier and Giorgio Maria Di Nunzio and Petra Galuščáková and Alba García Seco de Herrera and Guglielmo Faggioli and Nicola Ferro},
year = {2024},
date = {2024-01-01},
booktitle = {Experimental IR Meets Multilinguality, Multimodality, and Interaction.
Proceedings of the Fifteenth International Conference of the CLEF Association (CLEF 2024)},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Hasanain, Maram; Suwaileh, Reem; Weering, Sanne; Li, Chengkai; Caselli, Tommaso; Zaghouani, Wajdi; Barrón-Cedeño, Alberto; Nakov, Preslav; Alam, Firoj
Overview of the CLEF-2024 CheckThat! Lab Task 1 on Check-Worthiness Estimation of Multigenre Content Proceedings Article
In: Working Notes of CLEF 2024 – Conference and Labs of the Evaluation Forum, Grenoble, France, 2024.
@inproceedings{clef-checkthat:2024:task1,
title = {Overview of the CLEF-2024 CheckThat! Lab Task 1 on Check-Worthiness Estimation of Multigenre Content},
author = {Maram Hasanain and Reem Suwaileh and Sanne Weering and Chengkai Li and Tommaso Caselli and Wajdi Zaghouani and Alberto Barrón-Cedeño and Preslav Nakov and Firoj Alam},
year = {2024},
date = {2024-01-01},
booktitle = {Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum},
address = {Grenoble, France},
series = {CLEF~2024},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Struß, Julia Maria; Ruggeri, Federico; Barrón-Cedeño, Alberto; Alam, Firoj; Dimitrov, Dimitar; Galassi, Andrea; Pachov, Georgi; Koychev, Ivan; Nakov, Preslav; Siegel, Melanie; Wiegand, Michael; Hasanain, Maram; Suwaileh, Reem; Zaghouani, Wajdi
Overview of the CLEF-2024 CheckThat! Lab Task 2 on Subjectivity in News Articles Proceedings Article
In: Working Notes of CLEF 2024 – Conference and Labs of the Evaluation Forum, Grenoble, France, 2024.
@inproceedings{clef-checkthat:2024:task2,
title = {Overview of the CLEF-2024 CheckThat! Lab Task 2 on Subjectivity in News Articles},
author = {Julia Maria Struß and Federico Ruggeri and Alberto Barrón-Cedeño and Firoj Alam and Dimitar Dimitrov and Andrea Galassi and Georgi Pachov and Ivan Koychev and Preslav Nakov and Melanie Siegel and Michael Wiegand and Maram Hasanain and Reem Suwaileh and Wajdi Zaghouani},
year = {2024},
date = {2024-01-01},
booktitle = {Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum},
address = {Grenoble, France},
series = {CLEF~2024},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Piskorski, Jakub; Stefanovitch, Nicolas; Alam, Firoj; Campos, Ricardo; Dimitrov, Dimitar; Jorge, Alípio; Pollak, Senja; Ribin, Nikolay; Fijavž, Zoran; Hasanain, Maram; Guimarães, Nuno; Pacheco, Ana Filipa; Sartori, Elisa; Silvano, Purificação; Zwitter, Ana Vitez; Koychev, Ivan; Yu, Nana; Nakov, Preslav; Martino, Giovanni Da San
Overview of the CLEF-2024 CheckThat! Lab Task 3 on Persuasion Techniques Proceedings Article
In: Working Notes of CLEF 2024 – Conference and Labs of the Evaluation Forum, Grenoble, France, 2024.
@inproceedings{clef-checkthat:2024:task3,
title = {Overview of the CLEF-2024 CheckThat! Lab Task 3 on Persuasion Techniques},
author = {Jakub Piskorski and Nicolas Stefanovitch and Firoj Alam and Ricardo Campos and Dimitar Dimitrov and Alípio Jorge and Senja Pollak and Nikolay Ribin and Zoran Fijavž and Maram Hasanain and Nuno Guimarães and Ana Filipa Pacheco and Elisa Sartori and Purificação Silvano and Ana Vitez Zwitter and Ivan Koychev and Nana Yu and Preslav Nakov and Giovanni Da San Martino},
year = {2024},
date = {2024-01-01},
booktitle = {Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum},
address = {Grenoble, France},
series = {CLEF~2024},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Suwaileh, Reem; Hasanain, Maram; Hubail, Fatema; Zaghouani, Wajdi; Alam, Firoj
ThatiAR: Subjectivity Detection in Arabic News Sentences Journal Article
In: arXiv: 2406.05559, 2024.
@article{ThatiAR2024,
title = {ThatiAR: Subjectivity Detection in Arabic News Sentences},
author = {Reem Suwaileh and Maram Hasanain and Fatema Hubail and Wajdi Zaghouani and Firoj Alam},
year = {2024},
date = {2024-01-01},
journal = {arXiv: 2406.05559},
keywords = {},
pubstate = {published},
tppubtype = {article}
}
Barrón-Cedeño, Alberto; Alam, Firoj; Chakraborty, Tanmoy; Elsayed, Tamer; Nakov, Preslav; Przybyła, Piotr; Struß, Julia Maria; Haouari, Fatima; Hasanain, Maram; Ruggeri, Federico; Song, Xingyi; Suwaileh, Reem
The CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles, Authorities, and Adversarial Robustness Proceedings Article
In: Goharian, Nazli; Tonellotto, Nicola; He, Yulan; Lipani, Aldo; McDonald, Graham; Macdonald, Craig; Ounis, Iadh (Ed.): Advances in Information Retrieval, pp. 449–458, Springer Nature Switzerland, 2024, ISBN: 978-3-031-56069-9.
@inproceedings{CheckThat:ECIR2024b,
title = {The CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles, Authorities, and Adversarial Robustness},
author = {Alberto Barrón-Cedeño and Firoj Alam and Tanmoy Chakraborty and Tamer Elsayed and Preslav Nakov and Piotr Przybyła and Julia Maria Struß and Fatima Haouari and Maram Hasanain and Federico Ruggeri and Xingyi Song and Reem Suwaileh},
editor = {Nazli Goharian and Nicola Tonellotto and Yulan He and Aldo Lipani and Graham McDonald and Craig Macdonald and Iadh Ounis},
isbn = {978-3-031-56069-9},
year = {2024},
date = {2024-01-01},
booktitle = {Advances in Information Retrieval},
pages = {449–458},
publisher = {Springer Nature Switzerland},
abstract = {The first five editions of the CheckThat! lab focused on the main tasks of the information verification pipeline: check-worthiness, evidence retrieval and pairing, and verification. Since the 2023 edition, it has been focusing on new problems that can support the research and decision making during the verification process. In this new edition, we focus on new problems and —for the first time— we propose six tasks in fifteen languages (Arabic, Bulgarian, English, Dutch, French, Georgian, German, Greek, Italian, Polish, Portuguese, Russian, Slovene, Spanish, and code-mixed Hindi-English): Task 1 estimation of check-worthiness (the only task that has been present in all CheckThat! editions), Task 2 identification of subjectivity (a follow up of CheckThat! 2023 edition), Task 3 identification of persuasion (a follow up of SemEval 2023), Task 4 detection of hero, villain, and victim from memes (a follow up of CONSTRAINT 2022), Task 5 Rumor Verification using Evidence from Authorities (a first), and Task 6 robustness of credibility assessment with adversarial examples (a first). These tasks represent challenging classification and retrieval problems at the document and at the span level, including multilingual and multimodal settings.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Barrón-Cedeño, Alberto; Alam, Firoj; Struß, Julia Maria; Nakov, Preslav; Chakraborty, Tanmoy; Elsayed, Tamer; Przybyła, Piotr; Caselli, Tommaso; Martino, Giovanni Da San; Haouari, Fatima; Li, Chengkai; Piskorski, Jakub; Ruggeri, Federico; Song, Xingyi; Suwaileh, Reem
Overview of the CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles, Authorities and Adversarial Robustness Proceedings Article
In: Goeuriot, Lorraine; Mulhem, Philippe; Quénot, Georges; Schwab, Didier; Soulier, Laure; Nunzio, Giorgio Maria Di; Galuščáková, Petra; de Herrera, Alba García Seco; Faggioli, Guglielmo; Ferro, Nicola (Ed.): Experimental IR Meets Multilinguality, Multimodality, and Interaction. Proceedings of the Fifteenth International Conference of the CLEF Association (CLEF 2024), 2024.
@inproceedings{clef-checkthat:2024-lncsb,
title = {Overview of the CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles,
Authorities and Adversarial Robustness},
author = {Alberto Barrón-Cedeño and Firoj Alam and Julia Maria Struß and Preslav Nakov and Tanmoy Chakraborty and Tamer Elsayed and Piotr Przybyła and Tommaso Caselli and Giovanni Da San Martino and Fatima Haouari and Chengkai Li and Jakub Piskorski and Federico Ruggeri and Xingyi Song and Reem Suwaileh},
editor = {Lorraine Goeuriot and Philippe Mulhem and Georges Quénot and Didier Schwab and Laure Soulier and Giorgio Maria Di Nunzio and Petra Galuščáková and Alba García Seco de Herrera and Guglielmo Faggioli and Nicola Ferro},
year = {2024},
date = {2024-01-01},
booktitle = {Experimental IR Meets Multilinguality, Multimodality, and Interaction.
Proceedings of the Fifteenth International Conference of the CLEF Association (CLEF 2024)},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Hasanain, Maram; Suwaileh, Reem; Weering, Sanne; Li, Chengkai; Caselli, Tommaso; Zaghouani, Wajdi; Barrón-Cedeño, Alberto; Nakov, Preslav; Alam, Firoj
Overview of the CLEF-2024 CheckThat! Lab Task 1 on Check-Worthiness Estimation of Multigenre Content Proceedings Article
In: Working Notes of CLEF 2024 – Conference and Labs of the Evaluation Forum, Grenoble, France, 2024.
@inproceedings{clef-checkthat:2024:task1b,
title = {Overview of the CLEF-2024 CheckThat! Lab Task 1 on Check-Worthiness Estimation of Multigenre Content},
author = {Maram Hasanain and Reem Suwaileh and Sanne Weering and Chengkai Li and Tommaso Caselli and Wajdi Zaghouani and Alberto Barrón-Cedeño and Preslav Nakov and Firoj Alam},
year = {2024},
date = {2024-01-01},
booktitle = {Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum},
address = {Grenoble, France},
series = {CLEF~2024},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Struß, Julia Maria; Ruggeri, Federico; Barrón-Cedeño, Alberto; Alam, Firoj; Dimitrov, Dimitar; Galassi, Andrea; Pachov, Georgi; Koychev, Ivan; Nakov, Preslav; Siegel, Melanie; Wiegand, Michael; Hasanain, Maram; Suwaileh, Reem; Zaghouani, Wajdi
Overview of the CLEF-2024 CheckThat! Lab Task 2 on Subjectivity in News Articles Proceedings Article
In: Working Notes of CLEF 2024 – Conference and Labs of the Evaluation Forum, Grenoble, France, 2024.
@inproceedings{clef-checkthat:2024:task2b,
title = {Overview of the CLEF-2024 CheckThat! Lab Task 2 on Subjectivity in News Articles},
author = {Julia Maria Struß and Federico Ruggeri and Alberto Barrón-Cedeño and Firoj Alam and Dimitar Dimitrov and Andrea Galassi and Georgi Pachov and Ivan Koychev and Preslav Nakov and Melanie Siegel and Michael Wiegand and Maram Hasanain and Reem Suwaileh and Wajdi Zaghouani},
year = {2024},
date = {2024-01-01},
booktitle = {Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum},
address = {Grenoble, France},
series = {CLEF~2024},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Struß, Julia Maria; Ruggeri, Federico; Barrón-Cedeño, Alberto; Alam, Firoj; Dimitrov, Dimitar; Galassi, Andrea; Pachov, Georgi; Koychev, Ivan; Nakov, Preslav; Siegel, Melanie; Wiegand, Michael; Hasanain, Maram; Suwaileh, Reem; Zaghouani, Wajdi
Overview of the CLEF-2024 CheckThat! Lab Task 2 on Subjectivity in News Articles Proceedings Article
In: Working Notes of CLEF 2024 – Conference and Labs of the Evaluation Forum, Grenoble, France, 2024.
@inproceedings{clef-checkthat:2024:task2c,
title = {Overview of the CLEF-2024 CheckThat! Lab Task 2 on Subjectivity in News Articles},
author = {Julia Maria Struß and Federico Ruggeri and Alberto Barrón-Cedeño and Firoj Alam and Dimitar Dimitrov and Andrea Galassi and Georgi Pachov and Ivan Koychev and Preslav Nakov and Melanie Siegel and Michael Wiegand and Maram Hasanain and Reem Suwaileh and Wajdi Zaghouani},
url = {https://cris.unibo.it/handle/11585/980321},
year = {2024},
date = {2024-01-01},
urldate = {2024-01-01},
booktitle = {Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum},
address = {Grenoble, France},
series = {CLEF~2024},
abstract = {We present an overview of Task 2 of the seventh edition of the CheckThat! lab at the 2024 iteration of the Conference and Labs of the Evaluation Forum (CLEF). The task focuses on subjectivity detection in news articles and was offered in five languages: Arabic, Bulgarian, English, German, and Italian, as well as in a multilingual setting. The datasets for each language were carefully curated and annotated, comprising over 10,000 sentences from news articles. The task challenged participants to develop systems capable of distinguishing between subjective statements (reflecting personal opinions or biases) and objective ones (presenting factual information) at the sentence level. A total of 15 teams participated in the task, submitting 36 valid runs across all language tracks. The participants used a variety of approaches, with transformer-based models being the most popular choice. Strategies included fine-tuning monolingual and multilingual models, and leveraging English models with automatic translation for the non-English datasets. Some teams also explored ensembles, feature engineering, and innovative techniques such as few-shot learning and in-context learning with large language models. The evaluation was based on macro-averaged F1 score. The results varied across languages, with the best performance achieved for Italian and German, followed by English. The Arabic track proved particularly challenging, with no team surpassing an F1 score of 0.50. This task contributes to the broader goal of enhancing the reliability of automated content analysis in the context of misinformation detection and fact-checking. The paper provides detailed insights into the datasets, participant approaches, and results, offering a benchmark for the current state of subjectivity detection across multiple languages.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
Piskorski, Jakub; Stefanovitch, Nicolas; Alam, Firoj; Campos, Ricardo; Dimitrov, Dimitar; Jorge, Alípio; Pollak, Senja; Ribin, Nikolay; Fijavž, Zoran; Hasanain, Maram; Guimarães, Nuno; Pacheco, Ana Filipa; Sartori, Elisa; Silvano, Purificação; Zwitter, Ana Vitez; Koychev, Ivan; Yu, Nana; Nakov, Preslav; Martino, Giovanni Da San
Overview of the CLEF-2024 CheckThat! Lab Task 3 on Persuasion Techniques Proceedings Article
In: Working Notes of CLEF 2024 – Conference and Labs of the Evaluation Forum, Grenoble, France, 2024.
@inproceedings{clef-checkthat:2024:task3c,
title = {Overview of the CLEF-2024 CheckThat! Lab Task 3 on Persuasion Techniques},
author = {Jakub Piskorski and Nicolas Stefanovitch and Firoj Alam and Ricardo Campos and Dimitar Dimitrov and Alípio Jorge and Senja Pollak and Nikolay Ribin and Zoran Fijavž and Maram Hasanain and Nuno Guimarães and Ana Filipa Pacheco and Elisa Sartori and Purificação Silvano and Ana Vitez Zwitter and Ivan Koychev and Nana Yu and Preslav Nakov and Giovanni Da San Martino},
url = {https://ceur-ws.org/Vol-3740/paper-26.pdf},
year = {2024},
date = {2024-01-01},
urldate = {2024-01-01},
booktitle = {Working Notes of CLEF 2024 - Conference and Labs of the Evaluation Forum},
address = {Grenoble, France},
series = {CLEF~2024},
abstract = {We present an overview of CheckThat! Lab’s 2024 Task 3, which focuses on detecting 23 persuasion techniques
at the text-span level in online media. The task covers five languages, namely, Arabic, Bulgarian, English,
Portuguese, and Slovene, and highly-debated topics in the media, e.g., the Isreali–Palestian conflict, the Russia–
Ukraine war, climate change, COVID-19, abortion, etc. A total of 23 teams registered for the task, and two of
them submitted system responses which were compared against a baseline and a task organizers’ system, which
used a state-of-the-art transformer-based architecture. We provide a description of the dataset and the overall
task setup, including the evaluation methodology, and an overview of the participating systems. The datasets
accompanied with the evaluation scripts are released to the research community, which we believe will foster
research on persuasion technique detection and analysis of online media content in various fields and contexts.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}
at the text-span level in online media. The task covers five languages, namely, Arabic, Bulgarian, English,
Portuguese, and Slovene, and highly-debated topics in the media, e.g., the Isreali–Palestian conflict, the Russia–
Ukraine war, climate change, COVID-19, abortion, etc. A total of 23 teams registered for the task, and two of
them submitted system responses which were compared against a baseline and a task organizers’ system, which
used a state-of-the-art transformer-based architecture. We provide a description of the dataset and the overall
task setup, including the evaluation methodology, and an overview of the participating systems. The datasets
accompanied with the evaluation scripts are released to the research community, which we believe will foster
research on persuasion technique detection and analysis of online media content in various fields and contexts.
Barrón-Cedeño, Alberto; Alam, Firoj; Struß, Julia Maria; Nakov, Preslav; Chakraborty, Tanmoy; Elsayed, Tamer; Przybyła, Piotr; Caselli, Tommaso; Martino, Giovanni Da San; Haouari, Fatima; Li, Chengkai; Piskorski, Jakub; Ruggeri, Federico; Song, Xingyi; Suwaileh, Reem
Overview of the CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles, Authorities and Adversarial Robustness Proceedings Article
In: Goeuriot, Lorraine; Mulhem, Philippe; Quénot, Georges; Schwab, Didier; Soulier, Laure; Nunzio, Giorgio Maria Di; Galuščáková, Petra; Herrera, Alba García Seco; Faggioli, Guglielmo; Ferro, Nicola (Ed.): Experimental IR Meets Multilinguality, Multimodality, and Interaction. Proceedings of the Fifteenth International Conference of the CLEF Association (CLEF 2024), 2024.
@inproceedings{clef-checkthat:2024-lncsc,
title = {Overview of the CLEF-2024 CheckThat! Lab: Check-Worthiness, Subjectivity, Persuasion, Roles, Authorities and Adversarial Robustness},
author = {Alberto Barrón-Cedeño and Firoj Alam and Julia Maria Struß and Preslav Nakov and Tanmoy Chakraborty and Tamer Elsayed and Piotr Przybyła and Tommaso Caselli and Giovanni Da San Martino and Fatima Haouari and Chengkai Li and Jakub Piskorski and Federico Ruggeri and Xingyi Song and Reem Suwaileh},
editor = {Lorraine Goeuriot and Philippe Mulhem and Georges Quénot and Didier Schwab and Laure Soulier and Giorgio Maria Di Nunzio and Petra Galuščáková and Alba García Seco Herrera and Guglielmo Faggioli and Nicola Ferro},
url = {https://ceur-ws.org/Vol-3740/paper-24.pdf},
year = {2024},
date = {2024-01-01},
urldate = {2024-01-01},
booktitle = {Experimental IR Meets Multilinguality, Multimodality, and Interaction.
Proceedings of the Fifteenth International Conference of the CLEF Association (CLEF 2024)},
abstract = {We present an overview of the CheckThat! Lab 2024 Task 1, part of CLEF 2024. Task 1 involves determining whether a text item is check-worthy, with a special emphasis on COVID-19, political news, and political debates and speeches. It is conducted in three languages: Arabic, Dutch, and English. Additionally, Spanish was offered for extra training data during the development phase. A total of 75 teams registered, with 37 teams submitting 236 runs and 17 teams submitting system description papers. Out of these, 13, 15 and 26 teams participated for Arabic, Dutch and English, respectively. Among these teams, the use of transformer pre-trained language models (PLMs) was the most frequent. A few teams also employed Large Language Models (LLMs). We provide a description of the dataset, the task setup, including evaluation settings, and a brief overview of the participating systems. As is customary in the CheckThat! Lab, we release all the datasets as well as the evaluation scripts to the research community. This will enable further research on identifying relevant check-worthy content that can assist various stakeholders, such as fact-checkers, journalists, and policymakers.},
keywords = {},
pubstate = {published},
tppubtype = {inproceedings}
}