@article{herde2023multi,
  abstract = {Solving complex classification tasks using deep neural networks typically requires large amounts of annotated data. However, corresponding class labels are noisy when provided by error-prone annotators, e.g., crowdworkers. Training standard deep neural networks leads to subpar performances in such multi-annotator supervised learning settings. We address this issue by presenting a probabilistic training framework named multi-annotator deep learning (MaDL). A downstream ground truth and an annotator performance model are jointly trained in an end-to-end learning approach. The ground truth model learns to predict instances' true class labels, while the annotator performance model infers probabilistic estimates of annotators' performances. A modular network architecture enables us to make varying assumptions regarding annotators' performances, e.g., an optional class or instance dependency. Further, we learn annotator embeddings to estimate annotators' densities within a latent space as proxies of their potentially correlated annotations. Together with a weighted loss function, we improve the learning from correlated annotation patterns. In a comprehensive evaluation, we examine three research questions about multi-annotator supervised learning. Our findings show MaDL's state-of-the-art performance and robustness against many correlated, spamming annotators.},
  author = {Herde, Marek and Huseljic, Denis and Sick, Bernhard},
  codeurl = {https://github.com/ies-research/multi-annotator-deep-learning},
  interhash = {f3f942451e0beb8358412bfe5ea5618a},
  intrahash = {585c0c2c8eb52d717ffe6c603d01084a},
  journal = {Transactions on Machine Learning Research},
  title = {Multi-annotator Deep Learning: A Probabilistic Framework for Classification},
  url = {https://openreview.net/forum?id=MgdoxzImlK},
  year = 2023
}

@inproceedings{kaur2014scholarometer,
  abstract = {Scholarometer (scholarometer.indiana.edu) is a social tool developed to facilitate citation analysis and help evaluate the impact of authors. The Scholarometer service allows scholars to compute various citation-based impact measures. In exchange, users provide disciplinary annotations of authors, which allow for the computation of discipline-specific statistics and discipline-neutral impact metrics. We present here two improvements of our system. First, we integrated a new universal impact metric hs that uses crowdsourced data to calculate the global rank of a scholar across disciplinary boundaries. Second, improvements made in ambiguous name classification have increased the accuracy from 80% to 87%.},
  acmid = {2615669},
  address = {New York, NY, USA},
  author = {Kaur, Jasleen and JafariAsbagh, Mohsen and Radicchi, Filippo and Menczer, Filippo},
  booktitle = {Proceedings of the 2014 ACM Conference on Web Science},
  doi = {10.1145/2615569.2615669},
  interhash = {bfb4274f2a002cde9efbe71faf295e6a},
  intrahash = {4edc2b8ed7acdd1ef8be4d6eefea8718},
  isbn = {978-1-4503-2622-3},
  location = {Bloomington, Indiana, USA},
  numpages = {2},
  pages = {285--286},
  publisher = {ACM},
  series = {WebSci '14},
  title = {Scholarometer: A System for Crowdsourcing Scholarly Impact Metrics},
  url = {http://doi.acm.org/10.1145/2615569.2615669},
  year = 2014
}

@inproceedings{jaschke2013attribute,
  abstract = {We propose an approach for supporting attribute exploration by web information retrieval, in particular by posing appropriate queries to search engines, crowd sourcing systems, and the linked open data cloud. We discuss underlying general assumptions for this to work and the degree to which these can be taken for granted.},
  author = {Jäschke, Robert and Rudolph, Sebastian},
  booktitle = {Contributions to the 11th International Conference on Formal Concept Analysis},
  editor = {Cellier, Peggy and Distel, Felix and Ganter, Bernhard},
  interhash = {000ab7b0ae3ecd1d7d6ceb39de5c11d4},
  intrahash = {45e900e280661d775d8da949baee3747},
  month = may,
  organization = {Technische Universität Dresden},
  pages = {19--34},
  title = {Attribute Exploration on the Web},
  url = {http://nbn-resolving.de/urn:nbn:de:bsz:14-qucosa-113133},
  urn = {urn:nbn:de:bsz:14-qucosa-113133},
  year = 2013
}

@inproceedings{quinn2011human,
  abstract = {The rapid growth of human computation within research and industry has produced many novel ideas aimed at organizing web users to do great things. However, the growth is not adequately supported by a framework with which to understand each new system in the context of the old. We classify human computation systems to help identify parallels between different systems and reveal "holes" in the existing work as opportunities for new research. Since human computation is often confused with "crowdsourcing" and other terms, we explore the position of human computation with respect to these related topics.},
  acmid = {1979148},
  address = {New York, NY, USA},
  author = {Quinn, Alexander J. and Bederson, Benjamin B.},
  booktitle = {Proceedings of the SIGCHI Conference on Human Factors in Computing Systems},
  doi = {10.1145/1978942.1979148},
  interhash = {f319e8c67a7af1afd804774ccba7b717},
  intrahash = {5d672c8a77dc513a877d5d8e05edbb39},
  isbn = {978-1-4503-0228-9},
  location = {Vancouver, BC, Canada},
  numpages = {10},
  pages = {1403--1412},
  publisher = {ACM},
  series = {CHI '11},
  title = {Human computation: a survey and taxonomy of a growing field},
  url = {http://doi.acm.org/10.1145/1978942.1979148},
  year = 2011
}

@inproceedings{zogaj2013crowdtesting,
  address = {Utrecht, Netherlands (accepted for publication)},
  author = {Zogaj, S. and Bretschneider, U.},
  booktitle = {European Conference on Information Systems (ECIS 2013)},
  interhash = {c2769b1bd671291c6d1912e2f9a09e46},
  intrahash = {51c70e597f74ea0c51819a82d597ccf7},
  title = {Crowdtesting with testCloud – Managing the Challenges of an Intermediary},
  year = 2013
}