@article{herde2023multi,
  abstract = {Solving complex classification tasks using deep neural networks typically requires large amounts of annotated data. However, corresponding class labels are noisy when provided by error-prone annotators, e.g., crowdworkers. Training standard deep neural networks leads to subpar performances in such multi-annotator supervised learning settings. We address this issue by presenting a probabilistic training framework named multi-annotator deep learning (MaDL). A downstream ground truth and an annotator performance model are jointly trained in an end-to-end learning approach. The ground truth model learns to predict instances' true class labels, while the annotator performance model infers probabilistic estimates of annotators' performances. A modular network architecture enables us to make varying assumptions regarding annotators' performances, e.g., an optional class or instance dependency. Further, we learn annotator embeddings to estimate annotators' densities within a latent space as proxies of their potentially correlated annotations. Together with a weighted loss function, we improve the learning from correlated annotation patterns. In a comprehensive evaluation, we examine three research questions about multi-annotator supervised learning. Our findings show MaDL's state-of-the-art performance and robustness against many correlated, spamming annotators.},
  author = {Herde, Marek and Huseljic, Denis and Sick, Bernhard},
  codeurl = {https://github.com/ies-research/multi-annotator-deep-learning},
  interhash = {f3f942451e0beb8358412bfe5ea5618a},
  intrahash = {585c0c2c8eb52d717ffe6c603d01084a},
  journal = {Transactions on Machine Learning Research},
  title = {Multi-annotator Deep Learning: A Probabilistic Framework for Classification},
  url = {https://openreview.net/forum?id=MgdoxzImlK},
  year = 2023
}

@inproceedings{zogaj2013crowdtesting,
  address = {Utrecht, Netherlands (accepted for publication)},
  author = {Zogaj, S. and Bretschneider, U.},
  booktitle = {European Conference on Information Systems (ECIS 2013)},
  interhash = {c2769b1bd671291c6d1912e2f9a09e46},
  intrahash = {51c70e597f74ea0c51819a82d597ccf7},
  title = {Crowdtesting with testCloud – Managing the Challenges of an Intermediary},
  year = 2013
}

@article{ls_leimeister,
  author = {Leimeister, J. M.},
  interhash = {e6742b6302c6fc5eef07b825f69f8cc0},
  intrahash = {8d057705257d8b68337906fad63afe6f},
  journal = {Zeitschrift für Controlling und Management (ZFCM)},
  note = {JML_391},
  number = 56,
  pages = {388-392},
  title = {Crowdsourcing: Crowdfunding, Crowdvoting, Crowdcreation },
  url = {http://pubs.wi-kassel.de/wp-content/uploads/2013/03/JML_391.pdf},
  year = 2012
}

@inproceedings{bretschneider2011schne,
  address = {Dresden, Germany },
  author = {Bretschneider, U. and Leimeister, J. M.},
  booktitle = {Proceedings zum Workshop Gemeinschaft in Neuen Medien (GeNeMe 11)},
  editor = {Meißner, K. and Engelien, M.},
  interhash = {4c9b72a60d16b988a84a107c98c67b06},
  intrahash = {d59149ed85b5c753941228ecd26bec6f},
  note = 356,
  pages = {1-13},
  title = {Schöne neue Crowdsourcing Welt: Billige Arbeitskräfte, Weisheit der Massen?},
  url = {http://pubs.wi-kassel.de/wp-content/uploads/2013/03/JML_297.pdf},
  year = 2011
}

@inproceedings{ls_leimeister,
  address = {Shanghai, China},
  author = {Blohm, I. and Riedl, C. and Leimeister, J. M. and Krcmar, H.},
  booktitle = {32. International Conference on Information Systems},
  interhash = {8c2c9954da066c721858314f7d6a08a7},
  intrahash = {62f04d200a89ec49c0d9b2efe973a27a},
  number = 32,
  pages = {1-24},
  title = {Idea Evaluation Mechanisms for Collective Intelligence in Open Innovation Communities: Do Traders outperform Raters?},
  url = {http://pubs.wi-kassel.de/wp-content/uploads/2013/03/JML_304.pdf},
  year = 2011
}

@inproceedings{ls_leimeister,
  address = {Bonn, Germany},
  author = {Huber, M. J. and Bretschneider, U. and Leimeister, J. M. and Krcmar, H.},
  booktitle = {International Reports on Socio-Informatics - Open Design Spaces Supporting User Innovation. International Workshop on Open Design Spaces (ODS) 2009},
  editor = {Pipek, V. and Rohde, M.},
  interhash = {5517901e4891039d9411bf0675786628},
  intrahash = {7143e141fbc702b9b85c3c108c7cb11f},
  note = {140 (24-09)},
  title = {Making innovation happen: Tool-support for software related communities for innovations},
  url = {http://www.uni-kassel.de/fb7/ibwl/leimeister/pub/JML_126.pdf},
  year = 2009
}

@article{ls_leimeister,
  author = {Leimeister, J. M. and Huber, M. and Bretschneider, U. and Krcmar, H.},
  interhash = {d19c11633bdc1c5431ae87ae327cc224},
  intrahash = {4f677bc3f6cd3fe0bdee234a8813fc73},
  journal = {Journal of Management Information Systems (JMIS)},
  note = {138 (22-09)},
  number = 1,
  pages = {197-224},
  title = {Leveraging Crowdsourcing: Activation-Supporting components for IT-based ideas competition},
  url = {http://www.uni-kassel.de/fb7/ibwl/leimeister/pub/JML_124.pdf},
  volume = 26,
  year = 2009
}

@inproceedings{ls_leimeister,
  address = {Bilbao, Spain},
  author = {Blohm, I. and Fähling, J. and Leimeister, J. M. and Krcmar, H. and Fischer, J.},
  booktitle = {22. ISPIM Conference 2010},
  interhash = {9fa366d690b0820bac4b867269a14dba},
  intrahash = {e98feabe5b9628669d1f269885002d01},
  note = {196 (44-10)},
  number = 22,
  title = {Accelerating customer integration into innovation
processes using Pico-Jobs},
  url = {http://www.uni-kassel.de/fb7/ibwl/leimeister/pub/JML_196.pdf},
  year = 2010
}

@inproceedings{ls_leimeister,
  address = {Shanghai, China},
  author = {Blohm, I. and Riedl, C. and Leimeister, J. M. and Krcmar, H.},
  booktitle = {International Conference on Information Systems 2011 (ICIS) 2011},
  interhash = {8c2c9954da066c721858314f7d6a08a7},
  intrahash = {bab885c4b0a9bfcaaae883cd1d58f4f5},
  note = {286 (49-11)},
  pages = {1-24},
  title = {Idea Evaluation Mechanisms for Collective Intelligence in Open Innovation Communities: Do Traders outperform Raters?},
  url = {http://www.uni-kassel.de/fb7/ibwl/leimeister/pub/JML_286.pdf},
  year = 2011
}

@inbook{ls_leimeister,
  author = {Fähling, J. and Blohm, I. and Leimeister, J. M. and Krcmar, H. and Fischer, J.},
  booktitle = {International Journal of Technology Marketing},
  edition = 6,
  interhash = {8ea8daa11944f66c4fcc2f602ec411c3},
  intrahash = {7fda9d1bb7d58369fdc2195b956ebb87},
  note = {289 (52-11)},
  number = 2,
  pages = {130-147},
  title = {Accelerating Customer Integration into Innovation Processes using Pico Jobs},
  url = {http://www.uni-kassel.de/fb7/ibwl/leimeister/pub/JML_289.pdf},
  year = 2011
}

@incollection{ls_leimeister,
  author = {Fähling, J. and Blohm, I. and Leimeister, J. M. and Krcmar, H. and Fischer, J.},
  booktitle = {International Journal of Technology Marketing},
  edition = 6,
  interhash = {8ea8daa11944f66c4fcc2f602ec411c3},
  intrahash = {96460b1195556ae4d4609436927e0834},
  note = {289 (52-11)},
  number = 2,
  pages = {130-147},
  title = {Accelerating Customer Integration into Innovation Processes using Pico Jobs},
  url = {http://www.uni-kassel.de/fb7/ibwl/leimeister/pub/JML_289.pdf},
  year = 2011
}

@inproceedings{ls_leimeister,
  address = {Shanghai, China},
  author = {Blohm, I. and Riedl, C. and Leimeister, J. M. and Krcmar, H.},
  booktitle = {International Conference on Information Systems 2011 (ICIS) 2011},
  interhash = {8c2c9954da066c721858314f7d6a08a7},
  intrahash = {bab885c4b0a9bfcaaae883cd1d58f4f5},
  note = {286 (49-11)},
  pages = {1-24},
  title = {Idea Evaluation Mechanisms for Collective Intelligence in Open Innovation Communities: Do Traders outperform Raters?},
  url = {http://www.uni-kassel.de/fb7/ibwl/leimeister/pub/JML_286.pdf},
  year = 2011
}

@inproceedings{ls_leimeister,
  address = {Bilbao, Spain},
  author = {Blohm, I. and Fähling, J. and Leimeister, J. M. and Krcmar, H. and Fischer, J.},
  booktitle = {22. ISPIM Conference 2010},
  interhash = {9fa366d690b0820bac4b867269a14dba},
  intrahash = {e98feabe5b9628669d1f269885002d01},
  note = {196 (44-10)},
  number = 22,
  title = {Accelerating customer integration into innovation processes using Pico-Jobs},
  url = {http://pubs.wi-kassel.de/wp-content/uploads/2013/03/JML_250.pdf},
  year = 2010
}

@article{ls_leimeister,
  author = {Leimeister, J. M. and Huber, M. and Bretschneider, U. and Krcmar, H.},
  interhash = {d19c11633bdc1c5431ae87ae327cc224},
  intrahash = {4f677bc3f6cd3fe0bdee234a8813fc73},
  journal = {Journal of Management Information Systems (JMIS)},
  note = {138 (22-09)},
  number = 1,
  pages = {197-224},
  title = {Leveraging Crowdsourcing: Activation-Supporting components for IT-based ideas competition},
  url = {http://pubs.wi-kassel.de/wp-content/uploads/2013/03/JML_145.pdf},
  volume = 26,
  year = 2009
}

@inproceedings{ls_leimeister,
  address = {Bonn, Germany},
  author = {Huber, M. J. and Bretschneider, U. and Leimeister, J. M. and Krcmar, H.},
  booktitle = {International Reports on Socio-Informatics - Open Design Spaces Supporting User Innovation. International Workshop on Open Design Spaces (ODS) 2009},
  editor = {Pipek, V. and Rohde, M.},
  interhash = {5517901e4891039d9411bf0675786628},
  intrahash = {7143e141fbc702b9b85c3c108c7cb11f},
  note = {140 (24-09)},
  title = {Making innovation happen: Tool-support for software related communities for innovations},
  url = {http://pubs.wi-kassel.de/wp-content/uploads/2013/03/JML_161.pdf},
  year = 2009
}