For some problems, human assistance is needed in addition to automated (algorithmic) computation. In sharp contrast to existing data management approaches, where human input is either ad-hoc or is never used, we describe the design of the first declarative language involving human-computable functions, standard relational operators, as well as algorithmic computation. We consider the challenges involved in optimizing queries posed in this language, in particular, the tradeoffs between uncertainty, cost and performance, as well as combination of human and algorithmic evidence. We believe that the vision laid out in this paper can act as a road-map for a new area of data management research where human computation is routinely used in data analytics.

@inproceedings{parameswaran2011answering, author = {Parameswaran, Aditya and Polyzotis, Neoklis}, title = {Answering Queries using Humans, Algorithms and Databases}, booktitle = {Conference on Inovative Data Systems Research (CIDR 2011)}, year = {2011}, pages = {160--166}, url = {http://ilpubs.stanford.edu:8090/986/}, keywords = {algorithm, cirg, collective, computing, database, human, intelligence, social}, abstract = {For some problems, human assistance is needed in addition to automated (algorithmic) computation. In sharp contrast to existing data management approaches, where human input is either ad-hoc or is never used, we describe the design of the first declarative language involving human-computable functions, standard relational operators, as well as algorithmic computation. We consider the challenges involved in optimizing queries posed in this language, in particular, the tradeoffs between uncertainty, cost and performance, as well as combination of human and algorithmic evidence. We believe that the vision laid out in this paper can act as a road-map for a new area of data management research where human computation is routinely used in data analytics.} }

%0 = inproceedings %A = Parameswaran, Aditya and Polyzotis, Neoklis %B = Conference on Inovative Data Systems Research (CIDR 2011) %D = 2011 %T = Answering Queries using Humans, Algorithms and Databases %U = http://ilpubs.stanford.edu:8090/986/

Mechanical Turk (<i>MTurk</i>) provides an on-demand source of human computation. This provides a tremendous opportunity to explore algorithms which incorporate human computation as a function call. However, various systems challenges make this difficult in practice, and most uses of MTurk post large numbers of independent tasks. TurKit is a toolkit for prototyping and exploring algorithmic human computation, while maintaining a straight-forward imperative programming style. We present the crash-and-rerun programming model that makes TurKit possible, along with a variety of applications for human computation algorithms. We also present case studies of TurKit used for real experiments across different fields.

@inproceedings{little2010turkit, author = {Little, Greg and Chilton, Lydia B. and Goldman, Max and Miller, Robert C.}, title = {TurKit: human computation algorithms on mechanical turk}, booktitle = {Proceedings of the 23nd annual ACM symposium on User interface software and technology}, publisher = {ACM}, address = {New York, NY, USA}, year = {2010}, pages = {57--66}, url = {http://doi.acm.org/10.1145/1866029.1866040}, doi = {10.1145/1866029.1866040}, isbn = {978-1-4503-0271-5}, keywords = {algorithm, cirg, collective, computing, human, intelligence, social, turkit}, abstract = {Mechanical Turk (<i>MTurk</i>) provides an on-demand source of human computation. This provides a tremendous opportunity to explore algorithms which incorporate human computation as a function call. However, various systems challenges make this difficult in practice, and most uses of MTurk post large numbers of independent tasks. TurKit is a toolkit for prototyping and exploring algorithmic human computation, while maintaining a straight-forward imperative programming style. We present the crash-and-rerun programming model that makes TurKit possible, along with a variety of applications for human computation algorithms. We also present case studies of TurKit used for real experiments across different fields.} }

%0 = inproceedings %A = Little, Greg and Chilton, Lydia B. and Goldman, Max and Miller, Robert C. %B = Proceedings of the 23nd annual ACM symposium on User interface software and technology %C = New York, NY, USA %D = 2010 %I = ACM %T = TurKit: human computation algorithms on mechanical turk %U = http://doi.acm.org/10.1145/1866029.1866040