In this paper, we develop a decision-theoretic framework for ordering candidate matches for user confirmation using the concept of the value of perfect information (VPI). At the core of this concept is a utility function that quantifies the desirability of a given state; thus, we devise a utility function for dataspaces based on query result quality. We show in practice how to efficiently apply VPI in concert with this utility function to order user confirmations. A detailed experimental evaluation on both real and synthetic datasets shows that the ordering of user feedback produced by this VPI-based approach yields a dataspace with a significantly higher utility than a wide range of other ordering strategies. Finally, we outline the design of Roomba, a system that utilizes this decision-theoretic framework to guide a dataspace in soliciting user feedback in a pay-as-you-go manner.}, acmid = {1376701}, address = {New York, NY, USA}, author = {Jeffery, Shawn R. and Franklin, Michael J. and Halevy, Alon Y.}, booktitle = {Proceedings of the 2008 ACM SIGMOD international conference on Management of data}, doi = {10.1145/1376616.1376701}, interhash = {3ceaf563712b776c1ed97a8cb061f63b}, intrahash = {3bff24fb9eb1e39fa97a524aabb8dee9}, isbn = {978-1-60558-102-6}, location = {Vancouver, Canada}, numpages = {14}, pages = {847--860}, publisher = {ACM}, title = {Pay-as-you-go user feedback for dataspace systems}, url = {http://doi.acm.org/10.1145/1376616.1376701}, year = 2008 } @mastersthesis{olson2012cloud, abstract = {My thesis describes the design and implementation of systems that empower individuals to help their communities respond to critical situations and to participate in research that helps them understand and improve their environments. People want to help their communities respond to threats such as earthquakes, wildfires, mudslides and hurricanes, and they want to participate in research that helps them understand and improve their environment. “Citizen Science” projects that facilitate this interaction include projects that monitor climate change, water quality and animal habitats. My thesis explores the design and analysis of community-based sense and response systems that enable individuals to participate in critical community activities and scientific research that monitors their environments.}, author = {Olson, Michael J.}, interhash = {a9cdee464e76cd5210c13d7f66981e83}, intrahash = {d9e22a1a5e9404a805aee5cb0fd406c4}, school = {California Institute of Technology}, title = {Cloud computing for citizen science}, type = {Master's thesis}, url = {http://resolver.caltech.edu/CaltechTHESIS:08232011-122341638}, year = 2012 } @inproceedings{ls_leimeister, address = {Cologne, Germany}, author = {Gierczak, Michael and Söllner, Matthias and Leimeister, Jan Marco}, booktitle = {ConLife 2012 Academic Conference}, interhash = {55d8152a9bc0f44d511c03baadb60426}, intrahash = {36de6094774092f9602aa515dc783599}, note = 339, title = {Untersuchung bestehender Geschäftsmodelle etablierter Cloud Anbieter}, url = {http://pubs.wi-kassel.de/wp-content/uploads/2013/03/JML_360.pdf}, year = 2012 } @article{raykar2010learning, abstract = {For many supervised learning tasks it may be infeasible (or very expensive) to obtain objective and reliable labels. Instead, we can collect subjective (possibly noisy) labels from multiple experts or annotators. In practice, there is a substantial amount of disagreement among the annotators, and hence it is of great practical interest to address conventional supervised learning problems in this scenario. In this paper we describe a probabilistic approach for supervised learning when we have multiple annotators providing (possibly noisy) labels but no absolute gold standard. The proposed algorithm evaluates the different experts and also gives an estimate of the actual hidden labels. Experimental results indicate that the proposed method is superior to the commonly used majority voting baseline.}, acmid = {1859894}, author = {Raykar, Vikas C. and Yu, Shipeng and Zhao, Linda H. and Valadez, Gerardo Hermosillo and Florin, Charles and Bogoni, Luca and Moy, Linda}, interhash = {8113daf47997fddf48e4c6c79f2eba56}, intrahash = {14220abe8babfab01c0cdd5ebd5e4b7c}, issn = {1532-4435}, issue_date = {3/1/2010}, journal = {Journal of Machine Learning Research}, month = aug, numpages = {26}, pages = {1297--1322}, publisher = {JMLR.org}, title = {Learning From Crowds}, url = {http://dl.acm.org/citation.cfm?id=1756006.1859894}, volume = 11, year = 2010 } @incollection{li2011incorporating, abstract = {In scientific cooperation network, ambiguous author names may occur due to the existence of multiple authors with the same name. Users of these networks usually want to know the exact author of a paper, whereas we do not have any unique identifier to distinguish them. In this paper, we focus ourselves on such problem, we propose a new method that incorporates user feedback into the model for name disambiguation of scientific cooperation network. Perceptron is used as the classifier. Two features and a constraint drawn from user feedback are incorporated into the perceptron to enhance the performance of name disambiguation. Specifically, we construct user feedback as a training stream, and refine the perceptron continuously. Experimental results show that the proposed algorithm can learn continuously and significantly outperforms the previous methods without introducing user interactions.}, address = {Berlin/Heidelberg}, affiliation = {Intelligent and Distributed Computing Lab, School of Computer Science and Technology, Huazhong University of Science and Technology, Wuhan, 430074 P.R. China}, author = {Li, Yuhua and Wen, Aiming and Lin, Quan and Li, Ruixuan and Lu, Zhengding}, booktitle = {Web-Age Information Management}, doi = {10.1007/978-3-642-23535-1_39}, editor = {Wang, Haixun and Li, Shijun and Oyama, Satoshi and Hu, Xiaohua and Qian, Tieyun}, interhash = {3baace12cb4481dcceb53c2d47f413b5}, intrahash = {96f2ae8551126527c2dfe69c8fa22f6c}, isbn = {978-3-642-23534-4}, keyword = {Computer Science}, pages = {454--466}, publisher = {Springer}, series = {Lecture Notes in Computer Science}, title = {Incorporating User Feedback into Name Disambiguation of Scientific Cooperation Network}, url = {http://dx.doi.org/10.1007/978-3-642-23535-1_39}, volume = 6897, year = 2011 } @article{lofi2012information, abstract = {Recent years brought tremendous advancements in the area of automated information extraction. But still, problem scenarios remain where even state-of-the-art algorithms do not provide a satisfying solution. In these cases, another aspiring recent trend can be exploited to achieve the required extraction quality: explicit crowdsourcing of human intelligence tasks. In this paper, we discuss the synergies between information extraction and crowdsourcing. In particular, we methodically identify and classify the challenges and fallacies that arise when combining both approaches. Furthermore, we argue that for harnessing the full potential of either approach, true hybrid techniques must be considered. To demonstrate this point, we showcase such a hybrid technique, which tightly interweaves information extraction with crowdsourcing and machine learning to vastly surpass the abilities of either technique.}, address = {Berlin/Heidelberg}, affiliation = {Institut für Informationssysteme, Technische Universität Braunschweig, Braunschweig, Germany}, author = {Lofi, Christoph and Selke, Joachim and Balke, Wolf-Tilo}, doi = {10.1007/s13222-012-0092-8}, interhash = {941feeaa7bb134e0a5f8b5c0225756b8}, intrahash = {37cc8f1d19105a073544d6594fbbc033}, issn = {1618-2162}, journal = {Datenbank-Spektrum}, keyword = {Computer Science}, number = 2, pages = {109--120}, publisher = {Springer}, title = {Information Extraction Meets Crowdsourcing: A Promising Couple}, url = {http://dx.doi.org/10.1007/s13222-012-0092-8}, volume = 12, year = 2012 } @inproceedings{paton2011feedback, abstract = {User feedback is gaining momentum as a means of addressing the difficulties underlying information integration tasks. It can be used to assist users in building information integration systems and to improve the quality of existing systems, e.g., in dataspaces. Existing proposals in the area are confined to specific integration sub-problems considering a specific kind of feedback sought, in most cases, from a single user. We argue in this paper that, in order to maximize the benefits that can be drawn from user feedback, it should be considered and managed as a first class citizen. Accordingly, we present generic operations that underpin the management of feedback within information integration systems, and that are applicable to feedback of different kinds, potentially supplied by multiple users with different expectations. We present preliminary solutions that can be adopted for realizing such operations, and sketch a research agenda for the information integration community.}, author = {Paton, Norman W. and Fernandes, Alvaro A. A. and Hedeler, Cornelia and Embury, Suzanne M.}, booktitle = {Proceedings of the Conference on Innovative Data Systems Research (CIDR)}, interhash = {1874e5c09919244808457021d2d884d1}, intrahash = {cd75210156615616e4f25c91143040c4}, pages = {175--183}, title = {User Feedback as a First Class Citizen in Information Integration Systems}, url = {http://www.cidrdb.org/cidr2011/Papers/CIDR11_Paper21.pdf}, year = 2011 } @article{dean2008mapreduce, abstract = {MapReduce is a programming model and an associated implementation for processing and generating large datasets that is amenable to a broad variety of real-world tasks. Users specify the computation in terms of a map and a reduce function, and the underlying runtime system automatically parallelizes the computation across large-scale clusters of machines, handles machine failures, and schedules inter-machine communication to make efficient use of the network and disks. Programmers find the system easy to use: more than ten thousand distinct MapReduce programs have been implemented internally at Google over the past four years, and an average of one hundred thousand MapReduce jobs are executed on Google's clusters every day, processing a total of more than twenty petabytes of data per day.}, acmid = {1327492}, address = {New York, NY, USA}, author = {Dean, Jeffrey and Ghemawat, Sanjay}, doi = {10.1145/1327452.1327492}, interhash = {b8a00982bf087c8543855897b7362a04}, intrahash = {bff539224836d703c2d21141985fa1a3}, issn = {0001-0782}, issue_date = {January 2008}, journal = {Communications of the ACM}, month = jan, number = 1, numpages = {7}, pages = {107--113}, publisher = {ACM}, title = {MapReduce: simplified data processing on large clusters}, url = {http://doi.acm.org/10.1145/1327452.1327492}, volume = 51, year = 2008 } @inproceedings{chai2009efficiently, abstract = {Many applications increasingly employ information extraction and integration (IE/II) programs to infer structures from unstructured data. Automatic IE/II are inherently imprecise. Hence such programs often make many IE/II mistakes, and thus can significantly benefit from user feedback. Today, however, there is no good way to automatically provide and process such feedback. When finding an IE/II mistake, users often must alert the developer team (e.g., via email or Web form) about the mistake, and then wait for the team to manually examine the program internals to locate and fix the mistake, a slow, error-prone, and frustrating process.
In this paper we propose a solution for users to directly provide feedback and for IE/II programs to automatically process such feedback. In our solution a developer U uses hlog, a declarative IE/II language, to write an IE/II program P. Next, U writes declarative user feedback rules that specify which parts of P's data (e.g., input, intermediate, or output data) users can edit, and via which user interfaces. Next, the so-augmented program P is executed, then enters a loop of waiting for and incorporating user feedback. Given user feedback F on a data portion of P, we show how to automatically propagate F to the rest of P, and to seamlessly combine F with prior user feedback. We describe the syntax and semantics of hlog, a baseline execution strategy, and then various optimization techniques. Finally, we describe experiments with real-world data that demonstrate the promise of our solution.}, acmid = {1559857}, address = {New York, NY, USA}, author = {Chai, Xiaoyong and Vuong, Ba-Quy and Doan, AnHai and Naughton, Jeffrey F.}, booktitle = {Proceedings of the 35th SIGMOD international conference on Management of data}, doi = {10.1145/1559845.1559857}, interhash = {5860215447e374b059597c0e3864e388}, intrahash = {d6c9fbf442a935dc0618107f8fb54d44}, isbn = {978-1-60558-551-2}, location = {Providence, Rhode Island, USA}, numpages = {14}, pages = {87--100}, publisher = {ACM}, title = {Efficiently incorporating user feedback into information extraction and integration programs}, url = {http://doi.acm.org/10.1145/1559845.1559857}, year = 2009 } @inproceedings{marcus2011crowdsourced, abstract = {Amazon's Mechanical Turk (\MTurk") service allows users to post short tasks (\HITs") that other users can receive a small amount of money for completing. Common tasks on the system include labelling a collection of images, com- bining two sets of images to identify people which appear in both, or extracting sentiment from a corpus of text snippets. Designing a work ow of various kinds of HITs for ltering, aggregating, sorting, and joining data sources together is common, and comes with a set of challenges in optimizing the cost per HIT, the overall time to task completion, and the accuracy of MTurk results. We propose Qurk, a novel query system for managing these work ows, allowing crowd- powered processing of relational databases. We describe a number of query execution and optimization challenges, and discuss some potential solutions.}, author = {Marcus, Adam and Wu, Eugene and Madden, Samuel and Miller, Robert C.}, booktitle = {Proceedings of the 5th Biennial Conference on Innovative Data Systems Research}, doi = {1721.1/62827}, interhash = {b6b7d67c3c09259fb2d5df3f52e24c9d}, intrahash = {29723ba38aa6039091769cd2f69a1514}, month = jan, pages = {211--214}, publisher = {CIDR}, title = {Crowdsourced Databases: Query Processing with People}, url = {http://dspace.mit.edu/handle/1721.1/62827}, year = 2011 } @inproceedings{horowitz2010anatomy, abstract = {We present Aardvark, a social search engine. With Aardvark, users ask a question, either by instant message, email, web input, text message, or voice. Aardvark then routes the question to the person in the user's extended social network most likely to be able to answer that question. As compared to a traditional web search engine, where the challenge lies in finding the right document to satisfy a user's information need, the challenge in a social search engine like Aardvark lies in finding the right person to satisfy a user's information need. Further, while trust in a traditional search engine is based on authority, in a social search engine like Aardvark, trust is based on intimacy. We describe how these considerations inform the architecture, algorithms, and user interface of Aardvark, and how they are reflected in the behavior of Aardvark users.}, acmid = {1772735}, address = {New York, NY, USA}, author = {Horowitz, Damon and Kamvar, Sepandar D.}, booktitle = {Proceedings of the 19th international conference on World wide web}, doi = {10.1145/1772690.1772735}, interhash = {418d79b49ede3a8d15ef5eb8453094f0}, intrahash = {787ecbd5796ada03f15bdda85497e1fd}, isbn = {978-1-60558-799-8}, location = {Raleigh, North Carolina, USA}, numpages = {10}, pages = {431--440}, publisher = {ACM}, title = {The anatomy of a large-scale social search engine}, url = {http://doi.acm.org/10.1145/1772690.1772735}, year = 2010 } @inproceedings{chan2009mathematical, abstract = {Human computation is a technique that makes use of human abilities for computation to solve problems. Social games use the power of the Internet game players to solve human computation problems. In previous works, many social games were proposed and were quite successful, but no formal framework exists for designing social games in general. A formal framework is important because it lists out the design elements of a social game, the characteristics of a human computation problem, and their relationships. With a formal framework, it simplifies the way to design a social game for a specific problem. In this paper, our contributions are: (1) formulate a formal model on social games, (2) analyze the framework and derive some interesting properties based on model's interactions, (3) illustrate how some current social games can be realized with the proposed formal model, and (4) describe how to design a social game for solving a specific problem with the use of the proposed formal model. This paper presents a set of design guidelines derived from the formal model and demonstrates that the model can help to design a social game for solving a specific problem in a formal and structural way.}, author = {Chan, Kam Tong and King, I. and Yuen, Man-Ching}, booktitle = {Proceedings of the International Conference on Computational Science and Engineering, CSE '09}, doi = {10.1109/CSE.2009.166}, interhash = {a54732b662bcb0d763139a38f6525b56}, intrahash = {216d582316e970eb498423ee8448edbe}, month = aug, pages = {1205--1210}, title = {Mathematical Modeling of Social Games}, url = {http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=5283086&tag=1}, volume = 4, year = 2009 }