This article examines the reliability of implicit feedback generated from clickthrough data and query reformulations in World Wide Web (WWW) search. Analyzing the users' decision process using eyetracking and comparing implicit feedback against manual relevance judgments, we conclude that clicks are informative but biased. While this makes the interpretation of clicks as absolute relevance judgments difficult, we show that relative preferences derived from clicks are reasonably accurate on average. We find that such relative preferences are accurate not only between results from an individual query, but across multiple sets of results within chains of query reformulations.

@article{joachims2007evaluating, author = {Joachims, Thorsten and Granka, Laura and Pan, Bing and Hembrooke, Helene and Radlinski, Filip and Gay, Geri}, title = {Evaluating the accuracy of implicit feedback from clicks and query reformulations in Web search}, journal = {ACM Trans. Inf. Syst.}, publisher = {ACM}, address = {New York, NY, USA}, year = {2007}, volume = {25}, number = {2}, pages = {7}, url = {http://portal.acm.org/citation.cfm?id=1229181}, doi = {10.1145/1229179.1229181}, issn = {1046-8188}, keywords = {learning-to-rank, feedback, click, ranking, search, web}, abstract = {This article examines the reliability of implicit feedback generated from clickthrough data and query reformulations in World Wide Web (WWW) search. Analyzing the users' decision process using eyetracking and comparing implicit feedback against manual relevance judgments, we conclude that clicks are informative but biased. While this makes the interpretation of clicks as absolute relevance judgments difficult, we show that relative preferences derived from clicks are reasonably accurate on average. We find that such relative preferences are accurate not only between results from an individual query, but across multiple sets of results within chains of query reformulations.} }

%0 = article %A = Joachims, Thorsten and Granka, Laura and Pan, Bing and Hembrooke, Helene and Radlinski, Filip and Gay, Geri %C = New York, NY, USA %D = 2007 %I = ACM %T = Evaluating the accuracy of implicit feedback from clicks and query reformulations in Web search %U = http://portal.acm.org/citation.cfm?id=1229181

This paper examines the reliability of implicit feedback generated from clickthrough data in WWW search. Analyzing the users' decision process using eyetracking and comparing implicit feedback against manual relevance judgments, we conclude that clicks are informative but biased. While this makes the interpretation of clicks as absolute relevance judgments difficult, we show that relative preferences derived from clicks are reasonably accurate on average.

@inproceedings{joachims2005accurately, author = {Joachims, Thorsten and Granka, Laura and Pan, Bing and Hembrooke, Helene and Gay, Geri}, title = {Accurately interpreting clickthrough data as implicit feedback}, booktitle = {Proceedings of the 28th annual international ACM SIGIR conference on Research and development in information retrieval}, publisher = {ACM}, address = {New York, NY, USA}, year = {2005}, pages = {154--161}, url = {http://doi.acm.org/10.1145/1076034.1076063}, doi = {10.1145/1076034.1076063}, isbn = {1-59593-034-5}, keywords = {implicit, feedback, click, study, search, data, evaluation, user, web}, abstract = {This paper examines the reliability of implicit feedback generated from clickthrough data in WWW search. Analyzing the users' decision process using eyetracking and comparing implicit feedback against manual relevance judgments, we conclude that clicks are informative but biased. While this makes the interpretation of clicks as absolute relevance judgments difficult, we show that relative preferences derived from clicks are reasonably accurate on average.} }

%0 = inproceedings %A = Joachims, Thorsten and Granka, Laura and Pan, Bing and Hembrooke, Helene and Gay, Geri %B = Proceedings of the 28th annual international ACM SIGIR conference on Research and development in information retrieval %C = New York, NY, USA %D = 2005 %I = ACM %T = Accurately interpreting clickthrough data as implicit feedback %U = http://doi.acm.org/10.1145/1076034.1076063

This paper examines the reliability of implicit feedback generated from clickthrough data in WWW search. Analyzing the users' decision process using eyetracking and comparing implicit feedback against manual relevance judgments, we conclude that clicks are informative but biased. While this makes the interpretation of clicks as absolute relevance judgments difficult, we show that relative preferences derived from clicks are reasonably accurate on average.

@inproceedings{joachims2005accurately, author = {Joachims, Thorsten and Granka, Laura and Pan, Bing and Hembrooke, Helene and Gay, Geri}, title = {Accurately interpreting clickthrough data as implicit feedback}, booktitle = {SIGIR '05: Proceedings of the 28th annual international ACM SIGIR conference on Research and development in information retrieval}, publisher = {ACM}, address = {New York, NY, USA}, year = {2005}, pages = {154--161}, url = {http://portal.acm.org/citation.cfm?id=1076063}, doi = {10.1145/1076034.1076063}, isbn = {1-59593-034-5}, keywords = {learning-to-rank, feedback, click, ranking, search, web}, abstract = {This paper examines the reliability of implicit feedback generated from clickthrough data in WWW search. Analyzing the users' decision process using eyetracking and comparing implicit feedback against manual relevance judgments, we conclude that clicks are informative but biased. While this makes the interpretation of clicks as absolute relevance judgments difficult, we show that relative preferences derived from clicks are reasonably accurate on average.} }

%0 = inproceedings %A = Joachims, Thorsten and Granka, Laura and Pan, Bing and Hembrooke, Helene and Gay, Geri %B = SIGIR '05: Proceedings of the 28th annual international ACM SIGIR conference on Research and development in information retrieval %C = New York, NY, USA %D = 2005 %I = ACM %T = Accurately interpreting clickthrough data as implicit feedback %U = http://portal.acm.org/citation.cfm?id=1076063