@inproceedings{cremonesi2010performance,
  abstract = {In many commercial systems, the 'best bet' recommendations are shown, but the predicted rating values are not. This is usually referred to as a top-N recommendation task, where the goal of the recommender system is to find a few specific items which are supposed to be most appealing to the user. Common methodologies based on error metrics (such as RMSE) are not a natural fit for evaluating the top-N recommendation task. Rather, top-N performance can be directly measured by alternative methodologies based on accuracy metrics (such as precision/recall). An extensive evaluation of several state-of-the art recommender algorithms suggests that algorithms optimized for minimizing RMSE do not necessarily perform as expected in terms of top-N recommendation task. Results show that improvements in RMSE often do not translate into accuracy improvements. In particular, a naive non-personalized algorithm can outperform some common recommendation approaches and almost match the accuracy of sophisticated algorithms. Another finding is that the very few top popular items can skew the top-N performance. The analysis points out that when evaluating a recommender algorithm on the top-N recommendation task, the test set should be chosen carefully in order to not bias accuracy metrics towards non-personalized solutions. Finally, we offer practitioners new variants of two collaborative filtering algorithms that, regardless of their RMSE, significantly outperform other recommender algorithms in pursuing the top-N recommendation task, with offering additional practical advantages. This comes at surprise given the simplicity of these two methods.},
  acmid = {1864721},
  address = {New York, NY, USA},
  author = {Cremonesi, Paolo and Koren, Yehuda and Turrin, Roberto},
  booktitle = {Proceedings of the Fourth ACM Conference on Recommender Systems},
  doi = {10.1145/1864708.1864721},
  interhash = {04cb3373b65b03e03225f447250e7873},
  intrahash = {aeab7f02942cfeb97ccc7ae0a1d60801},
  isbn = {978-1-60558-906-0},
  location = {Barcelona, Spain},
  numpages = {8},
  pages = {39--46},
  publisher = {ACM},
  series = {RecSys '10},
  title = {Performance of Recommender Algorithms on Top-n Recommendation Tasks},
  url = {http://doi.acm.org/10.1145/1864708.1864721},
  year = 2010
}