One of the most important features of the Web graph and social networks is that they are constantly evolving. The classical computational paradigm, which assumes a fixed data set as an input to an algorithm that terminates, is inadequate for such settings. In this paper we study the problem of computing PageRank on an evolving graph. We propose an algorithm that, at any moment in the time and by crawling a small portion of the graph, provides an estimate of the PageRank that is close to the true PageRank of the graph at that moment. We will also evaluate our algorithm experimentally on real data sets and on randomly generated inputs. Under a stylized model of graph evolution, we show that our algorithm achieves a provable performance guarantee that is significantly better than the naive algorithm that crawls the nodes in a round-robin fashion.

@inproceedings{bahmani2012pagerank, author = {Bahmani, Bahman and Kumar, Ravi and Mahdian, Mohammad and Upfal, Eli}, title = {PageRank on an evolving graph}, booktitle = {Proceedings of the 18th ACM SIGKDD international conference on Knowledge discovery and data mining}, publisher = {ACM}, address = {New York, NY, USA}, year = {2012}, pages = {24--32}, url = {http://doi.acm.org/10.1145/2339530.2339539}, doi = {10.1145/2339530.2339539}, isbn = {978-1-4503-1462-6}, keywords = {engine, pagerank, search, time, web}, abstract = {One of the most important features of the Web graph and social networks is that they are constantly evolving. The classical computational paradigm, which assumes a fixed data set as an input to an algorithm that terminates, is inadequate for such settings. In this paper we study the problem of computing PageRank on an evolving graph. We propose an algorithm that, at any moment in the time and by crawling a small portion of the graph, provides an estimate of the PageRank that is close to the true PageRank of the graph at that moment. We will also evaluate our algorithm experimentally on real data sets and on randomly generated inputs. Under a stylized model of graph evolution, we show that our algorithm achieves a provable performance guarantee that is significantly better than the naive algorithm that crawls the nodes in a round-robin fashion.} }

%0 = inproceedings %A = Bahmani, Bahman and Kumar, Ravi and Mahdian, Mohammad and Upfal, Eli %B = Proceedings of the 18th ACM SIGKDD international conference on Knowledge discovery and data mining %C = New York, NY, USA %D = 2012 %I = ACM %T = PageRank on an evolving graph %U = http://doi.acm.org/10.1145/2339530.2339539

To optimize the performance of web crawlers, various page importance measures have been studied to select and order URLs in crawling. Most sophisticated measures (e.g. breadth-first and PageRank) are based on link structure. In this paper, we treat the problem from another perspective and propose to measure page importance through mining user interest and behaviors from web browse logs. Unlike most existing approaches which work on single URL, in this paper, both the log mining and the crawl ordering are performed at the granularity of URL pattern. The proposed URL pattern-based crawl orderings are capable to properly predict the importance of newly created (unseen) URLs. Promising experimental results proved the feasibility of our approach.

@inproceedings{liu2011browsing, author = {Liu, Minghai and Cai, Rui and Zhang, Ming and Zhang, Lei}, title = {User browsing behavior-driven web crawling}, booktitle = {Proceedings of the 20th ACM international conference on Information and knowledge management}, publisher = {ACM}, address = {New York, NY, USA}, year = {2011}, pages = {87--92}, url = {http://doi.acm.org/10.1145/2063576.2063593}, doi = {10.1145/2063576.2063593}, isbn = {978-1-4503-0717-8}, keywords = {crawling, crowd, engine, search, web}, abstract = {To optimize the performance of web crawlers, various page importance measures have been studied to select and order URLs in crawling. Most sophisticated measures (e.g. breadth-first and PageRank) are based on link structure. In this paper, we treat the problem from another perspective and propose to measure page importance through mining user interest and behaviors from web browse logs. Unlike most existing approaches which work on single URL, in this paper, both the log mining and the crawl ordering are performed at the granularity of URL pattern. The proposed URL pattern-based crawl orderings are capable to properly predict the importance of newly created (unseen) URLs. Promising experimental results proved the feasibility of our approach.} }

%0 = inproceedings %A = Liu, Minghai and Cai, Rui and Zhang, Ming and Zhang, Lei %B = Proceedings of the 20th ACM international conference on Information and knowledge management %C = New York, NY, USA %D = 2011 %I = ACM %T = User browsing behavior-driven web crawling %U = http://doi.acm.org/10.1145/2063576.2063593

@book{croft2010search, author = {Croft, W. Bruce and Metzler, Donald and Strohman, Trevor}, title = {Search engines: information retrieval in practice}, publisher = {Addison-Wesley}, address = {Boston}, year = {2010}, edition = {1st}, url = {http://www.amazon.com/Search-Engines-Information-Retrieval-Practice/dp/0136072240}, isbn = {9780136072249 0136072240}, keywords = {engine, information, practice, retrieval, search} }

%0 = book %A = Croft, W. Bruce and Metzler, Donald and Strohman, Trevor %C = Boston %D = 2010 %I = Addison-Wesley %T = Search engines: information retrieval in practice %U = http://www.amazon.com/Search-Engines-Information-Retrieval-Practice/dp/0136072240

We present Aardvark, a social search engine. With Aardvark, users ask a question, either by instant message, email, web input, text message, or voice. Aardvark then routes the question to the person in the user's extended social network most likely to be able to answer that question. As compared to a traditional web search engine, where the challenge lies in finding the right document to satisfy a user's information need, the challenge in a social search engine like Aardvark lies in finding the right person to satisfy a user's information need. Further, while trust in a traditional search engine is based on authority, in a social search engine like Aardvark, trust is based on intimacy. We describe how these considerations inform the architecture, algorithms, and user interface of Aardvark, and how they are reflected in the behavior of Aardvark users.

@inproceedings{horowitz2010anatomy, author = {Horowitz, Damon and Kamvar, Sepandar D.}, title = {The anatomy of a large-scale social search engine}, booktitle = {Proceedings of the 19th international conference on World wide web}, publisher = {ACM}, address = {New York, NY, USA}, year = {2010}, pages = {431--440}, url = {http://doi.acm.org/10.1145/1772690.1772735}, doi = {10.1145/1772690.1772735}, isbn = {978-1-60558-799-8}, keywords = {aardvark, computing, engine, search, social, web}, abstract = {We present Aardvark, a social search engine. With Aardvark, users ask a question, either by instant message, email, web input, text message, or voice. Aardvark then routes the question to the person in the user's extended social network most likely to be able to answer that question. As compared to a traditional web search engine, where the challenge lies in finding the right document to satisfy a user's information need, the challenge in a social search engine like Aardvark lies in finding the right person to satisfy a user's information need. Further, while trust in a traditional search engine is based on authority, in a social search engine like Aardvark, trust is based on intimacy. We describe how these considerations inform the architecture, algorithms, and user interface of Aardvark, and how they are reflected in the behavior of Aardvark users.} }

%0 = inproceedings %A = Horowitz, Damon and Kamvar, Sepandar D. %B = Proceedings of the 19th international conference on World wide web %C = New York, NY, USA %D = 2010 %I = ACM %T = The anatomy of a large-scale social search engine %U = http://doi.acm.org/10.1145/1772690.1772735

Web search queries capture the information need of search engine users. Search engines store these queries in their logs and analyze them to guide their search results.
this work, we argue that not only a search engine can benefit from data stored in these logs, but also the web users. We first show how clickthrough logs can be collected in a distributed fashion using the http referer field in web server access logs. We then perform a set of experiments to study the information value of search engine queries when treated as "tags" or "labels" for the web pages that both appear as a result and the user actually clicks on. We ask how much extra information these query tags provide for web pages
comparing them to tags from the del.icio.us bookmarking site and to the pagetext. We find that query tags can provide substantially many (on average 250 tags per URL), new tags (on average 125 tags per URL are not present in the pagetext) for a large fraction of the Web.

@inproceedings{antonellis2009tagging, author = {Antonellis, Ioannis and Garcia-Molina, Hector and Karim, Jawed}, title = {Tagging with Queries: How and Why?}, editor = {Baeza-Yates, Ricardo A. and Boldi, Paolo and Ribeiro-Neto, Berthier A. and Cambazoglu, Berkant Barla}, booktitle = {WSDM (Late Breaking-Results)}, publisher = {ACM}, address = {New York, NY, USA}, year = {2009}, url = {http://www.wsdm2009.org/wsdm2009_antonellis.pdf}, isbn = {978-1-60558-390-7}, keywords = {engine, logsonomy, query-log-analysis, search, tagging}, abstract = {Web search queries capture the information need of search engine users. Search engines store these queries in their logs and analyze them to guide their search results.
this work, we argue that not only a search engine can benefit from data stored in these logs, but also the web users. We first show how clickthrough logs can be collected in a distributed fashion using the http referer field in web server access logs. We then perform a set of experiments to study the information value of search engine queries when treated as "tags" or "labels" for the web pages that both appear as a result and the user actually clicks on. We ask how much extra information these query tags provide for web pages
comparing them to tags from the del.icio.us bookmarking site and to the pagetext. We find that query tags can provide substantially many (on average 250 tags per URL), new tags (on average 125 tags per URL are not present in the pagetext) for a large fraction of the Web.} }

%0 = inproceedings %A = Antonellis, Ioannis and Garcia-Molina, Hector and Karim, Jawed %B = WSDM (Late Breaking-Results) %C = New York, NY, USA %D = 2009 %I = ACM %T = Tagging with Queries: How and Why? %U = http://www.wsdm2009.org/wsdm2009_antonellis.pdf

Dieser Beitrag untersucht, in welchem Umfang Dokumente von Dokumentenservern wissenschaftlicher Institutionen in den allgemeinen Suchmaschinen Google und Yahoo nachgewiesen sind und inwieweit wissenschaftliche Suchmaschinen für die Suche nach solchen Dokumenten besser geeignet sind. Dazu werden die fünf Suchmaschinen BASE, Google Scholar, OAIster, Scientific Commons und Scirus überblickartig beschrieben und miteinander verglichen. Hauptaugenmerk wird dabei auf die unterschiedlichen Inhalte, Suchfunktionen und Ausgabemöglichkeiten gelegt, mit Hilfe eines Retrievaltests wird speziell die Leistungsfähigkeit der Suchmaschinen beim Auffinden von Dokumenten, deren Volltexte im Sinne des Open Access direkt und ohne Beschränkungen aufrufbar sind, untersucht.

@incollection{pieper2009wissenschaftliche, author = {Pieper, Dirk and Wolf, Sebastian}, title = {Wissenschaftliche Dokumente in Suchmaschinen}, editor = {Dirk, Lewandowski}, booktitle = {Handbuch Internet-Suchmaschinen: Nutzerorientierung in Wissenschaft und Praxis}, publisher = {Akademische Verlagsgesellschaft AKA}, year = {2009}, pages = {356--374}, url = {http://eprints.rclis.org/12746/}, keywords = {engine, gaw, publication, research, science, search}, abstract = {Dieser Beitrag untersucht, in welchem Umfang Dokumente von Dokumentenservern wissenschaftlicher Institutionen in den allgemeinen Suchmaschinen Google und Yahoo nachgewiesen sind und inwieweit wissenschaftliche Suchmaschinen für die Suche nach solchen Dokumenten besser geeignet sind. Dazu werden die fünf Suchmaschinen BASE, Google Scholar, OAIster, Scientific Commons und Scirus überblickartig beschrieben und miteinander verglichen. Hauptaugenmerk wird dabei auf die unterschiedlichen Inhalte, Suchfunktionen und Ausgabemöglichkeiten gelegt, mit Hilfe eines Retrievaltests wird speziell die Leistungsfähigkeit der Suchmaschinen beim Auffinden von Dokumenten, deren Volltexte im Sinne des Open Access direkt und ohne Beschränkungen aufrufbar sind, untersucht.} }

%0 = incollection %A = Pieper, Dirk and Wolf, Sebastian %B = Handbuch Internet-Suchmaschinen: Nutzerorientierung in Wissenschaft und Praxis %D = 2009 %I = Akademische Verlagsgesellschaft AKA %T = Wissenschaftliche Dokumente in Suchmaschinen %U = http://eprints.rclis.org/12746/

In social bookmarking systems users describe bookmarks by keywords called tags. The structure behind these social systems, called folksonomies, can be viewed as a tripartite hypergraph of user, tag and resource nodes. This underlying network shows specific structural properties that explain its growth and the possibility of serendipitous exploration. Search engines filter the vast information of the web. Queries describe a user’s information need. In response to the displayed results of the search engine, users click on the links of the result page as they expect the answer to be of relevance. The clickdata can be represented as a folksonomy in which queries are descriptions of clicked URLs. This poster analyzes the topological characteristics of the resulting tripartite hypergraph of queries, users and bookmarks of two query logs and compares it two a snapshot of the folksonomy del.icio.us.

@inproceedings{Jaeschke2008logsonomy, author = {Jäschke, Robert and Krause, Beate and Hotho, Andreas and Stumme, Gerd}, title = {Logsonomy -- A Search Engine Folksonomy}, booktitle = {Proceedings of the Second International Conference on Weblogs and Social Media (ICWSM 2008)}, publisher = {AAAI Press}, address = {Menlo Park, CA, USA}, year = {2008}, pages = {192--193}, url = {http://www.kde.cs.uni-kassel.de/hotho/pub/2008/Krause2008logsonomy_short.pdf}, isbn = {978-1-57735-355-3}, keywords = {2008, engine, folksonomy, l3s, logsonomy, myown, search, wp5}, abstract = {In social bookmarking systems users describe bookmarks by keywords called tags. The structure behind these social systems, called folksonomies, can be viewed as a tripartite hypergraph of user, tag and resource nodes. This underlying network shows specific structural properties that explain its growth and the possibility of serendipitous exploration. Search engines filter the vast information of the web. Queries describe a user’s information need. In response to the displayed results of the search engine, users click on the links of the result page as they expect the answer to be of relevance. The clickdata can be represented as a folksonomy in which queries are descriptions of clicked URLs. This poster analyzes the topological characteristics of the resulting tripartite hypergraph of queries, users and bookmarks of two query logs and compares it two a snapshot of the folksonomy del.icio.us. } }

%0 = inproceedings %A = Jäschke, Robert and Krause, Beate and Hotho, Andreas and Stumme, Gerd %B = Proceedings of the Second International Conference on Weblogs and Social Media (ICWSM 2008) %C = Menlo Park, CA, USA %D = 2008 %I = AAAI Press %T = Logsonomy -- A Search Engine Folksonomy %U = http://www.kde.cs.uni-kassel.de/hotho/pub/2008/Krause2008logsonomy_short.pdf

Social bookmarking systems constitute an established part of the Web 2.0. In such systems users describe bookmarks by keywords called tags. The structure behind these social systems, called folksonomies, can be viewed as a tripartite hypergraph of user, tag and resource nodes. This underlying network shows specific structural properties that explain its growth and the possibility of serendipitous exploration. Today’s search engines represent the gateway to retrieve information from the World Wide Web. Short queries typically consisting of two to three words describe a user’s information need. In response to the displayed results of the search engine, users click on the links of the result page as they expect the answer to be of relevance. This clickdata can be represented as a folksonomy in which queries are descriptions of clicked URLs. The resulting network structure, which we will term logsonomy is very similar to the one of folksonomies. In order to find out about its properties, we analyze the topological characteristics of the tripartite hypergraph of queries, users and bookmarks on a large snapshot of del.icio.us and on query logs of two large search engines. All of the three datasets show small world properties. The tagging behavior of users, which is explained by preferential attachment of the tags in social bookmark systems, is reflected in the distribution of single query words in search engines. We can conclude that the clicking behaviour of search engine users based on the displayed search results and the tagging behaviour of social bookmarking users is driven by similar dynamics.

@inproceedings{krause2008logsonomy, author = {Krause, Beate and Jäschke, Robert and Hotho, Andreas and Stumme, Gerd}, title = {Logsonomy - Social Information Retrieval with Logdata}, booktitle = {HT '08: Proceedings of the Nineteenth ACM Conference on Hypertext and Hypermedia}, publisher = {ACM}, address = {New York, NY, USA}, year = {2008}, pages = {157--166}, url = {http://www.kde.cs.uni-kassel.de/pub/pdf/krause2008logsonomy.pdf}, doi = {10.1145/1379092.1379123}, isbn = {978-1-59593-985-2}, keywords = {2008, analysis, engine, information, l3s, logsonomy, myown, network, retrieval, search, sna, social, wp5}, abstract = {Social bookmarking systems constitute an established part of the Web 2.0. In such systems users describe bookmarks by keywords called tags. The structure behind these social systems, called folksonomies, can be viewed as a tripartite hypergraph of user, tag and resource nodes. This underlying network shows specific structural properties that explain its growth and the possibility of serendipitous exploration. Today’s search engines represent the gateway to retrieve information from the World Wide Web. Short queries typically consisting of two to three words describe a user’s information need. In response to the displayed results of the search engine, users click on the links of the result page as they expect the answer to be of relevance. This clickdata can be represented as a folksonomy in which queries are descriptions of clicked URLs. The resulting network structure, which we will term logsonomy is very similar to the one of folksonomies. In order to find out about its properties, we analyze the topological characteristics of the tripartite hypergraph of queries, users and bookmarks on a large snapshot of del.icio.us and on query logs of two large search engines. All of the three datasets show small world properties. The tagging behavior of users, which is explained by preferential attachment of the tags in social bookmark systems, is reflected in the distribution of single query words in search engines. We can conclude that the clicking behaviour of search engine users based on the displayed search results and the tagging behaviour of social bookmarking users is driven by similar dynamics.} }

%0 = inproceedings %A = Krause, Beate and Jäschke, Robert and Hotho, Andreas and Stumme, Gerd %B = HT '08: Proceedings of the Nineteenth ACM Conference on Hypertext and Hypermedia %C = New York, NY, USA %D = 2008 %I = ACM %T = Logsonomy - Social Information Retrieval with Logdata %U = http://www.kde.cs.uni-kassel.de/pub/pdf/krause2008logsonomy.pdf

It is crucial for a web crawler to distinguish between ephemeral and persistent content. Ephemeral content (e.g., quote of the day) is usually not worth crawling, because by the time it reaches the index it is no longer representative of the web page from which it was acquired. On the other hand, content that persists across multiple page updates (e.g., recent blog postings) may be worth acquiring, because it matches the page's true content for a sustained period of time.</p> <p>In this paper we characterize the longevity of information found on the web, via both empirical measurements and a generative model that coincides with these measurements. We then develop new recrawl scheduling policies that take longevity into account. As we show via experiments over real web data, our policies obtain better freshness at lower cost, compared with previous approaches.

@inproceedings{olston2008recrawl, author = {Olston, Christopher and Pandey, Sandeep}, title = {Recrawl scheduling based on information longevity}, booktitle = {Proceedings of the 17th international conference on World Wide Web}, publisher = {ACM}, address = {New York, NY, USA}, year = {2008}, pages = {437--446}, url = {http://doi.acm.org/10.1145/1367497.1367557}, doi = {10.1145/1367497.1367557}, isbn = {978-1-60558-085-2}, keywords = {crawling, engine, search, web}, abstract = {It is crucial for a web crawler to distinguish between ephemeral and persistent content. Ephemeral content (e.g., quote of the day) is usually not worth crawling, because by the time it reaches the index it is no longer representative of the web page from which it was acquired. On the other hand, content that persists across multiple page updates (e.g., recent blog postings) may be worth acquiring, because it matches the page's true content for a sustained period of time.</p> <p>In this paper we characterize the longevity of information found on the web, via both empirical measurements and a generative model that coincides with these measurements. We then develop new recrawl scheduling policies that take longevity into account. As we show via experiments over real web data, our policies obtain better freshness at lower cost, compared with previous approaches.} }

%0 = inproceedings %A = Olston, Christopher and Pandey, Sandeep %B = Proceedings of the 17th international conference on World Wide Web %C = New York, NY, USA %D = 2008 %I = ACM %T = Recrawl scheduling based on information longevity %U = http://doi.acm.org/10.1145/1367497.1367557

Crawling algorithms have been the subject of extensive research and optimizations, but some important questions remain open. In particular, given the unbounded number of pages available on the Web, search-engine operators constantly struggle with the following vexing questions: <i>When can I stop downloading the Web? How many pages should I download to cover "most" of the Web? How can I know I am not missing an important part when I stop?</i> In this paper we provide an answer to these questions by developing, in the context of a system that is given a set of trusted pages, a family of crawling algorithms that (1) provide a theoretical guarantee on how much of the "important" part of the Web it will download after crawling a certain number of pages and (2) give a high priority to important pages during a crawl, so that the search engine can index the most important part of the Web first. We prove the correctness of our algorithms by theoretical analysis and evaluate their performance experimentally based on 141 million URLs obtained from the Web. Our experiments demonstrate that even our simple algorithm is effective in downloading important pages early on and provides high "coverage" of the Web with a relatively small number of pages.

@inproceedings{cho2007rankmass, author = {Cho, Junghoo and Schonfeld, Uri}, title = {RankMass crawler: a crawler with high personalized pagerank coverage guarantee}, booktitle = {Proceedings of the 33rd international conference on Very large data bases}, publisher = {VLDB Endowment}, year = {2007}, pages = {375--386}, url = {http://dl.acm.org/citation.cfm?id=1325851.1325897}, isbn = {978-1-59593-649-3}, keywords = {crawling, engine, pagerank, search, web}, abstract = {Crawling algorithms have been the subject of extensive research and optimizations, but some important questions remain open. In particular, given the unbounded number of pages available on the Web, search-engine operators constantly struggle with the following vexing questions: <i>When can I stop downloading the Web? How many pages should I download to cover "most" of the Web? How can I know I am not missing an important part when I stop?</i> In this paper we provide an answer to these questions by developing, in the context of a system that is given a set of trusted pages, a family of crawling algorithms that (1) provide a theoretical guarantee on how much of the "important" part of the Web it will download after crawling a certain number of pages and (2) give a high priority to important pages during a crawl, so that the search engine can index the most important part of the Web first. We prove the correctness of our algorithms by theoretical analysis and evaluate their performance experimentally based on 141 million URLs obtained from the Web. Our experiments demonstrate that even our simple algorithm is effective in downloading important pages early on and provides high "coverage" of the Web with a relatively small number of pages.} }

%0 = inproceedings %A = Cho, Junghoo and Schonfeld, Uri %B = Proceedings of the 33rd international conference on Very large data bases %D = 2007 %I = VLDB Endowment %T = RankMass crawler: a crawler with high personalized pagerank coverage guarantee %U = http://dl.acm.org/citation.cfm?id=1325851.1325897

Search engines are the primary gateways of information access on the Web today. Behind the scenes, search engines crawl the Web to populate a local indexed repository of Web pages, used to answer user search queries. In an aggregate sense, the Web is very dynamic, causing any repository of Web pages to become out of date over time, which in turn causes query answer quality to degrade. Given the considerable size, dynamicity, and degree of autonomy of the Web as a whole, it is not feasible for a search engine to maintain its repository exactly synchronized with the Web.In this paper we study how to schedule Web pages for selective (re)downloading into a search engine repository. The scheduling objective is to maximize the quality of the user experience for those who query the search engine. We begin with a quantitative characterization of the way in which the discrepancy between the content of the repository and the current content of the live Web impacts the quality of the user experience. This characterization leads to a <i>user-centric</i> metric of the quality of a search engine's local repository. We use this metric to derive a policy for scheduling Web page (re)downloading that is driven by search engine usage and free of exterior tuning parameters. We then focus on the important subproblem of scheduling refreshing of Web pages already present in the repository, and show how to compute the priorities efficiently. We provide extensive empirical comparisons of our user-centric method against prior Web page refresh strategies, using real Web data. Our results demonstrate that our method requires far fewer resources to maintain same search engine quality level for users, leaving substantially more resources available for incorporating new Web pages into the search repository.

@inproceedings{pandey2005usercentric, author = {Pandey, Sandeep and Olston, Christopher}, title = {User-centric Web crawling}, booktitle = {Proceedings of the 14th international conference on World Wide Web}, publisher = {ACM}, address = {New York, NY, USA}, year = {2005}, pages = {401--411}, url = {http://doi.acm.org/10.1145/1060745.1060805}, doi = {10.1145/1060745.1060805}, isbn = {1-59593-046-9}, keywords = {crawling, engine, search, web}, abstract = {Search engines are the primary gateways of information access on the Web today. Behind the scenes, search engines crawl the Web to populate a local indexed repository of Web pages, used to answer user search queries. In an aggregate sense, the Web is very dynamic, causing any repository of Web pages to become out of date over time, which in turn causes query answer quality to degrade. Given the considerable size, dynamicity, and degree of autonomy of the Web as a whole, it is not feasible for a search engine to maintain its repository exactly synchronized with the Web.In this paper we study how to schedule Web pages for selective (re)downloading into a search engine repository. The scheduling objective is to maximize the quality of the user experience for those who query the search engine. We begin with a quantitative characterization of the way in which the discrepancy between the content of the repository and the current content of the live Web impacts the quality of the user experience. This characterization leads to a <i>user-centric</i> metric of the quality of a search engine's local repository. We use this metric to derive a policy for scheduling Web page (re)downloading that is driven by search engine usage and free of exterior tuning parameters. We then focus on the important subproblem of scheduling refreshing of Web pages already present in the repository, and show how to compute the priorities efficiently. We provide extensive empirical comparisons of our user-centric method against prior Web page refresh strategies, using real Web data. Our results demonstrate that our method requires far fewer resources to maintain same search engine quality level for users, leaving substantially more resources available for incorporating new Web pages into the search repository.} }

%0 = inproceedings %A = Pandey, Sandeep and Olston, Christopher %B = Proceedings of the 14th international conference on World Wide Web %C = New York, NY, USA %D = 2005 %I = ACM %T = User-centric Web crawling %U = http://doi.acm.org/10.1145/1060745.1060805

rrent search technologies work in a "one size fits all" fashion. Therefore, the answer to a query is independent of specific user information need. In this paper we describe a novel ranking technique for personalized search servicesthat combines content-based and community-based evidences. The community-based information is used in order to provide context for queries andis influenced by the current interaction of the user with the service. Ouralgorithm is evaluated using data derived from an actual service available on the Web an online bookstore. We show that the quality of content-based ranking strategies can be improved by the use of communityinformation as another evidential source of relevance. In our experiments the improvements reach up to 48% in terms of average precision.

@inproceedings{988728, author = {Almeida, Rodrigo B. and Almeida, Virgilio A. F.}, title = {A community-aware search engine}, booktitle = {Proceedings of the 13th international conference on World Wide Web}, publisher = {ACM Press}, address = {New York, NY, USA}, year = {2004}, pages = {413--421}, url = {http://doi.acm.org/10.1145/988672.988728}, isbn = {1-58113-844-X}, keywords = {search, engine, detection, hits, community, network}, abstract = {
rrent search technologies work in a "one size fits all" fashion. Therefore, the answer to a query is independent of specific user information need. In this paper we describe a novel ranking technique for personalized search servicesthat combines content-based and community-based evidences. The community-based information is used in order to provide context for queries andis influenced by the current interaction of the user with the service. Ouralgorithm is evaluated using data derived from an actual service available on the Web an online bookstore. We show that the quality of content-based ranking strategies can be improved by the use of communityinformation as another evidential source of relevance. In our experiments the improvements reach up to 48% in terms of average precision.} }

%0 = inproceedings %A = Almeida, Rodrigo B. and Almeida, Virgilio A. F. %B = Proceedings of the 13th international conference on World Wide Web %C = New York, NY, USA %D = 2004 %I = ACM Press %T = A community-aware search engine %U = http://doi.acm.org/10.1145/988672.988728

The key factors for the success of the World Wide Web are its large size and the lack of a centralized control over its contents. Both issues are also the most important source of problems for locating information. The Web is a context in which traditional Information Retrieval methods are challenged, and given the volume of the Web and its speed of change, the coverage of modern search engines is relatively small. Moreover, the distribution of quality is very skewed, and interesting pages are scarce in comparison with the rest of the content. Web crawling is the process used by search engines to collect pages from the Web. This thesis studies Web crawling at several different levels, ranging from the long-term goal of crawling important pages first, to the short-term goal of using the network connectivity efficiently, including implementation issues that are essential for crawling in practice. We start by designing a new model and architecture for aWeb crawler that tightly integrates the crawler with the rest of the search engine, providing access to the metadata and links of the documents that can be used to guide the crawling process effectively. We implement this design in the WIRE project as an efficient Web crawler that provides an experimental framework for this research. In fact, we have used our crawler to characterize the Chilean Web, using the results as feedback to improve the crawler design. We argue that the number of pages on the Web can be considered infinite, and given that a Web crawler cannot download all the pages, it is important to capture the most important ones as early as possible during the crawling process. We propose, study, and implement algorithms for achieving this goal, showing that we can crawl 50% of a large Web collection and capture 80% of its total Pagerank value in both simulated and real Web environments. We also model and study user browsing behavior in Web sites, concluding that it is not necessary to go deeper than five levels from the home page to capture most of the pages actually visited by people, and support this conclusion with log analysis of several Web sites. We also propose several mechanisms for server cooperation to reduce network traffic and improve the representation of aWeb page in a search engine with the help of Web site managers.

@phdthesis{castillo2004effective, author = {Castillo, Carlos}, title = {Effective Web Crawling}, school = {School of Engineering}, address = {Santiago, Chile}, year = {2004}, url = {http://www.chato.cl/crawling/}, keywords = {crawling, engine, search, web}, abstract = {The key factors for the success of the World Wide Web are its large size and the lack of a centralized control over its contents. Both issues are also the most important source of problems for locating information. The Web is a context in which traditional Information Retrieval methods are challenged, and given the volume of the Web and its speed of change, the coverage of modern search engines is relatively small. Moreover, the distribution of quality is very skewed, and interesting pages are scarce in comparison with the rest of the content. Web crawling is the process used by search engines to collect pages from the Web. This thesis studies Web crawling at several different levels, ranging from the long-term goal of crawling important pages first, to the short-term goal of using the network connectivity efficiently, including implementation issues that are essential for crawling in practice. We start by designing a new model and architecture for aWeb crawler that tightly integrates the crawler with the rest of the search engine, providing access to the metadata and links of the documents that can be used to guide the crawling process effectively. We implement this design in the WIRE project as an efficient Web crawler that provides an experimental framework for this research. In fact, we have used our crawler to characterize the Chilean Web, using the results as feedback to improve the crawler design. We argue that the number of pages on the Web can be considered infinite, and given that a Web crawler cannot download all the pages, it is important to capture the most important ones as early as possible during the crawling process. We propose, study, and implement algorithms for achieving this goal, showing that we can crawl 50% of a large Web collection and capture 80% of its total Pagerank value in both simulated and real Web environments. We also model and study user browsing behavior in Web sites, concluding that it is not necessary to go deeper than five levels from the home page to capture most of the pages actually visited by people, and support this conclusion with log analysis of several Web sites. We also propose several mechanisms for server cooperation to reduce network traffic and improve the representation of aWeb page in a search engine with the help of Web site managers.} }

%0 = phdthesis %A = Castillo, Carlos %C = Santiago, Chile %D = 2004 %T = Effective Web Crawling %U = http://www.chato.cl/crawling/

@inproceedings{conf/cikm/DingFJPCPRDS04, author = {Ding, Li and Finin, Timothy W. and Joshi, Anupam and Pan, Rong and Cost, R. Scott and Peng, Yun and Reddivari, Pavan and Doshi, Vishal and Sachs, Joel}, title = {Swoogle: a search and metadata engine for the semantic web.}, booktitle = {CIKM}, year = {2004}, pages = {652-659}, url = {http://dblp.uni-trier.de/db/conf/cikm/cikm2004.html#DingFJPCPRDS04}, keywords = {search, web, engine, metadate, swoogle, semantic, seminar2006} }

%0 = inproceedings %A = Ding, Li and Finin, Timothy W. and Joshi, Anupam and Pan, Rong and Cost, R. Scott and Peng, Yun and Reddivari, Pavan and Doshi, Vishal and Sachs, Joel %B = CIKM %D = 2004 %T = Swoogle: a search and metadata engine for the semantic web. %U = http://dblp.uni-trier.de/db/conf/cikm/cikm2004.html#DingFJPCPRDS04

Web usage mining can be very useful to search engines. This paper proposes a novel effective approach to exploit the relationships among users, queries and resources based on the search engine's log. How this method can be applied is illustrated by a Chinese image search engine.

@article{zhang2002web, author = {Zhang, Dell and Dong, Yisheng}, title = {A novel Web usage mining approach for search engines}, editor = {Akyildiz, Ian F. and Rudin, Harry}, journal = {Computer Networks}, publisher = {Elsevier}, year = {2002}, volume = {39}, number = {3}, pages = {303--310}, url = {http://www.sciencedirect.com/science/article/B6VRG-45H0GV7-5/2/16726cebdcde67ba7aeb95cc91e797bf}, doi = {10.1016/S1389-1286(02)00211-6}, issn = {1389-1286}, keywords = {engine, mining, search, usage, web}, abstract = {Web usage mining can be very useful to search engines. This paper proposes a novel effective approach to exploit the relationships among users, queries and resources based on the search engine's log. How this method can be applied is illustrated by a Chinese image search engine.} }

%0 = article %A = Zhang, Dell and Dong, Yisheng %D = 2002 %I = Elsevier %T = A novel Web usage mining approach for search engines %U = http://www.sciencedirect.com/science/article/B6VRG-45H0GV7-5/2/16726cebdcde67ba7aeb95cc91e797bf