We report a novel attack on two CAPTCHAs that have been widely deployed on the Internet, one being Google's home design and the other acquired by Google (i.e. reCAPTCHA). With a minor change, our attack program also works well on the latest ReCAPTCHA version, which uses a new defence mechanism that was unknown to us when we designed our attack. This suggests that our attack works in a fundamental level. Our attack appears to be applicable to a whole family of text CAPTCHAs that build on top of the popular segmentation-resistant mechanism of "crowding character together" for security. Next, we propose a novel framework that guides the application of our well-tested security engineering methodology for evaluating CAPTCHA robustness, and we propose a new general principle for CAPTCHA design.

@techreport{elahmad2011robustness, author = {El Ahmad, Ahmad S and Yan, Jeff and Tayara, Mohamad}, title = {The Robustness of Google CAPTCHAs}, year = {2011}, url = {http://homepages.cs.ncl.ac.uk/jeff.yan/google.pdf}, keywords = {captcha, character, google, image, ocr, recognition, segmentation}, abstract = {We report a novel attack on two CAPTCHAs that have been widely deployed on the Internet, one being Google's home design and the other acquired by Google (i.e. reCAPTCHA). With a minor change, our attack program also works well on the latest ReCAPTCHA version, which uses a new defence mechanism that was unknown to us when we designed our attack. This suggests that our attack works in a fundamental level. Our attack appears to be applicable to a whole family of text CAPTCHAs that build on top of the popular segmentation-resistant mechanism of "crowding character together" for security. Next, we propose a novel framework that guides the application of our well-tested security engineering methodology for evaluating CAPTCHA robustness, and we propose a new general principle for CAPTCHA design. } }

%0 = techreport %A = El Ahmad, Ahmad S and Yan, Jeff and Tayara, Mohamad %B = } %C = %D = 2011 %I = %T = The Robustness of Google CAPTCHAs} %U = http://homepages.cs.ncl.ac.uk/jeff.yan/google.pdf

We systematically study the design of image recognition CAPTCHAs (IRCs) in this paper. We first review and examine all existing IRCs schemes and evaluate each scheme against the practical requirements in CAPTCHA applications, particularly in large-scale real-life applications such as Gmail and Hotmail. Then we present a security analysis of the representative schemes we have identified. For the schemes that remain unbroken, we present our novel attacks. For the schemes for which known attacks are available, we propose a theoretical explanation why those schemes have failed. Next, we provide a simple but novel framework for guiding the design of robust IRCs. Then we propose an innovative IRC called Cortcha that is scalable to meet the requirements of large-scale applications. It relies on recognizing objects by exploiting the surrounding context, a task that humans can perform well but computers cannot. An infinite number of types of objects can be used to generate challenges, which can effectively disable the learning process in machine learning attacks. Cortcha does not require the images in its image database to be labeled. Image collection and CAPTCHA generation can be fully automated. Our usability studies indicate that, compared with Google's text CAPTCHA, Cortcha allows a slightly higher human accuracy rate but on average takes more time to solve a challenge.

@inproceedings{zhu2010attacks, author = {Zhu, Bin B. and Yan, Jeff and Li, Qiujie and Yang, Chao and Liu, Jia and Xu, Ning and Yi, Meng and Cai, Kaiwei}, title = {Attacks and design of image recognition CAPTCHAs}, booktitle = {CCS '10: Proceedings of the 17th ACM conference on Computer and communications security}, publisher = {ACM}, address = {New York, NY, USA}, year = {2010}, pages = {187--200}, url = {http://portal.acm.org/citation.cfm?id=1866307.1866329}, doi = {10.1145/1866307.1866329}, isbn = {978-1-4503-0245-6}, keywords = {captcha, image, recognition, security, web}, abstract = {We systematically study the design of image recognition CAPTCHAs (IRCs) in this paper. We first review and examine all existing IRCs schemes and evaluate each scheme against the practical requirements in CAPTCHA applications, particularly in large-scale real-life applications such as Gmail and Hotmail. Then we present a security analysis of the representative schemes we have identified. For the schemes that remain unbroken, we present our novel attacks. For the schemes for which known attacks are available, we propose a theoretical explanation why those schemes have failed. Next, we provide a simple but novel framework for guiding the design of robust IRCs. Then we propose an innovative IRC called Cortcha that is scalable to meet the requirements of large-scale applications. It relies on recognizing objects by exploiting the surrounding context, a task that humans can perform well but computers cannot. An infinite number of types of objects can be used to generate challenges, which can effectively disable the learning process in machine learning attacks. Cortcha does not require the images in its image database to be labeled. Image collection and CAPTCHA generation can be fully automated. Our usability studies indicate that, compared with Google's text CAPTCHA, Cortcha allows a slightly higher human accuracy rate but on average takes more time to solve a challenge.} }

%0 = inproceedings %A = Zhu, Bin B. and Yan, Jeff and Li, Qiujie and Yang, Chao and Liu, Jia and Xu, Ning and Yi, Meng and Cai, Kaiwei %B = CCS '10: Proceedings of the 17th ACM conference on Computer and communications security %C = New York, NY, USA %D = 2010 %I = ACM %T = Attacks and design of image recognition CAPTCHAs %U = http://portal.acm.org/citation.cfm?id=1866307.1866329

Image annotation is hard to do in an automatic way. In this paper, we propose a framework for image annotation that combines the benefits of three paradigms: automatic annotation, human intervention and entertainment activities. We also describe our proposal inside this framework, the ASAA (application for semi-automatic annotation) interface, a new computer game for image tagging. The application has a 3D game interface, and is supported by a game engine that uses a system for automatic image classification and gestural input to play the game. We present results of the performance of semantic models obtained with a training set enlarged by images annotated during the game activity as well as usability tests of the application.

@article{4563045, author = {Jesus, R. and Goncalves, D. and Abrantes, A.J. and Correia, N.}, title = {Playing games as a way to improve automatic image annotation}, journal = {Computer Vision and Pattern Recognition Workshops, 2008. CVPR Workshops 2008. IEEE Computer Society Conference on}, year = {2008}, pages = {1-8}, doi = {10.1109/CVPRW.2008.4563045}, keywords = {annotation, automatic, classification, game, image}, abstract = {Image annotation is hard to do in an automatic way. In this paper, we propose a framework for image annotation that combines the benefits of three paradigms: automatic annotation, human intervention and entertainment activities. We also describe our proposal inside this framework, the ASAA (application for semi-automatic annotation) interface, a new computer game for image tagging. The application has a 3D game interface, and is supported by a game engine that uses a system for automatic image classification and gestural input to play the game. We present results of the performance of semantic models obtained with a training set enlarged by images annotated during the game activity as well as usability tests of the application.} }

%0 = article %A = Jesus, R. and Goncalves, D. and Abrantes, A.J. and Correia, N. %D = 2008 %T = Playing games as a way to improve automatic image annotation

We introduce an interactive framework for image
derstanding, a game that is enjoyable and provide valuable
age annotations. When people play the game, they provide
eful information about contents of an image. In reality the
st accurate method to describe the content of an image is
nual labelling. Our approach is to motivate people to label
agers while entertaining themselves. Therefore if this game
comes popular it will be able to annotate most imagers on
e web within a couple of months. When considering
curacy we use a combination of computer vision techniques
secure the accuracy of image labelling. By doing this we
lieve our system will make a significant contribution to
dress the semantic gap in the computer vision sector.

@inproceedings{seneviratne2008image, author = {Seneviratne, Lasantha and Izquierdo, Ebroul}, title = {Image Annotation Through Gaming}, editor = {Simone, Francesca De and Nemrava, Jan and Bailer, Werner}, booktitle = {Proceedings of the 2nd K-Space PhD Jamboree Workshop}, publisher = {CEUR-WS}, year = {2008}, url = {http://ftp.informatik.rwth-aachen.de/Publications/CEUR-WS/Vol-379/paper8.pdf}, keywords = {annotation, game, image}, abstract = {We introduce an interactive framework for image
derstanding, a game that is enjoyable and provide valuable
age annotations. When people play the game, they provide
eful information about contents of an image. In reality the
st accurate method to describe the content of an image is
nual labelling. Our approach is to motivate people to label
agers while entertaining themselves. Therefore if this game
comes popular it will be able to annotate most imagers on
e web within a couple of months. When considering
curacy we use a combination of computer vision techniques
secure the accuracy of image labelling. By doing this we
lieve our system will make a significant contribution to
dress the semantic gap in the computer vision sector.
} }

%0 = inproceedings %A = Seneviratne, Lasantha and Izquierdo, Ebroul %B = Proceedings of the 2nd K-Space PhD Jamboree Workshop %D = 2008 %I = CEUR-WS %T = Image Annotation Through Gaming %U = http://ftp.informatik.rwth-aachen.de/Publications/CEUR-WS/Vol-379/paper8.pdf

This paper investigates new approaches to improve the efficiency of manual image annotation and help users to produce better annotation results in a given amount of time. Although important in practice, this issue has rarely been studied in a quantitative way before. To achieve this, we first propose two time models to analyze the annotation process for two popular manual annotation approaches, i.e., tagging and browsing. The complementary properties of these approaches have inspired us to merge them to develop a hybrid annotation algorithms called frequency-based annotation. Our experiments on large-scale multimedia collections have shown that the proposed algorithm can achieve an up to 40% annotation time reduction compared with the baseline methods. In other words, it can produce considerably better results using the same annotation time.

@inproceedings{yan2007efficient, author = {Yan, Rong and Natsev, Apostol and Campbell, Murray}, title = {An efficient manual image annotation approach based on tagging and browsing}, booktitle = {MS '07: Workshop on multimedia information retrieval on The many faces of multimedia semantics}, publisher = {ACM}, address = {New York, NY, USA}, year = {2007}, pages = {13--20}, url = {http://portal.acm.org/citation.cfm?id=1290071}, doi = {http://doi.acm.org/10.1145/1290067.1290071}, isbn = {978-1-59593-782-7}, keywords = {annotation, browsing, image, manual, tagging}, abstract = {This paper investigates new approaches to improve the efficiency of manual image annotation and help users to produce better annotation results in a given amount of time. Although important in practice, this issue has rarely been studied in a quantitative way before. To achieve this, we first propose two time models to analyze the annotation process for two popular manual annotation approaches, i.e., tagging and browsing. The complementary properties of these approaches have inspired us to merge them to develop a hybrid annotation algorithms called frequency-based annotation. Our experiments on large-scale multimedia collections have shown that the proposed algorithm can achieve an up to 40% annotation time reduction compared with the baseline methods. In other words, it can produce considerably better results using the same annotation time.} }

%0 = inproceedings %A = Yan, Rong and Natsev, Apostol and Campbell, Murray %B = MS '07: Workshop on multimedia information retrieval on The many faces of multimedia semantics %C = New York, NY, USA %D = 2007 %I = ACM %T = An efficient manual image annotation approach based on tagging and browsing %U = http://portal.acm.org/citation.cfm?id=1290071

We introduce a new interactive system: a game that is fun and can be used to create valuable output. When people play the game they help determine the contents of images by providing meaningful labels for them. If the game is played as much as popular online games, we estimate that most images on the Web can be labeled in a few months. Having proper labels associated with each image on the Web would allow for more accurate image search, improve the accessibility of sites (by providing descriptions of images to visually impaired individuals), and help users block inappropriate images. Our system makes a significant contribution because of its valuable output and because of the way it addresses the image-labeling problem. Rather than using computer vision techniques, which don't work well enough, we encourage people to do the work by taking advantage of their desire to be entertained.

@inproceedings{985733, author = {von Ahn, Luis and Dabbish, Laura}, title = {Labeling images with a computer game}, booktitle = {CHI '04: Proceedings of the SIGCHI conference on Human factors in computing systems}, publisher = {ACM}, address = {New York, NY, USA}, year = {2004}, pages = {319--326}, url = {http://portal.acm.org/citation.cfm?id=985733}, doi = {http://doi.acm.org/10.1145/985692.985733}, isbn = {1-58113-702-8}, keywords = {annotation, game, image, labeling}, abstract = {We introduce a new interactive system: a game that is fun and can be used to create valuable output. When people play the game they help determine the contents of images by providing meaningful labels for them. If the game is played as much as popular online games, we estimate that most images on the Web can be labeled in a few months. Having proper labels associated with each image on the Web would allow for more accurate image search, improve the accessibility of sites (by providing descriptions of images to visually impaired individuals), and help users block inappropriate images. Our system makes a significant contribution because of its valuable output and because of the way it addresses the image-labeling problem. Rather than using computer vision techniques, which don't work well enough, we encourage people to do the work by taking advantage of their desire to be entertained.} }

%0 = inproceedings %A = von Ahn, Luis and Dabbish, Laura %B = CHI '04: Proceedings of the SIGCHI conference on Human factors in computing systems %C = New York, NY, USA %D = 2004 %I = ACM %T = Labeling images with a computer game %U = http://portal.acm.org/citation.cfm?id=985733

@book{lynch1992image, author = {Lynch, Kevin}, title = {The image of the city}, publisher = {MIT Press}, year = {1992}, url = {http://books.google.de/books?id=phRPWsSpAgC}, isbn = {9780262620017}, keywords = {city, geo, geography, image, map, mapping, toread} }

%0 = book %A = Lynch, Kevin %D = 1992 %I = MIT Press %T = The image of the city %U = http://books.google.de/books?id=phRPWsSpAgC