steve
/
factor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

fix unit test in robots

db4
Doug Coleman 2009-04-07 10:08:01 -05:00
parent 63cf5b04e1
commit 65802b6aaa
2 changed files with 324 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
extra/robots/robots-tests.factor
View File

@ -1,26 +1,27 @@
! Copyright (C) 2009 Doug Coleman. ! Copyright (C) 2009 Doug Coleman.
! See http://factorcode.org/license.txt for BSD license. ! See http://factorcode.org/license.txt for BSD license.
USING: calendar io.encodings.utf8 io.files robots tools.test ; USING: calendar io.encodings.utf8 io.files robots tools.test
urls ;
IN: robots.tests IN: robots.tests
[ [
{ "http://www.chiplist.com/sitemap.txt" } { "http://www.chiplist.com/sitemap.txt" }
{ {
T{ rules T{ rules
{ user-agents V{ "*" } } { user-agents V{ "*" } }
{ allows V{ } } { allows V{ } }
{ disallows { disallows
V{ V{
"/cgi-bin/" URL" /cgi-bin/"
"/scripts/" URL" /scripts/"
"/ChipList2/scripts/" URL" /ChipList2/scripts/"
"/ChipList2/styles/" URL" /ChipList2/styles/"
"/ads/" URL" /ads/"
"/ChipList2/ads/" URL" /ChipList2/ads/"
"/advertisements/" URL" /advertisements/"
"/ChipList2/advertisements/" URL" /ChipList2/advertisements/"
"/graphics/" URL" /graphics/"
"/ChipList2/graphics/" URL" /ChipList2/graphics/"
} }
} }
{ visit-time { visit-time
@ -36,163 +37,163 @@ IN: robots.tests
T{ rules T{ rules
{ user-agents V{ "UbiCrawler" } } { user-agents V{ "UbiCrawler" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "DOC" } } { user-agents V{ "DOC" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "Zao" } } { user-agents V{ "Zao" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "sitecheck.internetseer.com" } } { user-agents V{ "sitecheck.internetseer.com" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "Zealbot" } } { user-agents V{ "Zealbot" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "MSIECrawler" } } { user-agents V{ "MSIECrawler" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "SiteSnagger" } } { user-agents V{ "SiteSnagger" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "WebStripper" } } { user-agents V{ "WebStripper" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "WebCopier" } } { user-agents V{ "WebCopier" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "Fetch" } } { user-agents V{ "Fetch" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "Offline Explorer" } } { user-agents V{ "Offline Explorer" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "Teleport" } } { user-agents V{ "Teleport" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "TeleportPro" } } { user-agents V{ "TeleportPro" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "WebZIP" } } { user-agents V{ "WebZIP" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "linko" } } { user-agents V{ "linko" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "HTTrack" } } { user-agents V{ "HTTrack" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "Microsoft.URL.Control" } } { user-agents V{ "Microsoft.URL.Control" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "Xenu" } } { user-agents V{ "Xenu" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "larbin" } } { user-agents V{ "larbin" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "libwww" } } { user-agents V{ "libwww" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "ZyBORG" } } { user-agents V{ "ZyBORG" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "Download Ninja" } } { user-agents V{ "Download Ninja" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "wget" } } { user-agents V{ "wget" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "grub-client" } } { user-agents V{ "grub-client" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "k2spider" } } { user-agents V{ "k2spider" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "NPBot" } } { user-agents V{ "NPBot" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
{ user-agents V{ "WebReaper" } } { user-agents V{ "WebReaper" } }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
T{ rules T{ rules
@ -327,8 +328,8 @@ IN: robots.tests
} }
} }
{ allows V{ } } { allows V{ } }
{ disallows V{ "/" } } { disallows V{ URL" /" } }
{ unknowns H{ } } { unknowns H{ } }
} }
} }
] [ "vocab:robots/robots.txt" utf8 file-contents parse-robots.txt ] unit-test ] [ "vocab:robots/robots.txt" utf8 file-contents parse-robots.txt ] unit-test

2
extra/robots/robots.factor
View File

@ -85,7 +85,7 @@ PRIVATE>
: parse-robots.txt ( string -- sitemaps rules-seq ) : parse-robots.txt ( string -- sitemaps rules-seq )
normalize-robots.txt [ normalize-robots.txt [
[ <rules> dup ] dip [ parse-robots.txt-line drop ] with each [ <rules> dup ] dip [ parse-robots.txt-line drop ] with each
] map first ; ] map ;
: robots ( url -- robots ) : robots ( url -- robots )
>url >url