http.client: use new http-get* word.
parent
18088e1aa3
commit
435f54cf59
|
@ -7,7 +7,7 @@ IN: bootstrap.image.download
|
||||||
CONSTANT: url URL" http://downloads.factorcode.org/images/latest/"
|
CONSTANT: url URL" http://downloads.factorcode.org/images/latest/"
|
||||||
|
|
||||||
: download-checksums ( -- alist )
|
: download-checksums ( -- alist )
|
||||||
url "checksums.txt" >url derive-url http-get nip
|
url "checksums.txt" >url derive-url http-get*
|
||||||
string-lines [ " " split1 ] { } map>assoc ;
|
string-lines [ " " split1 ] { } map>assoc ;
|
||||||
|
|
||||||
: file-checksum ( image -- checksum )
|
: file-checksum ( image -- checksum )
|
||||||
|
|
|
@ -53,9 +53,7 @@ fry http.client kernel urls ;
|
||||||
URL" http://www.oracle.com"
|
URL" http://www.oracle.com"
|
||||||
}
|
}
|
||||||
2 <semaphore> '[
|
2 <semaphore> '[
|
||||||
_ [
|
_ [ http-get* ] with-semaphore
|
||||||
http-get nip
|
|
||||||
] with-semaphore
|
|
||||||
] parallel-map"""
|
] parallel-map"""
|
||||||
} ;
|
} ;
|
||||||
|
|
||||||
|
|
|
@ -108,7 +108,7 @@ M: byte-array parse-feed [ bytes>xml xml>feed ] with-html-entities ;
|
||||||
|
|
||||||
: download-feed ( url -- feed )
|
: download-feed ( url -- feed )
|
||||||
#! Retrieve an news syndication file, return as a feed tuple.
|
#! Retrieve an news syndication file, return as a feed tuple.
|
||||||
http-get nip parse-feed ;
|
http-get* parse-feed ;
|
||||||
|
|
||||||
! Atom generation
|
! Atom generation
|
||||||
|
|
||||||
|
|
|
@ -23,7 +23,7 @@ ERROR: bad-response json status ;
|
||||||
] unless ;
|
] unless ;
|
||||||
|
|
||||||
: json-data ( url -- json )
|
: json-data ( url -- json )
|
||||||
http-get nip json> check-status "data" of ;
|
http-get* json> check-status "data" of ;
|
||||||
|
|
||||||
: get-short-url ( short-url path -- data )
|
: get-short-url ( short-url path -- data )
|
||||||
<bitly-url> swap "shortUrl" set-query-param json-data ;
|
<bitly-url> swap "shortUrl" set-query-param json-data ;
|
||||||
|
|
|
@ -22,7 +22,7 @@ title content unescapedUrl url titleNoFormatting fileFormat ;
|
||||||
PRIVATE>
|
PRIVATE>
|
||||||
|
|
||||||
: google-search ( query -- results )
|
: google-search ( query -- results )
|
||||||
search-url http-get nip json>
|
search-url http-get* json>
|
||||||
{ "responseData" "results" } deep-at
|
{ "responseData" "results" } deep-at
|
||||||
[ \ search-result from-slots ] map ;
|
[ \ search-result from-slots ] map ;
|
||||||
|
|
||||||
|
|
|
@ -53,7 +53,7 @@ ERROR: response-error response error ;
|
||||||
|
|
||||||
: translate-tts ( text -- file )
|
: translate-tts ( text -- file )
|
||||||
"http://translate.google.com/translate_tts?tl=en" >url
|
"http://translate.google.com/translate_tts?tl=en" >url
|
||||||
swap "q" set-query-param http-get nip
|
swap "q" set-query-param http-get*
|
||||||
temporary-file ".mp3" append
|
temporary-file ".mp3" append
|
||||||
[ binary set-file-contents ] keep ;
|
[ binary set-file-contents ] keep ;
|
||||||
|
|
||||||
|
|
|
@ -25,7 +25,7 @@ TUPLE: post title postedBy points id url commentCount postedAgo ;
|
||||||
] map ;
|
] map ;
|
||||||
|
|
||||||
: hacker-news-items ( -- seq )
|
: hacker-news-items ( -- seq )
|
||||||
"http://api.ihackernews.com/page" http-get nip
|
"http://api.ihackernews.com/page" http-get*
|
||||||
json> "items" of items> ;
|
json> "items" of items> ;
|
||||||
|
|
||||||
: write-title ( title url -- )
|
: write-title ( title url -- )
|
||||||
|
|
|
@ -6,7 +6,7 @@ sequences sets splitting unicode.case unicode.categories urls
|
||||||
urls.encoding shuffle ;
|
urls.encoding shuffle ;
|
||||||
IN: html.parser.analyzer
|
IN: html.parser.analyzer
|
||||||
|
|
||||||
: scrape-html ( url -- headers vector )
|
: scrape-html ( url -- response vector )
|
||||||
http-get parse-html ;
|
http-get parse-html ;
|
||||||
|
|
||||||
: attribute ( tag string -- obj/f )
|
: attribute ( tag string -- obj/f )
|
||||||
|
|
|
@ -21,7 +21,7 @@ SYMBOLS: latest-sources last-built-sources ;
|
||||||
[ maybe-download-image drop ] [ file-checksum ] bi ;
|
[ maybe-download-image drop ] [ file-checksum ] bi ;
|
||||||
|
|
||||||
: latest-counter ( -- counter )
|
: latest-counter ( -- counter )
|
||||||
counter-url get-global http-get nip string>number ;
|
counter-url get-global http-get* string>number ;
|
||||||
|
|
||||||
: update-sources ( -- )
|
: update-sources ( -- )
|
||||||
#! Must be run from builds-dir
|
#! Must be run from builds-dir
|
||||||
|
|
|
@ -43,7 +43,7 @@ public_description subscribers title url ;
|
||||||
TUPLE: page url data before after ;
|
TUPLE: page url data before after ;
|
||||||
|
|
||||||
: json-page ( url -- page )
|
: json-page ( url -- page )
|
||||||
>url dup http-get nip json> "data" of {
|
>url dup http-get* json> "data" of {
|
||||||
[ "children" of [ parse-data ] map ]
|
[ "children" of [ parse-data ] map ]
|
||||||
[ "before" of [ f ] when-json-null ]
|
[ "before" of [ f ] when-json-null ]
|
||||||
[ "after" of [ f ] when-json-null ]
|
[ "after" of [ f ] when-json-null ]
|
||||||
|
@ -54,7 +54,7 @@ TUPLE: page url data before after ;
|
||||||
|
|
||||||
: get-user-info ( username -- user )
|
: get-user-info ( username -- user )
|
||||||
"http://api.reddit.com/user/%s/about" sprintf
|
"http://api.reddit.com/user/%s/about" sprintf
|
||||||
http-get nip json> parse-data ;
|
http-get* json> parse-data ;
|
||||||
|
|
||||||
: get-url-info ( url -- page )
|
: get-url-info ( url -- page )
|
||||||
"http://api.reddit.com/api/info?url=%s" sprintf json-page ;
|
"http://api.reddit.com/api/info?url=%s" sprintf json-page ;
|
||||||
|
|
|
@ -29,8 +29,8 @@ visit-time request-rate crawl-delay unknowns ;
|
||||||
: >robots.txt-url ( url -- url' )
|
: >robots.txt-url ( url -- url' )
|
||||||
>url URL" robots.txt" derive-url ;
|
>url URL" robots.txt" derive-url ;
|
||||||
|
|
||||||
: get-robots.txt ( url -- headers robots.txt )
|
: get-robots.txt ( url -- robots.txt )
|
||||||
>robots.txt-url http-get ;
|
>robots.txt-url http-get* ;
|
||||||
|
|
||||||
: normalize-robots.txt ( string -- sitemaps seq )
|
: normalize-robots.txt ( string -- sitemaps seq )
|
||||||
string-lines
|
string-lines
|
||||||
|
@ -98,5 +98,4 @@ PRIVATE>
|
||||||
] map ;
|
] map ;
|
||||||
|
|
||||||
: url>robots ( url -- robots )
|
: url>robots ( url -- robots )
|
||||||
>url
|
>url dup get-robots.txt parse-robots.txt <robots> ;
|
||||||
dup get-robots.txt nip parse-robots.txt <robots> ;
|
|
||||||
|
|
|
@ -15,7 +15,7 @@ IN: rosetta-code.web-scraping
|
||||||
! and popular such as CPAN for Perl or Boost for C++.
|
! and popular such as CPAN for Perl or Boost for C++.
|
||||||
|
|
||||||
: web-scraping-main ( -- )
|
: web-scraping-main ( -- )
|
||||||
"http://tycho.usno.navy.mil/cgi-bin/timer.pl" http-get nip
|
"http://tycho.usno.navy.mil/cgi-bin/timer.pl" http-get*
|
||||||
[ "UTC" swap start [ 9 - ] [ 1 - ] bi ] keep subseq print ;
|
[ "UTC" swap start [ 9 - ] [ 1 - ] bi ] keep subseq print ;
|
||||||
|
|
||||||
MAIN: web-scraping-main
|
MAIN: web-scraping-main
|
||||||
|
|
|
@ -5,8 +5,7 @@ IN: twitter.prettyprint
|
||||||
|
|
||||||
MEMO: load-http-image ( url -- image/f )
|
MEMO: load-http-image ( url -- image/f )
|
||||||
'[ _
|
'[ _
|
||||||
[ http-get [ check-response drop ] dip ]
|
[ http-get* ] [ image-class ] bi load-image*
|
||||||
[ image-class ] bi load-image*
|
|
||||||
] [ drop f ] recover ;
|
] [ drop f ] recover ;
|
||||||
|
|
||||||
: user-image ( user -- image/f )
|
: user-image ( user -- image/f )
|
||||||
|
|
|
@ -35,7 +35,7 @@ TUPLE: fjsc < dispatcher ;
|
||||||
|
|
||||||
: do-compile-url ( url -- response )
|
: do-compile-url ( url -- response )
|
||||||
[
|
[
|
||||||
absolute-url http-get nip 'expression' parse fjsc-compile write "();" write
|
absolute-url http-get* 'expression' parse fjsc-compile write "();" write
|
||||||
] with-string-writer
|
] with-string-writer
|
||||||
"application/javascript" <content> ;
|
"application/javascript" <content> ;
|
||||||
|
|
||||||
|
|
|
@ -40,7 +40,7 @@ IN: wikipedia
|
||||||
"http://en.wikipedia.org/wiki/%s_%s" sprintf ;
|
"http://en.wikipedia.org/wiki/%s_%s" sprintf ;
|
||||||
|
|
||||||
: (historical-events) ( timestamp -- seq )
|
: (historical-events) ( timestamp -- seq )
|
||||||
historical-url http-get nip string>xml "ul" deep-tags-named ;
|
historical-url http-get* string>xml "ul" deep-tags-named ;
|
||||||
|
|
||||||
: items>sequence ( tag -- seq )
|
: items>sequence ( tag -- seq )
|
||||||
children-tags [ deep-children>string ] map ;
|
children-tags [ deep-children>string ] map ;
|
||||||
|
@ -67,7 +67,7 @@ PRIVATE>
|
||||||
|
|
||||||
: article. ( name -- )
|
: article. ( name -- )
|
||||||
"http://en.wikipedia.org/wiki/%s" sprintf
|
"http://en.wikipedia.org/wiki/%s" sprintf
|
||||||
http-get nip parse-html "content" find-by-id-between
|
http-get* parse-html "content" find-by-id-between
|
||||||
[ html-text. ] with-string-writer string-lines
|
[ html-text. ] with-string-writer string-lines
|
||||||
[ [ blank? ] trim ] map harvest [
|
[ [ blank? ] trim ] map harvest [
|
||||||
R/ </ "<" re-replace
|
R/ </ "<" re-replace
|
||||||
|
|
|
@ -16,7 +16,7 @@ SYMBOL: wolfram-api-id
|
||||||
: query ( query -- xml )
|
: query ( query -- xml )
|
||||||
url-encode wolfram-api-id get-global
|
url-encode wolfram-api-id get-global
|
||||||
"http://api.wolframalpha.com/v2/query?input=%s&appid=%s"
|
"http://api.wolframalpha.com/v2/query?input=%s&appid=%s"
|
||||||
sprintf http-get nip string>xml
|
sprintf http-get* string>xml
|
||||||
dup "error" tag-named [
|
dup "error" tag-named [
|
||||||
"msg" tag-named children>string throw
|
"msg" tag-named children>string throw
|
||||||
] when* ;
|
] when* ;
|
||||||
|
|
|
@ -11,14 +11,14 @@ IN: xkcd
|
||||||
<PRIVATE
|
<PRIVATE
|
||||||
|
|
||||||
: comic-image ( url -- image )
|
: comic-image ( url -- image )
|
||||||
http-get nip
|
http-get*
|
||||||
R" http://imgs\.xkcd\.com/comics/[^\.]+\.(png|jpg)"
|
R" http://imgs\.xkcd\.com/comics/[^\.]+\.(png|jpg)"
|
||||||
first-match >string load-http-image ;
|
first-match >string load-http-image ;
|
||||||
|
|
||||||
: comic-image. ( url -- ) comic-image image. ;
|
: comic-image. ( url -- ) comic-image image. ;
|
||||||
|
|
||||||
: comic-string ( url -- string )
|
: comic-string ( url -- string )
|
||||||
http-get nip string>xml
|
http-get* string>xml
|
||||||
"transcript" "id" deep-tag-with-attr children>string ;
|
"transcript" "id" deep-tag-with-attr children>string ;
|
||||||
|
|
||||||
: comic-text. ( url -- )
|
: comic-text. ( url -- )
|
||||||
|
|
|
@ -57,4 +57,4 @@ CONSTANT: factor-id "fRrVAKzV34GDyeRw6bUHDhEWHRedwfOC7e61wwXZLgGF80E67spxdQXuugB
|
||||||
swap >>query ;
|
swap >>query ;
|
||||||
|
|
||||||
: yahoo-search ( search -- seq )
|
: yahoo-search ( search -- seq )
|
||||||
query http-get nip string>xml parse-yahoo ;
|
query http-get* string>xml parse-yahoo ;
|
||||||
|
|
Loading…
Reference in New Issue