diff --git a/basis/furnace/syndication/syndication.factor b/basis/furnace/syndication/syndication.factor index b71abf6d86..9bd4fe4933 100644 --- a/basis/furnace/syndication/syndication.factor +++ b/basis/furnace/syndication/syndication.factor @@ -39,16 +39,18 @@ M: object >entry "UTF-8" >>content-charset utf8 >>content-encoding ; -TUPLE: feed-action < action title url entries ; +TUPLE: feed-action < action title url hubs entries ; : ( -- action ) feed-action new-action dup '[ feed new _ - [ title>> call >>title ] - [ url>> call adjust-url >>url ] - [ entries>> call process-entries >>entries ] - tri + { + [ title>> call >>title ] + [ url>> call adjust-url >>url ] + [ hubs>> [ call [ adjust-url ] map >>hubs ] when* ] + [ entries>> call process-entries >>entries ] + } cleave ] >>display ; diff --git a/basis/syndication/pubsubhubbub/pubsubhubbub-tests.factor b/basis/syndication/pubsubhubbub/pubsubhubbub-tests.factor new file mode 100644 index 0000000000..2feb47ef4b --- /dev/null +++ b/basis/syndication/pubsubhubbub/pubsubhubbub-tests.factor @@ -0,0 +1,41 @@ +! Copyright (c) 2010 Samuel Tardieu. +! See http://factorcode.org/license.txt for BSD license. +USING: http syndication.pubsubhubbub tools.test urls ; + +[ + T{ request + { method "POST" } + { url URL" http://rfc1149.superfeedr.com:80/" } + { version "1.1" } + { header + H{ + { "user-agent" "Factor http.client" } + { "connection" "close" } + } + } + { post-data + T{ post-data + { data + B{ + 104 117 98 46 109 111 100 101 61 112 117 98 + 108 105 115 104 38 104 117 98 46 117 114 + 108 61 104 116 116 112 58 47 47 119 119 119 + 46 114 102 99 49 49 52 57 46 110 101 116 47 + 98 108 111 103 47 102 101 101 100 47 38 104 + 117 98 46 117 114 108 61 104 116 116 112 58 + 47 47 119 119 119 46 114 102 99 49 49 52 57 + 46 110 101 116 47 98 108 111 103 47 101 110 + 47 102 101 101 100 47 + } + } + { content-type + "application/x-www-form-urlencoded" + } + } + } + { cookies V{ } } + { redirects 10 } + } +] [ + { "http://www.rfc1149.net/blog/feed/" "http://www.rfc1149.net/blog/en/feed/" } "http://rfc1149.superfeedr.com/" +] unit-test diff --git a/basis/syndication/pubsubhubbub/pubsubhubbub.factor b/basis/syndication/pubsubhubbub/pubsubhubbub.factor new file mode 100644 index 0000000000..52b298a878 --- /dev/null +++ b/basis/syndication/pubsubhubbub/pubsubhubbub.factor @@ -0,0 +1,20 @@ +! Copyright (c) 2010 Samuel Tardieu. +! See http://factorcode.org/license.txt for BSD license. +USING: accessors byte-arrays kernel http http.client sequences urls.encoding ; +IN: syndication.pubsubhubbub + + ( feeds -- post-data ) + [ url-encode "hub.url=" prepend ] map "&" join + "hub.mode=publish&" prepend >byte-array + "application/x-www-form-urlencoded" [ data<< ] keep ; + +PRIVATE> + +: ( feeds url -- request ) + [ ] [ ] bi* ; + +: ping ( feeds url -- ) + http-request drop + dup code>> 204 = [ drop ] [ download-failed ] if ; \ No newline at end of file diff --git a/basis/syndication/syndication.factor b/basis/syndication/syndication.factor index 0cc124c088..63f40473ce 100644 --- a/basis/syndication/syndication.factor +++ b/basis/syndication/syndication.factor @@ -5,13 +5,13 @@ USING: xml.traversal kernel assocs math.order strings sequences xml.data xml.writer io.streams.string combinators xml xml.entities.html io.files io http.client namespaces make xml.syntax hashtables calendar.format accessors continuations -urls present byte-arrays ; +urls present byte-arrays fry arrays ; IN: syndication : any-tag-named ( tag names -- tag-inside ) [ f ] 2dip [ tag-named nip dup ] with find 2drop ; -TUPLE: feed title url entries ; +TUPLE: feed title url entries hubs ; : ( -- feed ) feed new ; @@ -65,15 +65,19 @@ TUPLE: entry title url description date ; [ "item" tags-named [ rss2.0-entry ] map set-entries ] tri ; -: atom-entry-link ( tag -- url/f ) - "link" tags-named - [ "rel" attr { f "alternate" } member? ] find nip - dup [ "href" attr >url ] when ; +: atom-links ( tag rel -- seq ) + [ "links" tags-named ] dip + dup "alternate" = [ f 2array ] [ 1array ] if + '[ "rel" attr _ member? ] filter + [ "href" attr >url ] map ; + +: atom-link ( tag rel -- url/f ) + atom-links [ f ] [ first ] if-empty ; : atom1.0-entry ( tag -- entry ) swap { [ "title" tag-named children>string >>title ] - [ atom-entry-link >>url ] + [ "alternate" atom-link >>url ] [ { "content" "summary" } any-tag-named dup children>> [ string? not ] any? @@ -88,11 +92,12 @@ TUPLE: entry title url description date ; } cleave ; : atom1.0 ( xml -- feed ) - swap - [ "title" tag-named children>string >>title ] - [ "link" tag-named "href" attr >url >>url ] - [ "entry" tags-named [ atom1.0-entry ] map set-entries ] - tri ; + swap { + [ "title" tag-named children>string >>title ] + [ "alternate" atom-link >>url ] + [ "hub" atom-links >>hubs ] + [ "entry" tags-named [ atom1.0-entry ] map set-entries ] + } cleave ; : xml>feed ( xml -- feed ) dup main>> { @@ -129,14 +134,21 @@ M: byte-array parse-feed [ bytes>xml xml>feed ] with-html-entities ; XML] ; +: hub>xml ( hub -- xml ) + present [XML /> XML] ; + : feed>xml ( feed -- xml ) - [ title>> ] - [ url>> present ] - [ entries>> [ entry>xml ] map ] tri + { + [ title>> ] + [ url>> present ] + [ hubs>> [ hub>xml ] map ] + [ entries>> [ entry>xml ] map ] + } cleave <-> /> <-> + <-> XML> ; diff --git a/extra/webapps/planet/planet.factor b/extra/webapps/planet/planet.factor index cf48f6dfbc..38a1ae94b1 100644 --- a/extra/webapps/planet/planet.factor +++ b/extra/webapps/planet/planet.factor @@ -2,7 +2,7 @@ ! See http://factorcode.org/license.txt for BSD license. USING: kernel accessors sequences sorting math math.order calendar timers logging concurrency.combinators namespaces -db.types db.tuples db fry locals hashtables +db.types db.tuples db fry locals hashtables continuations syndication urls xml.writer validators html.forms html.components @@ -15,7 +15,8 @@ furnace.redirection furnace.boilerplate furnace.auth.login furnace.auth -furnace.syndication ; +furnace.syndication +syndication.pubsubhubbub ; IN: webapps.planet TUPLE: planet < dispatcher ; @@ -51,6 +52,9 @@ posting "POSTINGS" { "date" "DATE" TIMESTAMP +not-null+ } } define-persistent +CONSTANT: hubs { { URL" http://pubsubhubbub.appspot.com/" + URL" http://pubsubhubbub.appspot.com/publish" } } + : ( id -- todo ) blog new swap >>id ; @@ -59,9 +63,12 @@ posting "POSTINGS" f select-tuples [ name>> ] sort-with ; +: sort-postings ( seq -- seq ) + [ date>> ] inv-sort-with ; + : postings ( -- seq ) posting new select-tuples - [ date>> ] inv-sort-with ; + sort-postings ; : ( -- action ) @@ -77,10 +84,19 @@ posting "POSTINGS" { planet "planet" } >>template ; +: hubs-urls ( -- seq ) + hubs [ first ] map ; + +: ping-hubs ( -- ) + { URL" http://planet.factorcode.org/feed.xml" } + hubs [ second ] map + [ [ ping ] [ 3drop ] recover ] with each ; + : ( -- action ) [ "Planet Factor" ] >>title [ URL" $planet" ] >>url + [ hubs-urls ] >>hubs [ postings ] >>entries ; :: ( entry name -- entry' ) @@ -102,12 +118,16 @@ posting "POSTINGS" : sort-entries ( entries -- entries' ) [ date>> ] inv-sort-with ; -: update-cached-postings ( -- ) - blogroll fetch-blogroll sort-entries 8 short head [ +: set-cached-postings ( seq -- ) + [ posting new delete-tuples [ insert-tuple ] each ] with-transaction ; +: update-cached-postings ( -- ) + blogroll fetch-blogroll sort-entries 8 short head sort-postings + dup postings = [ drop ] [ set-cached-postings ping-hubs ] if ; + : ( -- action ) [