steve
/
factor
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

mmmm....upgrades 

better html analyzer
db4
Doug Coleman 2008-05-23 23:34:10 -05:00
parent fad5df79df
commit 7a408dcf55
1 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
extra/html/parser/analyzer/analyzer.factor
View File

@ -3,25 +3,27 @@ arrays shuffle unicode.case namespaces splitting http
sequences.lib accessors io combinators http.client ; sequences.lib accessors io combinators http.client ;
IN: html.parser.analyzer IN: html.parser.analyzer
TUPLE: link attributes clickable ;
: scrape-html ( url -- vector ) : scrape-html ( url -- vector )
http-get parse-html ; http-get parse-html ;
: (find-relative) : (find-relative)
[ >r + dup r> ?nth* [ 2drop f f ] unless ] [ 2drop f ] if ; [ >r + dup r> ?nth* [ 2drop f f ] unless ] [ 2drop f ] if ; inline
: find-relative ( seq quot n -- i elt ) : find-relative ( seq quot n -- i elt )
>r over [ find drop ] dip r> swap pick >r over [ find drop ] dip r> swap pick
(find-relative) ; (find-relative) ; inline
: (find-all) ( n seq quot -- ) : (find-all) ( n seq quot -- )
2dup >r >r find-from [ 2dup >r >r find-from [
dupd 2array , 1+ r> r> (find-all) dupd 2array , 1+ r> r> (find-all)
] [ ] [
r> r> 3drop r> r> 3drop
] if* ; ] if* ; inline
: find-all ( seq quot -- alist ) : find-all ( seq quot -- alist )
[ 0 -rot (find-all) ] { } make ; [ 0 -rot (find-all) ] { } make ; inline
: (find-nth) ( offset seq quot n count -- obj ) : (find-nth) ( offset seq quot n count -- obj )
>r >r [ find-from ] 2keep 4 npick [ >r >r [ find-from ] 2keep 4 npick [
@ -33,14 +35,14 @@ IN: html.parser.analyzer
] if ] if
] [ ] [
2drop r> r> 2drop 2drop r> r> 2drop
] if ; ] if ; inline
: find-nth ( seq quot n -- i elt ) : find-nth ( seq quot n -- i elt )
0 -roll 0 (find-nth) ; 0 -roll 0 (find-nth) ; inline
: find-nth-relative ( seq quot n offest -- i elt ) : find-nth-relative ( seq quot n offest -- i elt )
>r [ find-nth ] 3keep 2drop nip r> swap pick >r [ find-nth ] 3keep 2drop nip r> swap pick
(find-relative) ; (find-relative) ; inline
: remove-blank-text ( vector -- vector' ) : remove-blank-text ( vector -- vector' )
[ [
@ -120,9 +122,14 @@ IN: html.parser.analyzer
[ [ name>> "a" = ] [ attributes>> "href" swap at ] bi and ] [ [ name>> "a" = ] [ attributes>> "href" swap at ] bi and ]
find-between-all ; find-between-all ;
: <link> ( vector -- link )
[ first attributes>> ]
[ [ name>> { text "img" } member? ] filter ] bi
link boa ;
: link. ( vector -- ) : link. ( vector -- )
[ second text>> write bl ] [ attributes>> "href" swap at write nl ]
[ first tag-link write nl ] bi ; [ clickable>> [ bl bl text>> print ] each nl ] bi ;
: find-by-text ( seq quot -- tag ) : find-by-text ( seq quot -- tag )
[ dup name>> text = ] prepose find drop ; [ dup name>> text = ] prepose find drop ;
@ -136,12 +143,12 @@ IN: html.parser.analyzer
: find-forms ( vector -- vector' ) : find-forms ( vector -- vector' )
"form" over find-opening-tags-by-name "form" over find-opening-tags-by-name
over [ >r first2 r> find-between* ] curry map swap [ >r first2 r> find-between* ] curry map
[ [ name>> { "form" "input" } member? ] filter ] map ; [ [ name>> { "form" "input" } member? ] filter ] map ;
: find-html-objects ( string vector -- vector' ) : find-html-objects ( string vector -- vector' )
find-opening-tags-by-name [ find-opening-tags-by-name ] keep
over [ >r first2 r> find-between* ] curry map ; [ >r first2 r> find-between* ] curry map ;
: form-action ( vector -- string ) : form-action ( vector -- string )
[ name>> "form" = ] find nip [ name>> "form" = ] find nip