From 05154abe1d771275ef4d3e92dd0884d2acc4b8cd Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Mon, 18 Feb 2008 08:13:35 -0600 Subject: [PATCH 1/9] builder: initiate build when new images are available --- extra/builder/builder.factor | 21 +++++++++++++++++++-- extra/builder/util/util.factor | 9 ++++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/extra/builder/builder.factor b/extra/builder/builder.factor index 0c9f4ab099..870c896542 100644 --- a/extra/builder/builder.factor +++ b/extra/builder/builder.factor @@ -61,6 +61,13 @@ VAR: stamp ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +: copy-image ( -- ) + "../../factor/" my-arch boot-image-name append + my-arch boot-image-name + copy-file ; + +! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + : factor-binary ( -- name ) os { { "macosx" [ "./Factor.app/Contents/MacOS/factor" ] } @@ -112,7 +119,9 @@ SYMBOL: build-status make-vm [ "vm compile error" print "../compile-log" cat ] run-or-bail - [ retrieve-image ] [ "Image download error" print throw ] recover + ! [ retrieve-image ] [ "Image download error" print throw ] recover + + copy-image bootstrap [ "Bootstrap error" print "../boot-log" cat ] run-or-bail @@ -158,6 +167,8 @@ SYMBOL: builder-recipients ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +USE: bootstrap.image.download + : git-pull ( -- desc ) { "git" @@ -173,11 +184,17 @@ SYMBOL: builder-recipients git-id = not ; +: new-image-available? ( -- ? ) + my-arch boot-image-name need-new-image? + [ download-my-image t ] + [ f ] + if ; + : build-loop ( -- ) builds-check [ builds "/factor" append cd - updates-available? + updates-available? new-image-available? or [ build ] when ] diff --git a/extra/builder/util/util.factor b/extra/builder/util/util.factor index f9eb17c565..fd985f77cf 100644 --- a/extra/builder/util/util.factor +++ b/extra/builder/util/util.factor @@ -83,4 +83,11 @@ TUPLE: process* arguments stdin stdout stderr timeout ; USING: bootstrap.image bootstrap.image.download io.streams.null ; -: retrieve-image ( -- ) [ my-arch download-image ] with-null-stream ; \ No newline at end of file +: retrieve-image ( -- ) [ my-arch download-image ] with-null-stream ; + +! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +: copy-image ( -- ) + "../../factor/" my-arch boot-image-name append + my-arch boot-image-name + copy-file ; \ No newline at end of file From fa5b621257ba812814b2603d50584517d2933ce2 Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Mon, 18 Feb 2008 09:01:31 -0600 Subject: [PATCH 2/9] builder: run builder.test using a more robust process descriptor --- extra/builder/builder.factor | 24 +++++++++++++++++++----- extra/builder/util/util.factor | 9 +-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/extra/builder/builder.factor b/extra/builder/builder.factor index 870c896542..361401ad55 100644 --- a/extra/builder/builder.factor +++ b/extra/builder/builder.factor @@ -88,8 +88,20 @@ VAR: stamp 20 minutes>ms >>timeout >desc ; -: builder-test ( -- desc ) { factor-binary "-run=builder.test" } to-strings ; - +! : builder-test ( -- desc ) { factor-binary "-run=builder.test" } to-strings ; + +: builder-test-cmd ( -- cmd ) + { factor-binary "-run=builder.test" } to-strings ; + +: builder-test ( -- desc ) + + builder-test-cmd >>arguments + +closed+ >>stdin + "../test-log" >>stdout + +stdout+ >>stderr + 45 minutes>ms >>timeout + >desc ; + ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! SYMBOL: build-status @@ -125,9 +137,11 @@ SYMBOL: build-status bootstrap [ "Bootstrap error" print "../boot-log" cat ] run-or-bail - [ builder-test try-process ] - [ "Builder test error" print throw ] - recover +! [ builder-test try-process ] +! [ "Builder test error" print throw ] +! recover + + builder-test [ "Test error" print "../test-log" cat ] run-or-bail "Boot time: " write "../boot-time" eval-file milli-seconds>time print "Load time: " write "../load-time" eval-file milli-seconds>time print diff --git a/extra/builder/util/util.factor b/extra/builder/util/util.factor index fd985f77cf..f9eb17c565 100644 --- a/extra/builder/util/util.factor +++ b/extra/builder/util/util.factor @@ -83,11 +83,4 @@ TUPLE: process* arguments stdin stdout stderr timeout ; USING: bootstrap.image bootstrap.image.download io.streams.null ; -: retrieve-image ( -- ) [ my-arch download-image ] with-null-stream ; - -! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -: copy-image ( -- ) - "../../factor/" my-arch boot-image-name append - my-arch boot-image-name - copy-file ; \ No newline at end of file +: retrieve-image ( -- ) [ my-arch download-image ] with-null-stream ; \ No newline at end of file From ca484a7002de15578f7b23ec68d525cfb148964e Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Mon, 18 Feb 2008 12:52:20 -0600 Subject: [PATCH 3/9] factor.el: use ~/factor as the default location --- misc/factor.el | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/misc/factor.el b/misc/factor.el index 19b7f28e22..7225ef91fd 100644 --- a/misc/factor.el +++ b/misc/factor.el @@ -110,8 +110,8 @@ (require 'comint) -(defvar factor-binary "/scratch/repos/Factor/factor") -(defvar factor-image "/scratch/repos/Factor/factor.image") +(defvar factor-binary "~/factor/factor") +(defvar factor-image "~/factor/factor.image") (defun factor-telnet-to-port (port) (interactive "nPort: ") @@ -202,8 +202,8 @@ (defun run-factor () (interactive) (switch-to-buffer - (make-comint-in-buffer "factor" nil factor-binary nil - (concat "-i=" factor-image) + (make-comint-in-buffer "factor" nil (expand-file-name factor-binary) nil + (concat "-i=" (expand-file-name factor-image)) "-run=listener")) (factor-listener-mode)) From 38cb6947793d9ab6cfd55a2262aa4a8adefe7866 Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Mon, 18 Feb 2008 17:16:37 -0600 Subject: [PATCH 4/9] builder: benchmark deltas --- extra/builder/builder.factor | 57 ++++++++++++++++++++++++++++++++---- 1 file changed, 51 insertions(+), 6 deletions(-) diff --git a/extra/builder/builder.factor b/extra/builder/builder.factor index 361401ad55..e01ac04eb2 100644 --- a/extra/builder/builder.factor +++ b/extra/builder/builder.factor @@ -62,8 +62,12 @@ VAR: stamp ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! : copy-image ( -- ) - "../../factor/" my-arch boot-image-name append - my-arch boot-image-name + "../../factor/" my-boot-image-name append + "../" my-boot-image-name append + copy-file + + "../../factor/" my-boot-image-name append + my-boot-image-name copy-file ; ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -76,8 +80,7 @@ VAR: stamp case ; : bootstrap-cmd ( -- cmd ) - { factor-binary [ "-i=" my-boot-image-name append ] "-no-user-init" } - to-strings ; + { factor-binary { "-i=" my-boot-image-name } "-no-user-init" } to-strings ; : bootstrap ( -- desc ) @@ -104,6 +107,37 @@ VAR: stamp ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +USING: arrays assocs math ; + +: passing-benchmarks ( table -- table ) + [ second first2 number? swap number? and ] subset ; + +: simplify-table ( table -- table ) [ first2 second 2array ] map ; + +: benchmark-difference ( old-table benchmark-result -- result-diff ) + first2 >r + tuck swap at + r> + swap - + 2array ; + +: compare-tables ( old new -- table ) + [ passing-benchmarks simplify-table ] 2apply + [ benchmark-difference ] with map ; + +: show-benchmark-deltas ( -- ) + "Benchmark deltas: " print + + [ + "../../benchmarks" eval-file + "../benchmarks" eval-file + compare-tables . + ] + [ drop "Error generating benchmark deltas" . ] + recover ; + +! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + SYMBOL: build-status : (build) ( -- ) @@ -143,6 +177,8 @@ SYMBOL: build-status builder-test [ "Test error" print "../test-log" cat ] run-or-bail + + "Boot time: " write "../boot-time" eval-file milli-seconds>time print "Load time: " write "../load-time" eval-file milli-seconds>time print "Test time: " write "../test-time" eval-file milli-seconds>time print nl @@ -153,6 +189,12 @@ SYMBOL: build-status "Benchmarks: " print "../benchmarks" [ stdio get contents eval ] with-file-reader benchmarks. + nl + + show-benchmark-deltas + + "../benchmarks" "../../benchmarks" copy-file + ] with-file-writer build-status on ; @@ -175,9 +217,12 @@ SYMBOL: builder-recipients "../report" file>string >>body send ; +! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + : build ( -- ) [ (build) ] [ drop ] recover - [ send-builder-email ] [ drop "not sending mail" . ] recover ; + [ send-builder-email ] [ drop "not sending mail" . ] recover + ".." cd { "rm" "-rf" "factor" } run-process drop ; ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -199,7 +244,7 @@ USE: bootstrap.image.download = not ; : new-image-available? ( -- ? ) - my-arch boot-image-name need-new-image? + my-boot-image-name need-new-image? [ download-my-image t ] [ f ] if ; From 4acd6018b3cfe96c01757386d121ab14db1ac8cf Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Mon, 18 Feb 2008 19:11:26 -0600 Subject: [PATCH 5/9] builder: minor cleanup --- extra/builder/builder.factor | 8 -------- 1 file changed, 8 deletions(-) diff --git a/extra/builder/builder.factor b/extra/builder/builder.factor index e01ac04eb2..88fc35a4b8 100644 --- a/extra/builder/builder.factor +++ b/extra/builder/builder.factor @@ -91,8 +91,6 @@ VAR: stamp 20 minutes>ms >>timeout >desc ; -! : builder-test ( -- desc ) { factor-binary "-run=builder.test" } to-strings ; - : builder-test-cmd ( -- cmd ) { factor-binary "-run=builder.test" } to-strings ; @@ -171,14 +169,8 @@ SYMBOL: build-status bootstrap [ "Bootstrap error" print "../boot-log" cat ] run-or-bail -! [ builder-test try-process ] -! [ "Builder test error" print throw ] -! recover - builder-test [ "Test error" print "../test-log" cat ] run-or-bail - - "Boot time: " write "../boot-time" eval-file milli-seconds>time print "Load time: " write "../load-time" eval-file milli-seconds>time print "Test time: " write "../test-time" eval-file milli-seconds>time print nl From 0458a2e9c013f1369cff9e056f47df8a8f46c3b9 Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Tue, 19 Feb 2008 06:08:42 -0600 Subject: [PATCH 6/9] builder.benchmark: add --- extra/builder/benchmark/benchmark.factor | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 extra/builder/benchmark/benchmark.factor diff --git a/extra/builder/benchmark/benchmark.factor b/extra/builder/benchmark/benchmark.factor new file mode 100644 index 0000000000..48891593d2 --- /dev/null +++ b/extra/builder/benchmark/benchmark.factor @@ -0,0 +1,41 @@ + +USING: kernel continuations arrays assocs sequences sorting math + io io.styles prettyprint builder.util ; + +IN: builder.benchmark + +: passing-benchmarks ( table -- table ) + [ second first2 number? swap number? and ] subset ; + +: simplify-table ( table -- table ) [ first2 second 2array ] map ; + +: benchmark-difference ( old-table benchmark-result -- result-diff ) + first2 >r + tuck swap at + r> + swap - + 2array ; + +: compare-tables ( old new -- table ) + [ passing-benchmarks simplify-table ] 2apply + [ benchmark-difference ] with map ; + +: benchmark-deltas ( -- table ) + "../../benchmarks" "../benchmarks" [ eval-file ] 2apply + compare-tables + sort-values ; + +: benchmark-deltas. ( deltas -- ) + standard-table-style + [ + [ [ "Benchmark" write ] with-cell [ "Delta (ms)" write ] with-cell ] + with-row + [ [ swap [ write ] with-cell pprint-cell ] with-row ] + assoc-each + ] + tabular-output ; + +: show-benchmark-deltas ( -- ) + [ benchmark-deltas benchmark-deltas. ] + [ drop "Error generating benchmark deltas" . ] + recover ; \ No newline at end of file From 84963c16ab709060f3788b1ac58da86e20673e78 Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Tue, 19 Feb 2008 06:09:06 -0600 Subject: [PATCH 7/9] builder: factor out benchmark delta code --- extra/builder/builder.factor | 34 ++-------------------------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/extra/builder/builder.factor b/extra/builder/builder.factor index 88fc35a4b8..fbff4a40e3 100644 --- a/extra/builder/builder.factor +++ b/extra/builder/builder.factor @@ -1,7 +1,8 @@ USING: kernel namespaces sequences splitting system combinators continuations parser io io.files io.launcher io.sockets prettyprint threads - bootstrap.image benchmark vars bake smtp builder.util accessors ; + bootstrap.image benchmark vars bake smtp builder.util accessors + builder.benchmark ; IN: builder @@ -105,37 +106,6 @@ VAR: stamp ! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -USING: arrays assocs math ; - -: passing-benchmarks ( table -- table ) - [ second first2 number? swap number? and ] subset ; - -: simplify-table ( table -- table ) [ first2 second 2array ] map ; - -: benchmark-difference ( old-table benchmark-result -- result-diff ) - first2 >r - tuck swap at - r> - swap - - 2array ; - -: compare-tables ( old new -- table ) - [ passing-benchmarks simplify-table ] 2apply - [ benchmark-difference ] with map ; - -: show-benchmark-deltas ( -- ) - "Benchmark deltas: " print - - [ - "../../benchmarks" eval-file - "../benchmarks" eval-file - compare-tables . - ] - [ drop "Error generating benchmark deltas" . ] - recover ; - -! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - SYMBOL: build-status : (build) ( -- ) From daaed1b4e9b098859671c9499050ddf578c01bca Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Tue, 19 Feb 2008 13:53:55 -0600 Subject: [PATCH 8/9] remove builder.server --- extra/builder/server/server.factor | 68 ------------------------------ 1 file changed, 68 deletions(-) delete mode 100644 extra/builder/server/server.factor diff --git a/extra/builder/server/server.factor b/extra/builder/server/server.factor deleted file mode 100644 index f3ec349557..0000000000 --- a/extra/builder/server/server.factor +++ /dev/null @@ -1,68 +0,0 @@ - -USING: kernel continuations namespaces threads match bake concurrency builder ; - -IN: builder.server - -! : build-server ( -- ) -! receive -! { -! { -! "start" -! [ [ build ] in-thread ] -! } - -! { -! { ?from ?tag "status" } -! [ `{ ?tag ,[ build-status get ] } ?from send ] -! } -! } -! match-cond -! build-server ; - -! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -! : build-server ( -- ) -! receive -! { -! { -! "start" -! [ -! [ [ build ] [ drop ] recover "idle" build-status set-global ] in-thread -! ] -! } - -! { -! { ?from ?tag "status" } -! [ `{ ?tag ,[ build-status get ] } ?from send ] -! } -! } -! match-cond -! build-server ; - -! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - -! : build-server ( -- ) -! receive -! { -! { -! "start" -! [ -! build-status get "idle" = -! build-status get f = -! or -! [ -! [ [ build ] [ drop ] recover "idle" build-status set-global ] -! in-thread -! ] -! when -! ] -! } - -! { -! { ?from ?tag "status" } -! [ `{ ?tag ,[ build-status get ] } ?from send ] -! } -! } -! match-cond -! build-server ; - From 200e949973177d6ccfb8bb2c993039b2c74a9a75 Mon Sep 17 00:00:00 2001 From: Eduardo Cavazos Date: Tue, 19 Feb 2008 14:37:33 -0600 Subject: [PATCH 9/9] builder: remove test-log if successful --- extra/builder/builder.factor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/builder/builder.factor b/extra/builder/builder.factor index fbff4a40e3..bc65035747 100644 --- a/extra/builder/builder.factor +++ b/extra/builder/builder.factor @@ -133,14 +133,14 @@ SYMBOL: build-status make-vm [ "vm compile error" print "../compile-log" cat ] run-or-bail - ! [ retrieve-image ] [ "Image download error" print throw ] recover - copy-image bootstrap [ "Bootstrap error" print "../boot-log" cat ] run-or-bail builder-test [ "Test error" print "../test-log" cat ] run-or-bail + "../test-log" delete-file + "Boot time: " write "../boot-time" eval-file milli-seconds>time print "Load time: " write "../load-time" eval-file milli-seconds>time print "Test time: " write "../test-time" eval-file milli-seconds>time print nl