diff --git a/extra/mason/build/build.factor b/extra/mason/build/build.factor index e037bdba0c..1b8089ed5e 100644 --- a/extra/mason/build/build.factor +++ b/extra/mason/build/build.factor @@ -42,9 +42,13 @@ IN: mason.build [ begin-build build-child - [ notify-report ] - [ status-clean eq? [ upload-docs release ] when ] bi + [ notify-report ] [ + status-clean eq? + [ notify-upload upload-docs release ] when + ] bi + notify-finish finish-build - ] [ cleanup ] [ ] continuations:cleanup ; + ] [ cleanup ] [ ] continuations:cleanup + notify-idle ; MAIN: build diff --git a/extra/mason/notify/notify.factor b/extra/mason/notify/notify.factor index b5580fe162..cdde08f979 100644 --- a/extra/mason/notify/notify.factor +++ b/extra/mason/notify/notify.factor @@ -27,6 +27,9 @@ IN: mason.notify : notify-heartbeat ( -- ) f f "heartbeat" status-notify ; +: notify-idle ( -- ) + f f "idle" status-notify ; + : notify-begin-build ( git-id -- ) [ "Starting build of GIT ID " write print flush ] [ f swap "git-id" status-notify ] @@ -51,6 +54,12 @@ IN: mason.notify [ name>> "report" status-notify ] [ email-report ] 2bi ] bi ; +: notify-upload ( -- ) + f f "upload" status-notify ; + +: notify-finish ( -- ) + f f "finish" status-notify ; + : notify-release ( archive-name -- ) [ "Uploaded " prepend [ print flush ] [ mason-tweet ] bi ] [ f swap "release" status-notify ] diff --git a/extra/mason/server/server.factor b/extra/mason/server/server.factor index 6db9332247..c74a9fa6b1 100644 --- a/extra/mason/server/server.factor +++ b/extra/mason/server/server.factor @@ -4,16 +4,20 @@ USING: accessors calendar db db.sqlite db.tuples db.types kernel math math.order sequences combinators.short-circuit ; IN: mason.server +CONSTANT: +idle+ "idle" CONSTANT: +starting+ "starting" CONSTANT: +make-vm+ "make-vm" CONSTANT: +boot+ "boot" CONSTANT: +test+ "test" -CONSTANT: +clean+ "status-clean" +CONSTANT: +upload+ "upload" +CONSTANT: +finish+ "finish" + CONSTANT: +dirty+ "status-dirty" CONSTANT: +error+ "status-error" +CONSTANT: +clean+ "status-clean" TUPLE: builder -host-name os cpu +host-name os cpu heartbeat-timestamp clean-git-id clean-timestamp last-release release-git-id last-git-id last-timestamp last-report @@ -24,6 +28,7 @@ builder "BUILDERS" { { "host-name" "HOST_NAME" TEXT +user-assigned-id+ } { "os" "OS" TEXT +user-assigned-id+ } { "cpu" "CPU" TEXT +user-assigned-id+ } + { "heartbeat-timestamp" "HEARTBEAT_TIMESTAMP" TIMESTAMP } { "clean-git-id" "CLEAN_GIT_ID" TEXT } { "clean-timestamp" "CLEAN_TIMESTAMP" TIMESTAMP } @@ -60,23 +65,11 @@ counter "COUNTER" { counter-tuple [ 0 or 1 + dup ] change-value update-tuple ] with-transaction ; -: crashed-builders ( -- seq ) +: funny-builders ( -- crashed broken ) builder new select-tuples - [ current-timestamp>> 5 hours ago before? ] filter ; - -: broken-builders ( -- seq ) - builder new select-tuples - [ - clean-timestamp>> - { [ not ] [ 1 weeks ago before? ] } 1|| - ] filter ; - -: funny-builders ( -- crashed broken limbo ) - builder new select-tuples - [ [ current-timestamp>> 1 hours ago before? ] filter ] - [ [ clean-timestamp>> 1 weeks ago before? ] filter ] - [ [ [ clean-git-id>> ] [ release-git-id>> ] bi = not ] filter ] - tri ; + [ [ heartbeat-timestamp>> 30 minutes ago before? ] filter ] + [ [ [ clean-git-id>> ] [ last-git-id>> ] bi = not ] filter ] + bi ; : os/cpu ( builder -- string ) [ os>> ] [ cpu>> ] bi "/" glue ; diff --git a/extra/mason/server/watchdog/watchdog.factor b/extra/mason/server/watchdog/watchdog.factor index 9a312407c9..72a1e388bc 100644 --- a/extra/mason/server/watchdog/watchdog.factor +++ b/extra/mason/server/watchdog/watchdog.factor @@ -9,7 +9,7 @@ IN: mason.server.watchdog -

Machines which have not sent a heartbeat for several hours:

+

Machines which are not sending heartbeats:

Dashboard @@ -31,6 +31,6 @@ IN: mason.server.watchdog : check-builders ( -- ) [ - funny-builders 2drop + funny-builders drop [ send-crashed-builder-email ] unless-empty ] with-mason-db ; diff --git a/extra/webapps/mason/dashboard.xml b/extra/webapps/mason/dashboard.xml index 547e3441d1..0a4908cced 100644 --- a/extra/webapps/mason/dashboard.xml +++ b/extra/webapps/mason/dashboard.xml @@ -4,17 +4,13 @@ Mason dashboard

Crashed build machines

-

Machines which have not sent a heartbeat for several hours:

+

Machines which are not sending heartbeats:

Broken build machines

-

Machines which have not had a successful build for over a week:

+

Machines with failing builds:

-

Build machines in limbo

-

Machines with a clean build that have not uploaded binary for that build:

- -

Force build now

Requires build engineer status.

diff --git a/extra/webapps/mason/dashboard/dashboard.factor b/extra/webapps/mason/dashboard/dashboard.factor index 82002778f3..4f49b52de6 100644 --- a/extra/webapps/mason/dashboard/dashboard.factor +++ b/extra/webapps/mason/dashboard/dashboard.factor @@ -20,7 +20,6 @@ IN: webapps.mason.downloads funny-builders [ builder-list ] tri@ [ "crashed" set-value ] - [ "broken" set-value ] - [ "limbo" set-value ] tri* + [ "broken" set-value ] bi* ] with-mason-db ] >>init ; diff --git a/extra/webapps/mason/download-package.xml b/extra/webapps/mason/download-package.xml index 9d18b6ed7c..ff366fb4f4 100644 --- a/extra/webapps/mason/download-package.xml +++ b/extra/webapps/mason/download-package.xml @@ -24,7 +24,7 @@ - + diff --git a/extra/webapps/mason/package/package.factor b/extra/webapps/mason/package/package.factor index 504ba7093f..ba04d25ec7 100644 --- a/extra/webapps/mason/package/package.factor +++ b/extra/webapps/mason/package/package.factor @@ -13,13 +13,16 @@ IN: webapps.mason.package : status-string ( builder -- string ) dup status>> { - { +dirty+ [ drop "Dirty" ] } - { +clean+ [ drop "Clean" ] } - { +error+ [ drop "Error" ] } + { +idle+ [ drop "Idle" ] } { +starting+ [ "Starting build" building ] } { +make-vm+ [ "Compiling VM" building ] } { +boot+ [ "Bootstrapping" building ] } { +test+ [ "Testing" building ] } + { +upload+ [ "Uploading package" building ] } + { +finish+ [ "Finishing build" building ] } + { +dirty+ [ drop "Dirty" ] } + { +clean+ [ drop "Clean" ] } + { +error+ [ drop "Error" ] } [ 2drop "Unknown" ] } case ; @@ -63,6 +66,7 @@ IN: webapps.mason.package [ release-git-id>> git-link "git-id" set-value ] [ requirements "requirements" set-value ] [ host-name>> "host-name" set-value ] + [ heartbeat-timestamp>> "heartbeat-timestamp" set-value ] [ current-status "status" set-value ] [ last-build-status "last-build" set-value ] [ clean-build-status "last-clean-build" set-value ] diff --git a/extra/webapps/mason/status-update/status-update.factor b/extra/webapps/mason/status-update/status-update.factor index e19d40e197..8d45c8e44a 100644 --- a/extra/webapps/mason/status-update/status-update.factor +++ b/extra/webapps/mason/status-update/status-update.factor @@ -5,24 +5,38 @@ furnace.redirection html.forms http.server.responses io kernel mason.server namespaces validators webapps.mason.utils ; IN: webapps.mason.status-update -: find-builder ( -- builder ) +: find-builder ( host-name os cpu -- builder ) builder new - "host-name" value >>host-name - "target-os" value >>os - "target-cpu" value >>cpu + swap >>cpu + swap >>os + swap >>host-name dup select-tuple [ ] [ dup insert-tuple ] ?if ; -: git-id ( builder id -- ) >>current-git-id +starting+ >>status drop ; +: heartbeat ( builder -- ) + now >>heartbeat-timestamp + drop ; -: make-vm ( builder -- ) +make-vm+ >>status drop ; +: status ( builder status -- ) + >>status + now >>current-timestamp + drop ; -: boot ( builder -- ) +boot+ >>status drop ; +: idle ( builder -- ) +idle+ status ; -: test ( builder -- ) +test+ >>status drop ; +: git-id ( builder id -- ) >>current-git-id +starting+ status ; -: report ( builder status content -- ) - [ >>status ] [ >>last-report ] bi* - dup status>> +clean+ = [ +: make-vm ( builder -- ) +make-vm+ status ; + +: boot ( builder -- ) +boot+ status ; + +: test ( builder -- ) +test+ status ; + +: report ( builder content status -- ) + [ + >>last-report + now >>current-timestamp + ] dip + +clean+ = [ dup current-git-id>> >>clean-git-id dup current-timestamp>> >>clean-timestamp ] when @@ -30,6 +44,10 @@ IN: webapps.mason.status-update dup current-timestamp>> >>last-timestamp drop ; +: upload ( builder -- ) +upload+ status ; + +: finish ( builder -- ) +finish+ status ; + : release ( builder name -- ) >>last-release dup clean-git-id>> >>release-git-id @@ -37,12 +55,15 @@ IN: webapps.mason.status-update : update-builder ( builder -- ) "message" value { - { "heartbeat" [ drop ] } + { "heartbeat" [ heartbeat ] } + { "idle" [ idle ] } { "git-id" [ "arg" value git-id ] } { "make-vm" [ make-vm ] } { "boot" [ boot ] } { "test" [ test ] } - { "report" [ "arg" value "report" value report ] } + { "report" [ "report" value "arg" value report ] } + { "upload" [ upload ] } + { "finish" [ finish ] } { "release" [ "arg" value release ] } } case ; @@ -63,8 +84,10 @@ IN: webapps.mason.status-update [ [ + "host-name" value + "target-os" value + "target-cpu" value find-builder - now >>current-timestamp [ update-builder ] [ update-tuple ] bi ] with-mason-db "OK" "text/plain"
Host name:
Last heartbeat:
Last heartbeat:
Current status:
Last build:
Last clean build: