Merge remote-tracking branch 'origin/master' into modern-harvey3

GNUmakefile

@@ -1,8 +1,9 @@
 ifdef CONFIG
 	VERSION = 0.99
 	GIT_LABEL = $(shell echo `git describe --all`-`git rev-parse HEAD`)
-
 	BUNDLE = Factor.app
+	DEBUG ?= 0
+	REPRODUCIBLE ?= 0
 
 	include $(CONFIG)
 
@@ -14,12 +15,16 @@ ifdef CONFIG
 
 	CXXFLAGS += -std=c++11
 
-	ifdef DEBUG
+	ifneq ($(DEBUG), 0)
 		CFLAGS += -g -DFACTOR_DEBUG
 	else
 		CFLAGS += -O3
 	endif
 
+	ifneq ($(REPRODUCIBLE), 0)
+		CFLAGS += -DFACTOR_REPRODUCIBLE
+	endif
+
 	ENGINE = $(DLL_PREFIX)factor$(DLL_SUFFIX)$(DLL_EXTENSION)
 	EXECUTABLE = factor$(EXE_SUFFIX)$(EXE_EXTENSION)
 	CONSOLE_EXECUTABLE = factor$(EXE_SUFFIX)$(CONSOLE_EXTENSION)
@@ -150,6 +155,7 @@ help:
 	@echo "Additional modifiers:"
 	@echo ""
 	@echo "DEBUG=1  compile VM with debugging information"
+	@echo "REPRODUCIBLE=1  compile VM without timestamp"
 	@echo "SITE_CFLAGS=...  additional optimization flags"
 	@echo "X11=1  force link with X11 libraries instead of Cocoa (only on Mac OS X)"
 

Nmakefile

@@ -65,6 +65,10 @@ LINK_FLAGS = $(LINK_FLAGS) /DEBUG
 CL_FLAGS = $(CL_FLAGS) /Zi /DFACTOR_DEBUG
 !ENDIF
 
+!IF DEFINED(REPRODUCIBLE)
+CL_FLAGS = $(CL_FLAGS) /DFACTOR_REPRODUCIBLE
+!ENDIF
+
 ML_FLAGS = /nologo /safeseh
 
 EXE_OBJS = vm\main-windows.obj vm\factor.res

basis/math/functions/functions-tests.factor

@@ -268,6 +268,7 @@ CONSTANT: log10-factorial-1000 0x1.40f3593ed6f8ep11
 { t } [ 3 15 roots [ 15 ^ 3 .01 ~ ] all? ] unit-test
 
 { .5 } [ 0 sigmoid ] unit-test
+{ t } [ 0 [ sigmoid logit ] keep .000001 ~ ] unit-test
 
 { 1 } [ 12 signum ] unit-test
 { -1 } [ -5.0 signum ] unit-test

basis/math/functions/functions.factor

@@ -434,8 +434,12 @@ M: float round-to-odd [ (round-to-odd?) ] (float-round) ;
     [ recip 2pi * 0 swap complex boa e^ ]
     [ <iota> [ ^ * ] 2with map ] tri ;
 
+! expit
 : sigmoid ( x -- y ) neg e^ 1 + recip ; inline
 
+: logit ( x -- y ) [ ] [ 1 swap - ] bi /f log ; inline
+
+
 GENERIC: signum ( x -- y )
 
 M: real signum sgn ;

build.sh

@@ -357,6 +357,7 @@ echo_build_info() {
     $ECHO NUM_CORES=$NUM_CORES
     $ECHO WORD=$WORD
     $ECHO DEBUG=$DEBUG
+    $ECHO REPRODUCIBLE=$REPRODUCIBLE
     $ECHO CURRENT_BRANCH=$CURRENT_BRANCH
     $ECHO FACTOR_BINARY=$FACTOR_BINARY
     $ECHO FACTOR_LIBRARY=$FACTOR_LIBRARY
@@ -734,7 +735,7 @@ usage() {
     $ECHO "  bootstrap - bootstrap with existing boot image"
     $ECHO "  net-bootstrap - recompile, download a boot image, bootstrap"
     $ECHO "  make-target - find and print the os-arch-cpu string"
-    $ECHO "  report - print the build variables"
+    $ECHO "  report|info - print the build variables"
     $ECHO "  update-boot-image - get the boot image for the current branch of for master"
     $ECHO ""
     $ECHO "If you are behind a firewall, invoke as:"
@@ -751,6 +752,13 @@ if [[ -n "$2" ]] ; then
     parse_build_info $2
 fi
 
+if [ "$#" -gt 3 ]; then
+	usage
+    $ECHO "error: too many arguments"
+    exit 1
+fi
+
+
 set_copy
 set_delete
 
@@ -763,13 +771,12 @@ case "$1" in
     deps-pkg) install_deps_pkg ;;
     self-update) update; make_boot_image; bootstrap;;
     quick-update) update; refresh_image ;;
-    update) update; download_and_bootstrap ;;
+    update|latest) update; download_and_bootstrap ;;
-    latest) update; download_and_bootstrap ;;
     compile) find_build_info; make_factor ;;
     bootstrap) get_config_info; bootstrap ;;
     net-bootstrap) net_bootstrap_no_pull ;;
     make-target) FIND_MAKE_TARGET=true; ECHO=false; find_build_info; exit_script ;;
-    report) find_build_info ;;
+    report|info) find_build_info ;;
     full-report) find_build_info; check_installed_programs; check_libraries ;;
     update-boot-image) find_build_info; check_installed_programs; update_boot_image;;
     *) usage ;;

extra/hacker-news/hacker-news.factor

@@ -9,20 +9,34 @@ kernel make math math.parser sequences ui urls vocabs ;
 IN: hacker-news
 
 <PRIVATE
-
+: hacker-news-ids ( endpoint -- ids )
-: hacker-news-recent-ids ( -- seq )
+    "https://hacker-news.firebaseio.com/v0/%s.json?print=pretty" sprintf
-    "https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty"
     http-get nip json> ;
 
 : hacker-news-id>json-url ( n -- url )
-    number>string
+    "https://hacker-news.firebaseio.com/v0/item/%d.json?print=pretty" sprintf ;
-    "https://hacker-news.firebaseio.com/v0/item/" ".json?print=pretty" surround ;
 
-: hacker-news-items ( seq -- seq' )
+: hacker-news-items ( n endpoint -- seq' )
+    hacker-news-ids swap short head
     [ hacker-news-id>json-url http-get nip json> ] parallel-map ;
 
-: hacker-news-recent-items ( n -- seq )
+: hacker-news-top-stories ( n -- seq )
-    [  hacker-news-recent-ids ] dip head hacker-news-items ;
+    "topstories" hacker-news-items ;
+
+: hacker-news-new-stories ( n -- seq )
+    "newstories" hacker-news-items ;
+
+: hacker-news-best-stories ( n -- seq )
+    "beststories" hacker-news-items ;
+
+: hacker-news-ask-stories ( n -- seq )
+    "askstories" hacker-news-items ;
+
+: hacker-news-show-stories ( n -- seq )
+    "showstories" hacker-news-items ;
+
+: hacker-news-job-stories ( n -- seq )
+    "jobstories" hacker-news-items ;
 
 : write-title ( title url -- )
     '[
@@ -69,8 +83,7 @@ PRIVATE>
         ]
     } cleave ;
 
-: banner. ( -- )
+: banner. ( str -- )
-    "Hacker News"
     "http://news.ycombinator.com" >url presented associate
     H{
         { font-size 20 }
@@ -79,7 +92,39 @@ PRIVATE>
         { foreground color: black }
     } assoc-union format nl ;
 
-: hacker-news. ( -- )
+: hacker-news-feed. ( seq -- )
-    30 hacker-news-recent-items
-    banner.
     [ 1 + post. ] each-index ;
+
+: hacker-news. ( str seq -- )
+    [ banner. ]
+    [ hacker-news-feed. ] bi* ;
+
+: hacker-news-top. ( -- )
+    "Hacker News - Top"
+    30 hacker-news-top-stories
+    hacker-news. ;
+
+: hacker-news-new. ( -- )
+    "Hacker News - New"
+    50 hacker-news-new-stories
+    hacker-news. ;
+
+: hacker-news-best. ( -- )
+    "Hacker News - Best"
+    50 hacker-news-best-stories
+    hacker-news. ;
+
+: hacker-news-ask. ( -- )
+    "Hacker News - Ask"
+    50 hacker-news-ask-stories
+    hacker-news. ;
+
+: hacker-news-show. ( -- )
+    "Hacker News - Show"
+    50 hacker-news-show-stories
+    hacker-news. ;
+
+: hacker-news-job. ( -- )
+    "Hacker News - Job"
+    50 hacker-news-job-stories
+    hacker-news. ;

extra/machine-learning/data-sets/data-sets.factor

@@ -1,9 +1,12 @@
-! Copyright (C) 2012 John Benediktsson
+! Copyright (C) 2012 John Benediktsson, Doug Coleman
 ! See http://factorcode.org/license.txt for BSD license
 
-USING: accessors ascii assocs csv io.encodings.utf8 io.files
+USING: accessors arrays ascii assocs byte-arrays combinators
-kernel math.parser sequences splitting ;
+combinators.short-circuit concurrency.combinators csv grouping
-
+http.client images images.viewer io io.directories
+io.encodings.binary io.encodings.utf8 io.files io.launcher
+io.pathnames kernel math math.parser namespaces sequences
+splitting ui.gadgets.panes ;
 IN: machine-learning.data-sets
 
 TUPLE: data-set
@@ -61,3 +64,55 @@ PRIVATE>
         "linnerud_physiological.csv" load-table
         [ >>targets ] [ >>target-names ] bi*
         "linnerud.rst" load-file >>description ;
+
+: download-to-directory ( url directory -- )
+    dup make-directories
+    [
+        dup { [ download-name exists? ] [ file-stem exists? ] } 1|| [
+            drop
+        ] [
+            download
+        ] if
+    ] with-directory ;
+
+: gzip-decompress-file ( path -- )
+    { "gzip" "-d" } swap suffix run-process drop ;
+
+: mnist-data>array ( bytes -- seq )
+    16 tail-slice 28 28 * <groups> [
+        >byte-array <image>
+            swap >>bitmap
+            { 28 28 } >>dim
+            L >>component-order
+            ubyte-components >>component-type
+    ] map ;
+
+: mnist-labels>array ( bytes -- seq )
+    8 tail-slice >array ;
+
+: image-grid. ( image-seq -- )
+    [
+        [
+            <image-gadget> output-stream get write-gadget
+        ] each
+        output-stream get stream-nl
+    ] each ;
+
+: load-mnist ( -- data-set )
+    "resource:datasets" dup make-directories [
+        {
+            "http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz"
+            "http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz"
+            "http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz"
+            "http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz"
+        }
+        [ [ "resource:datasets/" download-to-directory ] parallel-each ]
+        [ [ dup file-stem exists? [ drop ] [ file-name gzip-decompress-file ] if ] each ]
+        [ [ file-stem binary file-contents ] map ] tri
+        first4 {
+            [ mnist-data>array ]
+            [ mnist-labels>array ]
+            [ mnist-data>array ]
+            [ mnist-labels>array ]
+        } spread 4array
+    ] with-directory ;

vm/Config.unix

@@ -1,4 +1,4 @@
-ifndef DEBUG
+ifneq ($(DEBUG), 0)
 	SITE_CFLAGS += -fomit-frame-pointer
 endif
 

vm/master.hpp

@@ -52,8 +52,12 @@
 #define FACTOR_COMPILER_VERSION "unknown"
 #endif
 
-// Record compilation time
+#if defined(FACTOR_REPRODUCIBLE)
-#define FACTOR_COMPILE_TIME __DATE__ " " __TIME__
+  #define FACTOR_COMPILE_TIME "[reproducible]"
+#else
+  // Record compilation time
+  #define FACTOR_COMPILE_TIME  __DATE__ " " __TIME__
+#endif
 
 // Detect target CPU type
 #if defined(__arm__)