vm: interrupt stdin_loop when entering fep

The stdin_loop thread will keep trying to consume input unless we stop it by sending it a signal. Use SIGUSR2 to stop the read syscall and a mutex to hold up the loop while the fep is active.

vm/debug.cpp

@@ -438,6 +438,12 @@ void factor_vm::factorbug_usage(bool advanced_p)
 
 }
 
+static void exit_fep(factor_vm *vm)
+{
+	vm->unlock_console();
+	vm->fep_p = false;
+}
+
 void factor_vm::factorbug()
 {
 	if(fep_disabled)
@@ -452,6 +458,9 @@ void factor_vm::factorbug()
 	fep_p = true;
 
 	std::cout << "Starting low level debugger..." << std::endl;
+
+	lock_console();
+
 	if (!fep_help_was_shown) {
 		factorbug_usage(false);
 		fep_help_was_shown = true;
@@ -530,12 +539,12 @@ void factor_vm::factorbug()
 			dump_generations();
 		else if(strcmp(cmd,"c") == 0)
 		{
-			fep_p = false;
+			exit_fep(this);
 			return;
 		}
 		else if(strcmp(cmd,"t") == 0)
 		{
-			fep_p = false;
+			exit_fep(this);
 			general_error(ERROR_INTERRUPT,false_object,false_object);
 			assert(false);
 		}

vm/os-unix.cpp

@@ -317,8 +317,6 @@ void factor_vm::unix_init_signals()
 	sigaction_safe(SIGABRT,&synchronous_sigaction,NULL);
 
 	init_sigaction_with_handler(&enqueue_sigaction, enqueue_signal_handler);
-	sigaction_safe(SIGUSR1,&enqueue_sigaction,NULL);
-	sigaction_safe(SIGUSR2,&enqueue_sigaction,NULL);
 	sigaction_safe(SIGWINCH,&enqueue_sigaction,NULL);
 #ifdef SIGINFO
 	sigaction_safe(SIGINFO,&enqueue_sigaction,NULL);
@@ -336,6 +334,8 @@ void factor_vm::unix_init_signals()
 	io.launcher.unix for this. */
 	init_sigaction_with_handler(&ignore_sigaction, ignore_signal_handler);
 	sigaction_safe(SIGPIPE,&ignore_sigaction,NULL);
+	/* We send SIGUSR2 to the stdin_loop thread to interrupt it on FEP */
+	sigaction_safe(SIGUSR2,&ignore_sigaction,NULL);
 }
 
 /* On Unix, shared fds such as stdin cannot be set to non-blocking mode
@@ -355,6 +355,9 @@ extern "C" {
 
 	int size_read;
 	int size_write;
+
+	THREADHANDLE stdin_thread;
+	pthread_mutex_t stdin_mutex;
 }
 
 void safe_close(int fd)
@@ -404,13 +407,18 @@ void *stdin_loop(void *arg)
 	unsigned char buf[4096];
 	bool loop_running = true;
 
+	// If we fep, the parent thread will grab a mutex and send us SIGUSR2 to make
+	// us relinquish our hold on stdin.
 	sigset_t mask;
 	sigfillset(&mask);
-	pthread_sigmask(SIG_BLOCK, &mask, NULL);
+	sigdelset(&mask, SIGUSR2);
+	pthread_sigmask(SIG_SETMASK, &mask, NULL);
 
 	while(loop_running)
 	{
-		if(!safe_read(control_read,buf,1))
+		bool readp = safe_read(control_read,buf,1);
+
+		if(!readp)
 			break;
 
 		if(buf[0] != 'X')
@@ -418,6 +426,8 @@ void *stdin_loop(void *arg)
 
 		for(;;)
 		{
+			pthread_mutex_lock(&stdin_mutex);
+			pthread_mutex_unlock(&stdin_mutex);
 			ssize_t bytes = read(0,buf,sizeof(buf));
 			if(bytes < 0)
 			{
@@ -446,12 +456,24 @@ void *stdin_loop(void *arg)
 	return NULL;
 }
 
-void open_console()
+void factor_vm::open_console()
 {
 	safe_pipe(&control_read,&control_write);
 	safe_pipe(&size_read,&size_write);
 	safe_pipe(&stdin_read,&stdin_write);
-	start_thread(stdin_loop,NULL);
+	stdin_thread = start_thread(stdin_loop,NULL);
+	pthread_mutex_init(&stdin_mutex, NULL);
+}
+
+void factor_vm::lock_console()
+{
+	pthread_mutex_lock(&stdin_mutex);
+	pthread_kill(stdin_thread, SIGUSR2);
+}
+
+void factor_vm::unlock_console()
+{
+	pthread_mutex_unlock(&stdin_mutex);
 }
 
 }

vm/os-unix.hpp

@@ -9,6 +9,7 @@
 #include <dlfcn.h>
 #include <signal.h>
 #include <pthread.h>
+#include <sched.h>
 
 #include "atomic-gcc.hpp"
 
@@ -41,7 +42,6 @@ inline static THREADHANDLE thread_id() { return pthread_self(); }
 
 u64 nano_count();
 void sleep_nanos(u64 nsec);
-void open_console();
 
 void move_file(const vm_char *path1, const vm_char *path2);
 

vm/os-windows.cpp

@@ -301,6 +301,14 @@ void factor_vm::open_console()
 	SetConsoleCtrlHandler(factor::ctrl_handler, TRUE);
 }
 
+void factor_vm::lock_console()
+{
+}
+
+void factor_vm::unlock_console()
+{
+}
+
 void factor_vm::sampler_thread_loop()
 {
 	LARGE_INTEGER counter, new_counter, units_per_second;

vm/vm.hpp

@@ -736,6 +736,9 @@ struct factor_vm
 	void init_signals();
 	void start_sampling_profiler_timer();
 	void end_sampling_profiler_timer();
+	void open_console();
+	void lock_console();
+	void unlock_console();
 
 	// os-windows
   #if defined(WINDOWS)
@@ -747,7 +750,6 @@ struct factor_vm
 	void windows_image_path(vm_char *full_path, vm_char *temp_path, unsigned int length);
 	BOOL windows_stat(vm_char *path);
 
-	void open_console();
 	LONG exception_handler(PEXCEPTION_RECORD e, void *frame, PCONTEXT c, void *dispatch);
 
   #else  // UNIX