This patch fixes a bug where the daemon hasn't finished the shutdown of a
worker process when another worker process gets started.  The shudown of the
first worker ends up using variables belonging to the new worker process.

Symptoms of this problem are a hang or segfault during failover of resources
in a cluster.

--- evms-2.3.3/engine/daemon.c	2004-04-29 14:28:01.000000000 -0500
+++ evms-2.3.3-fix/engine/daemon.c	2004-05-21 15:43:53.000000000 -0500
@@ -98,13 +98,15 @@
 	} while ((rc == EAGAIN) && (retry > 0));				\
 }
 
-int worker_pid = 0;
-
-int input_pipe[2];
-int output_pipe[2];
-static pthread_mutex_t input_pipe_mutex  = PTHREAD_MUTEX_INITIALIZER;
-static pthread_mutex_t output_pipe_mutex = PTHREAD_MUTEX_INITIALIZER;
+typedef struct worker_context_s {
+	int pid;
+        int input_pipe[2];
+	int output_pipe[2];
+        pthread_mutex_t input_pipe_mutex;
+	pthread_mutex_t output_pipe_mutex;
+} worker_context_t;
 
+worker_context_t * worker = NULL;
 
 typedef struct msg_pool_ent_s {
 	element_t element;
@@ -310,10 +312,10 @@
 	input_msg_hdr.cmd       = msg->cmd;
 	input_msg_hdr.size      = msg->size;
 
-        pthread_mutex_lock(&input_pipe_mutex);
+        pthread_mutex_lock(&worker->input_pipe_mutex);
 
-	LOG_DEBUG("Request to write %zd bytes to fd %d.\n", sizeof(input_msg_hdr), input_pipe[1]);
-	bytes_written = write(input_pipe[1], &input_msg_hdr,  sizeof(input_msg_hdr));
+	LOG_DEBUG("Request to write %zd bytes to fd %d.\n", sizeof(input_msg_hdr), worker->input_pipe[1]);
+	bytes_written = write(worker->input_pipe[1], &input_msg_hdr,  sizeof(input_msg_hdr));
 	if (bytes_written == -1) {
 		LOG_DEBUG("Write of message header failed with errno %d: %s.\n", errno, strerror(errno));
 		LOG_PROC_EXIT_INT(errno);
@@ -323,8 +325,8 @@
 	}
 
 	if (msg->size > 0) {
-		LOG_DEBUG("Request to write %zd bytes to fd %d.\n", msg->size, input_pipe[1]);
-		bytes_written = write(input_pipe[1], msg->msg, msg->size);
+		LOG_DEBUG("Request to write %zd bytes to fd %d.\n", msg->size, worker->input_pipe[1]);
+		bytes_written = write(worker->input_pipe[1], msg->msg, msg->size);
 		if (bytes_written == -1) {
 			LOG_DEBUG("Write of message failed with errno %d: %s.\n", errno, strerror(errno));
 			LOG_PROC_EXIT_INT(errno);
@@ -334,7 +336,7 @@
 		}
 	}
 
-        pthread_mutex_unlock(&input_pipe_mutex);
+        pthread_mutex_unlock(&worker->input_pipe_mutex);
 
 	LOG_PROC_EXIT_INT(0);
 	return 0;
@@ -348,10 +350,10 @@
 
 	LOG_PROC_ENTRY();
 
-	pthread_mutex_lock(&output_pipe_mutex);
+	pthread_mutex_lock(&worker->output_pipe_mutex);
 
-	LOG_DEBUG("Request to read %zd bytes from fd %d.\n", sizeof(output_msg_hdr), output_pipe[0]);
-	bytes_read = read(output_pipe[0], &output_msg_hdr, sizeof(output_msg_hdr));
+	LOG_DEBUG("Request to read %zd bytes from fd %d.\n", sizeof(output_msg_hdr), worker->output_pipe[0]);
+	bytes_read = read(worker->output_pipe[0], &output_msg_hdr, sizeof(output_msg_hdr));
 	if (bytes_read == -1) {
 		LOG_SERIOUS("Read of message failed with errno %d: %s.\n", errno, strerror(errno));
 		LOG_PROC_EXIT_INT(errno);
@@ -374,8 +376,8 @@
 			return ENOMEM;
 		}
 
-		LOG_DEBUG("Request to read %d bytes from fd %d.\n", output_msg_hdr.size, output_pipe[0]);
-		bytes_read = read(output_pipe[0], response_msg->msg, output_msg_hdr.size);
+		LOG_DEBUG("Request to read %d bytes from fd %d.\n", output_msg_hdr.size, worker->output_pipe[0]);
+		bytes_read = read(worker->output_pipe[0], response_msg->msg, output_msg_hdr.size);
 		if (bytes_read == -1) {
 			LOG_SERIOUS("Read of message failed with errno %d: %s.\n", errno, strerror(errno));
 			LOG_PROC_EXIT_INT(errno);
@@ -383,9 +385,12 @@
 		} else {
 			LOG_DEBUG("%d of %d bytes read.\n", bytes_read, output_msg_hdr.size);
 		}
+
+	} else {
+		response_msg->msg = NULL;
 	}
 
-	pthread_mutex_unlock(&output_pipe_mutex);
+	pthread_mutex_unlock(&worker->output_pipe_mutex);
 
 	LOG_PROC_EXIT_INT(0);
 	return 0;
@@ -452,37 +457,53 @@
 
 	rc = receive_response_for_command(response_msg);
 
-	if (rc != 0) {
+	if (rc == 0) {
+		SEND_MSG(response_msg);
+
+		/* engine_free() handles NULL pointers. */
+		engine_free(response_msg->msg);
+
+	} else {
 		evms_host_to_net(&net_rc, int_f, rc);
 		response_msg->cmd |= COMMAND_RESPONSE;
 		response_msg->size = sizeof(net_rc);
 		response_msg->msg  = &net_rc;
+		SEND_MSG(response_msg);
 	}
 
-	SEND_MSG(response_msg);
-		
-	/* engine_free() handles NULL pointers. */
-	engine_free(response_msg->msg);
-
 	free_msg(response_msg);
 	LOG_PROC_EXIT_VOID();
 }
 
 
-static void shutdown_worker(void) {
+static void shutdown_worker(worker_context_t * wrkr) {
 
 	int status;
 	pid_t pid;
 	int timeout = 5;  /* seconds */
+	
+	LOG_PROC_ENTRY();
+
+	LOG_DEBUG("Worker context: %p   worker pid: %d\n", wrkr, wrkr->pid);
+
+	close(wrkr->input_pipe[0]);
+	close(wrkr->input_pipe[1]);
+	close(wrkr->output_pipe[0]);
+	close(wrkr->output_pipe[1]);
+
+	/* Update the node field in the lock file. */
+	lock_file->node[0] = '\0';
+	lseek(lock_file_fd, offsetof(lock_file_t, node), SEEK_SET);
+	write(lock_file_fd, lock_file->node, 1);
 
 	/* Nicely ask the worker to shutdown. */
-	kill(worker_pid, SIGTERM);
+	kill(wrkr->pid, SIGTERM);
 
 	do {
-		pid = waitpid(worker_pid, &status, WNOHANG);
+		pid = waitpid(wrkr->pid, &status, WNOHANG);
 
 		if (pid == 0) {
-			LOG_DEBUG("Wait for worker to exit.\n");
+			LOG_DEBUG("Wait for worker pid %d to exit.\n", wrkr->pid);
 			usleep(100000);
 			timeout--;
 		}
@@ -491,24 +512,18 @@
 
 	if (pid == 0) {
 		/* Nice didn't work.  Kill it. */
-		LOG_DEBUG("Kill the worker.\n");
-                kill(worker_pid, SIGKILL);
+		LOG_DEBUG("Kill worker pid %d.\n", wrkr->pid);
+                kill(wrkr->pid, SIGKILL);
 	}
 
-	/* Wait for the worker to terminate and cleanup the defuct process. */
-        pid = waitpid(worker_pid, &status, 0);
-	
-	worker_pid = 0;
+	/* Wait for the worker to terminate and cleanup the defunct process. */
+        pid = waitpid(wrkr->pid, &status, 0);
+	LOG_DEBUG("Worker pid %d is dead.\n", wrkr->pid);
 
-	/* Update the node field in the lock file. */
-	lock_file->node[0] = '\0';
-	lseek(lock_file_fd, offsetof(lock_file_t, node), SEEK_SET);
-	write(lock_file_fd, lock_file->node, 1);
+	LOG_DEBUG("Free worker context: %p\n", wrkr);
+	engine_free(wrkr);
 
-	close(input_pipe[0]);
-	close(input_pipe[1]);
-	close(output_pipe[0]);
-	close(output_pipe[1]);
+	LOG_PROC_EXIT_VOID();
 }
 
 
@@ -532,11 +547,12 @@
 	 * in while this worker should be running.  The protocol won't do that.
 	 */
 	if (worker_running) {
-		worker_running = FALSE;
-		shutdown_worker();
+		shutdown_worker(worker);
 	}
 
-	if (pipe(input_pipe)) {
+	worker = engine_alloc(sizeof(worker_context_t));
+
+	if (worker == NULL) {
 		response_msg = get_msg(msg);
 
 		evms_host_to_net(&net_rc, int_f, errno);
@@ -553,9 +569,12 @@
 		return;
 	}
 
-	LOG_DEBUG("input_pipe handles are %d (read) and %d (write).\n", input_pipe[0], input_pipe[1]);
+	LOG_DEBUG("new worker context: %p\n", worker);
+
+	pthread_mutex_init(&worker->input_pipe_mutex, NULL);
+	pthread_mutex_init(&worker->output_pipe_mutex, NULL);
 
-	if (pipe(output_pipe)) {
+	if (pipe(worker->input_pipe)) {
 		response_msg = get_msg(msg);
 
 		evms_host_to_net(&net_rc, int_f, errno);
@@ -568,18 +587,43 @@
 
 		free_msg(response_msg);
 
-		close(input_pipe[0]);
-		close(input_pipe[1]);
+		engine_free(worker);
+		worker = NULL;
 
 		LOG_PROC_EXIT_VOID();
 		return;
 	}
 
-	LOG_DEBUG("output_pipe handles are %d (read) and %d (write).\n", output_pipe[0], output_pipe[1]);
+	LOG_DEBUG("worker->input_pipe handles are %d (read) and %d (write).\n", worker->input_pipe[0], worker->input_pipe[1]);
+
+	if (pipe(worker->output_pipe)) {
+		response_msg = get_msg(msg);
+
+		evms_host_to_net(&net_rc, int_f, errno);
+
+		response_msg->cmd |= COMMAND_RESPONSE;
+		response_msg->size = sizeof(net_rc);
+		response_msg->msg  = &net_rc;
 
-	worker_pid = fork();
+		SEND_MSG(response_msg);
+
+		free_msg(response_msg);
+
+		close(worker->input_pipe[0]);
+		close(worker->input_pipe[1]);
+
+		engine_free(worker);
+		worker = NULL;
+
+		LOG_PROC_EXIT_VOID();
+		return;
+	}
+
+	LOG_DEBUG("worker->output_pipe handles are %d (read) and %d (write).\n", worker->output_pipe[0], worker->output_pipe[1]);
+
+	worker->pid = fork();
 	
-	switch (worker_pid) {
+	switch (worker->pid) {
 		case -1:
 			/* Fork failed.  Send back an error code. */
 			response_msg = get_msg(msg);
@@ -594,17 +638,19 @@
 
 			free_msg(response_msg);
 
-			close(input_pipe[0]);
-			close(input_pipe[1]);
-			close(output_pipe[0]);
-			close(output_pipe[1]);
+			close(worker->input_pipe[0]);
+			close(worker->input_pipe[1]);
+			close(worker->output_pipe[0]);
+			close(worker->output_pipe[1]);
+			engine_free(worker);
+			worker = NULL;
 
 			break;
 
 		case 0:
 			/* Child process */
-			dup2(input_pipe[0], 0);	/* Route stdin to input_pipe's read fd */
-			dup2(output_pipe[1], 1);/* Route stdout to output_pipe's write fd */
+			dup2(worker->input_pipe[0], 0);	/* Route stdin to worker->input_pipe's read fd */
+			dup2(worker->output_pipe[1], 1);/* Route stdout to worker->output_pipe's write fd */
 
 			LOG_DEFAULT("execvp(" WORKER_NAME ")\n");
 			execvp(WORKER_NAME, argv);
@@ -621,6 +667,7 @@
 			usleep(100000);
 
 			worker_running = TRUE;
+			LOG_DEBUG("New worker pid: %d\n", worker->pid);
 
 			/* Update the node field in the lock file. */
 			node_name = nodeid_to_string(&msg->node);
@@ -646,6 +693,8 @@
 
 static void process_close_engine(ece_msg_t * msg) {
 
+	worker_context_t * wrkr = worker;
+
 	LOG_PROC_ENTRY();
 
 	/*
@@ -659,7 +708,7 @@
 	/* Send the evms_close_engine() API to the worker. */
 	process_api(msg);
 
-	shutdown_worker();
+	shutdown_worker(wrkr);
 
 	LOG_PROC_EXIT_VOID();
 }