add infrastructure to receive higher order commands, like remove_device

From: Dan Williams <dan.j.williams@intel.com>

Each md_message encapsulates a single command.  A command includes an 'action'
member which describes what if any data comes after the action.  Communication
with the monitor involves updating the active_cmd pointer and then writing to
mgr_pipe.  Pass/fail status is returned via mon_pipe.

Signed-off-by: Dan Williams <dan.j.williams@intel.com>

managemon.c

@@ -74,10 +74,8 @@
 #endif
 #include	"mdadm.h"
 #include	"mdmon.h"
-#include	"msg.h"
 #include	<sys/socket.h>
 
-
 static void close_aa(struct active_array *aa)
 {
 	struct mdinfo *d;
@@ -108,7 +106,14 @@ static void free_aa(struct active_array *aa)
 
 static void write_wakeup(struct supertype *c)
 {
-	write(c->pipe[1], "PING", 4);
+	static struct md_generic_cmd cmd = { .action = md_action_ping_monitor };
+	int err;
+
+	active_cmd = &cmd;
+
+	/* send the monitor thread a pointer to the ping action */
+	write(c->mgr_pipe[1], &err, 1);
+	read(c->mon_pipe[0], &err, 1);
 }
 
 static void replace_array(struct supertype *container,
@@ -300,7 +305,29 @@ void manage(struct mdstat_ent *mdstat, struct active_array *aa,
 	}
 }
 
-void read_sock(int pfd)
+static int handle_message(struct supertype *container, struct md_message *msg)
+{
+	int err;
+	struct md_generic_cmd *cmd = msg->buf;
+
+	if (!cmd)
+		return 0;
+
+	switch (cmd->action) {
+	case md_action_remove_device:
+
+		/* forward to the monitor */
+		active_cmd = cmd;
+		write(container->mgr_pipe[1], &err, 1);
+		read(container->mon_pipe[0], &err, 1);
+		return err;
+
+	default:
+		return -1;
+	}
+}
+
+void read_sock(struct supertype *container)
 {
 	int fd;
 	struct md_message msg;
@@ -308,7 +335,7 @@ void read_sock(int pfd)
 	long fl;
 	int tmo = 3; /* 3 second timeout before hanging up the socket */
 
-	fd = accept(pfd, NULL, NULL);
+	fd = accept(container->sock, NULL, NULL);
 	if (fd < 0)
 		return;
 
@@ -317,12 +344,17 @@ void read_sock(int pfd)
 	fcntl(fd, F_SETFL, fl);
 
 	do {
+		int err;
+
 		msg.buf = NULL;
 
 		/* read and validate the message */
 		if (receive_message(fd, &msg, tmo) == 0) {
-			// FIXME: handle message contents
+			err = handle_message(container, &msg);
-			ack(fd, msg.seq, tmo);
+			if (!err)
+				ack(fd, msg.seq, tmo);
+			else
+				nack(fd, err, tmo);
 		} else {
 			terminate = 1;
 			nack(fd, -1, tmo);
@@ -343,7 +375,7 @@ void do_manager(struct supertype *container)
 
 		manage(mdstat, array_list, container);
 
-		read_sock(container->sock);
+		read_sock(container);
 
 		mdstat_wait_fd(container->sock);
 	} while(1);

mdadm.h

@@ -439,7 +439,8 @@ struct supertype {
 	struct active_array *arrays;
 	int devfd;
 	int sock; /* listen to external programs */
-	int pipe[2]; /* communicate between threads */
+	int mgr_pipe[2]; /* communicate between threads */
+	int mon_pipe[2]; /* communicate between threads */
 	int devnum;
 	char *devname; /* e.g. md0.  This appears in metadata_verison:
 			*  external:/md0/12

mdmon.c

@@ -44,6 +44,7 @@
 struct active_array *array_list;
 struct active_array *discard_this;
 struct active_array *pending_discard;
+struct md_generic_cmd *active_cmd;
 
 int run_child(void *v)
 {
@@ -54,19 +55,32 @@ int run_child(void *v)
 
 int clone_monitor(struct supertype *container)
 {
-	int pfd[2];
 	static char stack[4096];
 	int rv;
 
-	pipe(container->pipe);
+	rv = pipe(container->mgr_pipe);
+	if (rv < 0)
+		return rv;
+	rv = pipe(container->mon_pipe);
+	if (rv < 0)
+		goto err_mon_pipe;
 
 	rv = clone(run_child, stack+4096-64,
 		   CLONE_FS|CLONE_FILES|CLONE_VM|CLONE_SIGHAND|CLONE_THREAD,
 		   container);
-
 	if (rv < 0)
+		goto err_clone;
+	else
 		return rv;
-	return pfd[1];
+
+ err_clone:
+	close(container->mon_pipe[0]);
+	close(container->mon_pipe[1]);
+ err_mon_pipe:
+	close(container->mgr_pipe[0]);
+	close(container->mgr_pipe[1]);
+
+	return rv;
 }
 
 static struct superswitch *find_metadata_methods(char *vers)
@@ -156,7 +170,6 @@ static int make_control_sock(char *devname)
 int main(int argc, char *argv[])
 {
 	int mdfd;
-	int pipefd;
 	struct mdinfo *mdi, *di;
 	struct supertype *container;
 	if (argc != 2) {
@@ -254,8 +267,7 @@ int main(int argc, char *argv[])
 
 	mlockall(MCL_FUTURE);
 
-	pipefd = clone_monitor(container);
+	if (clone_monitor(container) < 0) {
-	if (pipefd < 0) {
 		fprintf(stderr, "md-manage: failed to start monitor process: %s\n",
 			strerror(errno));
 		exit(2);

mdmon.h

@@ -31,6 +31,7 @@ extern struct active_array *container;
 extern struct active_array *array_list;
 extern struct active_array *discard_this;
 extern struct active_array *pending_discard;
+extern struct md_generic_cmd *active_cmd;
 
 
 void do_monitor(struct supertype *container);

monitor.c

@@ -377,7 +377,60 @@ static void reconcile_failed(struct active_array *aa, struct mdinfo *failed)
 	}
 }
 
-static int wait_and_act(struct active_array *aa, int pfd, int nowait)
+static int handle_remove_device(struct md_remove_device_cmd *cmd, struct active_array *aa)
+{
+	struct active_array *a;
+	struct mdinfo *victim;
+	int rv;
+
+	/* scan all arrays for the given device, if ->state_fd is closed (-1)
+	 * in all cases then mark the disk as removed in the metadata.
+	 * Otherwise reply that it is busy.
+	 */
+
+	/* pass1 check that it is not in use anywhere */
+	/* note: we are safe from re-adds as long as the device exists in the
+	 * container
+	 */
+	for (a = aa; a; a = a->next) {
+		if (!a->container)
+			continue;
+		victim = find_device(a, major(cmd->rdev), minor(cmd->rdev));
+		if (!victim)
+			continue;
+		if (victim->state_fd > 0)
+			return -EBUSY;
+	}
+
+	/* pass2 schedule and process removal per array */
+	for (a = aa; a; a = a->next) {
+		if (!a->container)
+			continue;
+		victim = find_device(a, major(cmd->rdev), minor(cmd->rdev));
+		if (!victim)
+			continue;
+		victim->curr_state |= DS_REMOVE;
+		rv = read_and_act(a);
+		if (rv < 0)
+			return rv;
+	}
+
+	return 0;
+}
+
+static int handle_pipe(struct md_generic_cmd *cmd, struct active_array *aa)
+{
+	switch (cmd->action) {
+	case md_action_ping_monitor:
+		return 0;
+	case md_action_remove_device:
+		return handle_remove_device((void *) cmd, aa);
+	}
+
+	return -1;
+}
+
+static int wait_and_act(struct active_array *aa, int pfd, int monfd, int nowait)
 {
 	fd_set rfds;
 	int maxfd = 0;
@@ -408,9 +461,11 @@ static int wait_and_act(struct active_array *aa, int pfd, int nowait)
 			return rv;
 
 		if (FD_ISSET(pfd, &rfds)) {
-			char buf[4];
+			int err = -1;
-			read(pfd, buf, 4);
+
-			; // FIXME read from the pipe
+			if (read(pfd, &err, 1) > 0)
+				err = handle_pipe(active_cmd, aa);
+			write(monfd, &err, 1);
 		}
 	}
 
@@ -446,7 +501,8 @@ void do_monitor(struct supertype *container)
 	int rv;
 	int first = 1;
 	do {
-		rv = wait_and_act(container->arrays, container->pipe[0], first);
+		rv = wait_and_act(container->arrays, container->mgr_pipe[0],
+				  container->mon_pipe[1], first);
 		first = 0;
 	} while (rv >= 0);
 }

msg.h

@@ -17,8 +17,8 @@
  * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  */
 
-#ifndef _MSG_H
+
-#define _MSG_H
+struct mdinfo;
 
 struct mdinfo;
 struct md_message {
@@ -60,4 +60,3 @@ extern int connect_monitor(char *devname);
 extern int ping_monitor(char *devname);
 extern int send_remove_device(int fd, dev_t rdev, int seq, int tmo);
 
-#endif