--- /dev/null	Sat Nov 22 13:16:58 2014
+++ new/usr/src/test/os-tests/tests/tcpdefer/defer_test.c	Sat Nov 22 13:16:58 2014
@@ -0,0 +1,556 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2014 Ryan Zezeski
+ */
+
+/*
+ * Test TCP deferred accept(). Specifically, test the data/httpfilt
+ * socket filter modules. Deferred accept() allows the kernel to defer
+ * the return of accept() on behalf of the application for the purpose
+ * of making sure data is ready so that the first read() call does not
+ * block or return EAGAIN. This test exercises the code path from the
+ * application level, i.e. entire stack is included.
+ *
+ * datafilt - This module is generic TCP deferment. It assures 1 byte
+ * is ready before accept() returns.
+ *
+ * httpfilt - This module is for HTTP/TCP deferment. It guesses if a
+ * request is valid HTTP and defers accept() until the entire request
+ * is ready, sans the body.
+ */
+#include <arpa/inet.h>
+#include <errno.h>
+#include <kstat.h>
+#include <netinet/in.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#define	BUFSIZE			1024
+#define	COMMON_HEAD				       \
+	"Accept: */*\r\n"			       \
+	"Host: test.com\r\n"			       \
+	"User-Agent: test\r\n"			       \
+	"\r\n"
+#define	DATAFILT		"datafilt"
+#define	HTTPFILT		"httpfilt"
+#define	NUM_TESTS		12
+#define	PORT			9876
+
+/*
+ * Values used to indicate client/server state to each other.
+ */
+enum State {
+	/* client states */
+	CLIENT_CONNECTED,
+	CLIENT_SENT_ALL,
+	CLIENT_SENT_SOME,
+
+	/* server states */
+	SERVER_LISTENING,
+	SERVER_NO_ACCEPT
+};
+
+/*
+ * Each test is defined as a tdata structure in the test_data array.
+ * Adding a new test is a matter of making a new tdata structure and
+ * bumping NUM_TESTS.
+ *
+ * td_type	The filter to use: HTTPFILT | DATAFILT.
+ *
+ * td_name	The name of the test, written to stdout.
+ *
+ * td_msgs	An array of data msgs that will be sent by the client()
+ *		process.
+ *
+ * td_num_msgs	The number of msgs in td_msgs.
+ */
+struct tdata {
+	char *td_type;
+	char *td_name;
+	char *td_msgs[2];
+	int td_num_msgs;
+};
+
+static struct tdata test_data[NUM_TESTS] = {
+	{
+		HTTPFILT,
+		"httpfilt_GET",
+		{"GET /test HTTP/1.1\r\n", COMMON_HEAD},
+		2
+	},
+	{
+		HTTPFILT,
+		"httpfilt_GET_LF_only",
+		{"GET /test HTTP/1.1\n",
+			"Accept: */*\n"				       \
+			"Host: test.com\n"			       \
+			"User-Agent: test\n"			       \
+			"\n"},
+		2
+	},
+	{
+		HTTPFILT,
+		"httpfilt_PUT",
+		{"PUT /test HTTP/1.1\r\n", COMMON_HEAD},
+		2
+	},
+	{
+		HTTPFILT,
+		"httpfilt_POST",
+		{"POST /test HTTP/1.1\r\n", COMMON_HEAD},
+		2
+	},
+	{
+		HTTPFILT,
+		"httpfilt_HEAD",
+		{"HEAD /test HTTP/1.1\r\n", COMMON_HEAD},
+		2
+	},
+	{
+		HTTPFILT,
+		"httpfilt_OPTIONS",
+		{"OPTIONS /test HTTP/1.1\r\n", COMMON_HEAD},
+		2
+	},
+	{
+		HTTPFILT,
+		"httpfilt_TRACE",
+		{"TRACE /test HTTP/1.1\r\n", COMMON_HEAD},
+		2
+	},
+	{
+		HTTPFILT,
+		"httpfilt_CONNECT",
+		{"CONNECT /test HTTP/1.1\r\n", COMMON_HEAD},
+		2
+	},
+	{
+		HTTPFILT,
+		"httpfilt_VERSION-CONTROL",
+		{"VERSION-CONTROL /test HTTP/1.1\r\n", COMMON_HEAD},
+		2
+	},
+	{
+		/*
+		 * Verify that a starting space (non-HTTP) is not
+		 * deferred.
+		 */
+		HTTPFILT,
+		"httpfilt_space",
+		{" some data..."},
+		1
+	},
+	{
+		/* Verify that a bad method is not deferred. */
+		HTTPFILT,
+		"httpfilt_bad_method",
+		{"badmethod /test HTTP/1.1\r\n"},
+		1
+	},
+	{
+		DATAFILT,
+		"datafilt_one_byte",
+		{"X"},
+		1
+	}
+};
+
+static int	debug = 0;
+
+static int	client(int fd, char **msgs, int num_msgs);
+static int	server(int fd, char *td_type, char **msgs, int num_msgs);
+
+static void	dbg(const char *format, ...);
+static int	msgscmp(char *data, char **msgs, int num_msgs);
+static int	run(struct tdata tdata);
+static int	sendstate(int fd, enum State state);
+static int	waitforstate(int fd, enum State state);
+
+int
+main(int argc, char **argv)
+{
+	int c, rc = 0;
+
+	while ((c = getopt(argc, argv, "d")) != -1) {
+		switch (c) {
+		case 'd':
+			debug = 1;
+			break;
+		default:
+			break;
+		}
+	}
+
+	for (int i = 0; i < NUM_TESTS; i++)
+		rc += run(test_data[i]);
+
+	return (rc);
+}
+
+/*
+ * This test is made up of two cooperating processes which share state
+ * updates over a pipe. The sequence diagram below shows the actions
+ * taken on each side of the socket as well as the state messages. The
+ * numbers represent the order, two steps with the same number happen
+ * concurrently. The state messages act as synchronization points; a
+ * process will block until it receives the expected message. Notice
+ * that a server may accept multiple times before the socket is
+ * created, this proves deferment. A non-deferred socket will accept
+ * on the first try.
+ *
+ * CLI SOCK	CLIENT				SERVER	SERVER SOCK
+ *                |                                |
+ *                |                                |    (1) listen()
+ *                |<--------SERVER_LISTENING-------+
+ * (2) connect()  |                                |
+ *                +---------CLIENT_CONNECTED------>|
+ *                |                                |    (3) accept()
+ *                |<--------SERVER_NO_ACCEPT-------+
+ * (4.x) send()   |                                |
+ *                +---------CLIENT_SENT_SOME------>|
+ *                |                                |    (5.x) accept()
+ *                |<--------SERVER_NO_ACCEPT-------+
+ *                |                                |
+ *                +---------CLIENT_SENT_ALL------->|
+ * (6) close()    |                                |    (6) accept()
+ *                |                                |    (7) recv()
+ *                |                                |    (8) close()
+ *                |                                |
+ *                =                                =
+ *
+ */
+static int
+run(struct tdata tdata)
+{
+	int fd[2], rc;
+	pid_t child;
+
+	(void) printf("TEST STARTING: %s\n", tdata.td_name);
+
+	if (pipe(fd) != 0) {
+		perror("failed to create pipe");
+		return (1);
+	}
+
+	child = fork();
+	if (child == 0) {
+		exit(client(fd[1], tdata.td_msgs, tdata.td_num_msgs));
+	} else if (child > 0) {
+		rc = server(fd[0], tdata.td_type, tdata.td_msgs,
+		    tdata.td_num_msgs);
+		if (rc != 0)
+			(void) printf("TEST FAILED: %s\n", tdata.td_name);
+		else
+			(void) printf("TEST PASSED: %s\n", tdata.td_name);
+
+		(void) close(fd[0]);
+		(void) close(fd[1]);
+		(void) kill(child, SIGKILL);
+	} else {
+		perror("problem with fork()\n");
+		(void) printf("TEST FAILED: %s\n", tdata.td_name);
+		rc = 1;
+	}
+
+	return (rc);
+}
+
+static int
+server(int fd, char *filt, char **msgs, int num_msgs)
+{
+	int			csock, lsock, ndeferred, status;
+	char			buf[BUFSIZE];
+	kstat_ctl_t		*kc;
+	kstat_named_t		*kn;
+	kstat_t			*ks;
+	struct sockaddr_in	addr, cliaddr;
+	socklen_t		clilen;
+
+	lsock = socket(PF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0);
+
+	if (setsockopt(lsock, SOL_FILTER, FIL_ATTACH, filt, strlen(filt) + 1) <
+	    0) {
+		perror("couldn't set filter");
+		return (1);
+	}
+
+	if (lsock == -1) {
+		perror("socket");
+		return (1);
+	}
+
+	bzero(&addr, sizeof (addr));
+	addr.sin_family = AF_INET;
+	addr.sin_addr.s_addr = htonl(INADDR_ANY);
+	addr.sin_port = htons(PORT);
+	if (bind(lsock, (struct sockaddr *)&addr, sizeof (addr)) < 0) {
+		perror("server failed to bind");
+		return (1);
+	}
+
+	if (listen(lsock, 2) < 0) {
+		perror("listen failed");
+		return (1);
+	}
+
+	dbg("(1) [server] listen()\n");
+
+	char filter_name[1024] = "filter_";
+	(void) strcat(filter_name, filt);
+	kc = kstat_open();
+	ks = kstat_lookup(kc, "sockfs", 0, filter_name);
+	(void) kstat_read(kc, ks, NULL);
+	kn = kstat_data_lookup(ks, "ndeferred");
+	ndeferred = kn->value.ui64;
+
+	if (ndeferred != 0) {
+		(void) fprintf(stderr, "expected 0 deferred conns but got %d\n",
+		    ndeferred);
+		return (1);
+	}
+
+	if (sendstate(fd, SERVER_LISTENING) != 0)
+		return (1);
+
+	dbg("---SERVER_LISTENING----->\n");
+
+	if (waitforstate(fd, CLIENT_CONNECTED) != 0)
+		return (1);
+
+	clilen = sizeof (cliaddr);
+	csock = accept(lsock, (struct sockaddr *)&cliaddr, &clilen);
+	if ((csock == -1) && (errno != EAGAIN)) {
+		perror("problem accepting");
+		return (1);
+	} else if (csock != -1) {
+		(void) fprintf(stderr,
+		    "server accpeted before any data sent\n");
+		return (1);
+	}
+
+	dbg("(3) [server] accept()\n");
+
+	(void) kstat_chain_update(kc);
+	(void) kstat_read(kc, ks, NULL);
+	kn = kstat_data_lookup(ks, "ndeferred");
+	ndeferred = kn->value.ui64;
+	if (ndeferred != 1) {
+		(void) fprintf(stderr, "expected 1 deferred conns but got %d\n",
+		    ndeferred);
+		return (1);
+	}
+
+	if (sendstate(fd, SERVER_NO_ACCEPT) != 0)
+		return (1);
+
+	dbg("---SERVER_NO_ACCEPT--->\n");
+
+	/* The last msg is the final msg. */
+	for (int i = 0; i < num_msgs - 1; i++) {
+		if (waitforstate(fd, CLIENT_SENT_SOME) != 0)
+			return (1);
+
+		csock = accept(lsock, (struct sockaddr *)&cliaddr, &clilen);
+		if ((csock == -1) && (errno != EAGAIN)) {
+			perror("problem accepting");
+			return (1);
+		} else if (csock != -1) {
+			(void) fprintf(stderr,
+			    "server accpeted after some data sent\n");
+			return (1);
+		}
+
+		dbg("(5.%d) [server] accept()\n", i);
+
+		if (sendstate(fd, SERVER_NO_ACCEPT) != 0)
+			return (1);
+
+		dbg("---SERVER_NO_ACCEPT--->\n");
+
+	}
+
+	if (waitforstate(fd, CLIENT_SENT_ALL) != 0)
+		return (1);
+
+	csock = accept(lsock, (struct sockaddr *)&cliaddr, &clilen);
+	if (csock == -1) {
+		perror("problem accepting");
+		return (1);
+	}
+
+	dbg("(6) [server] accept()\n");
+
+	bzero(&buf, BUFSIZE);
+	if (recv(csock, buf, BUFSIZE, 0) == -1) {
+		perror("problemn receiving data");
+		return (1);
+	}
+
+	dbg("(7) [server] recv()\n");
+
+	if (msgscmp(buf, msgs, num_msgs) != 0) {
+		(void) fprintf(stderr, "data modified\n");
+		return (1);
+	}
+
+	(void) close(csock);
+	(void) close(lsock);
+
+	dbg("(8) [server] close()\n");
+
+	(void) wait(&status);
+	return (status);
+}
+
+static int
+client(int fd, char **msgs, int num_msgs)
+{
+	int			sock;
+	size_t			sz;
+	struct sockaddr_in	addr;
+
+	if (waitforstate(fd, SERVER_LISTENING) != 0)
+		return (1);
+
+	bzero(&addr, sizeof (addr));
+	addr.sin_family = AF_INET;
+	addr.sin_addr.s_addr = inet_addr("127.0.0.1");
+	addr.sin_port = htons(PORT);
+
+	sock = socket(AF_INET, SOCK_STREAM, 0);
+	if (sock < 0) {
+		perror("cannot create client socket");
+		return (1);
+	}
+
+	if (connect(sock, (struct sockaddr *)&addr, sizeof (addr)) < 0) {
+		(void) printf("failed to connect: %d\n", errno);
+		perror("cannot connect to server");
+		return (1);
+	}
+
+	dbg("(2) [client] connect()\n");
+
+	if (sendstate(fd, CLIENT_CONNECTED) != 0)
+		return (1);
+
+	dbg("<---CLIENT_CONNECTED-----\n");
+
+	if (waitforstate(fd, SERVER_NO_ACCEPT) != 0)
+		return (1);
+
+	for (int i = 0; i < num_msgs; i++) {
+		size_t len = strlen(msgs[i]);
+		sz = send(sock, msgs[i], len, 0);
+		if (sz != len) {
+			(void) fprintf(stderr,
+			    "client sent %zu bytes, but should have sent %zu\n",
+			    sz, len);
+			return (1);
+		}
+
+		dbg("(4.%d) [client] send()\n", i);
+
+		if (i == num_msgs - 1)
+			break;
+
+		if (sendstate(fd, CLIENT_SENT_SOME) != 0)
+			return (1);
+
+		dbg("<---CLIENT_SENT_SOME-----\n");
+
+		if (waitforstate(fd, SERVER_NO_ACCEPT) != 0)
+			return (1);
+	}
+
+	if (sendstate(fd, CLIENT_SENT_ALL) != 0)
+		return (1);
+
+	dbg("<---CLIENT_SENT_ALL-----\n");
+	(void) close(sock);
+	dbg("(6) [client] close()\n");
+
+	return (0);
+}
+
+/* Send a debug msg to stdout. */
+static void
+dbg(const char *format, ...)
+{
+	va_list	args;
+
+	if (!debug) {
+		return;
+	}
+
+	va_start(args, format);
+	(void) vprintf(format, args);
+	va_end(args);
+	(void) fflush(stdout);
+}
+
+/*
+ * Test for byte-for-byte equality between data and msgs. Return 0 if
+ * equal, 1 otherwise.
+ */
+static int
+msgscmp(char *data, char **msgs, int num_msgs)
+{
+	for (int i = 0, offset = 0; i < num_msgs; i++) {
+		int len = strlen(msgs[i]);
+		if (memcmp(&data[offset], msgs[i], len) != 0)
+			return (1);
+
+		offset += len;
+	}
+
+	return (0);
+}
+
+/* Send a state across the pipe. */
+static int sendstate(int fd, enum State state)
+{
+	size_t sz;
+
+	sz = write(fd, &state, sizeof (int));
+	return ((sz == sizeof (int)) ? 0 : 1);
+}
+
+/* Block on the given state. */
+static int waitforstate(int fd, enum State state)
+{
+	int i;
+	size_t sz;
+
+	sz = read(fd, &i, sizeof (int));
+	if (sz != sizeof (int)) {
+		(void) fprintf(stderr, "couldn't read int\n");
+		return (1);
+	}
+
+	if (i != state) {
+		(void) fprintf(stderr,
+		    "saw state %d but expected %d\n", i, state);
+		return (1);
+	}
+
+	return (0);
+}
