commit efe7d180292726775fb3ae5e6af593490a264c60
from: Omar Polo <op@omarpolo.com>
date: Sat Oct 02 17:20:56 2021 UTC

new I/O handling on top of bufferevents

This is a big change in how gmid handles I/O.  Initially we used a
hand-written loop over poll(2), that then was evolved into something
powered by libevent basic API.  This meant that there were a lot of
small "asynchronous" function that did one step, eventually scheduling
the re-execution, that called each others in a chain.

The new implementation revolves completely around libevent'
bufferevents.  It's more clear, as everything is implemented around the
client_read and client_write functions.

There is still space for improvements, like adding timeouts for one, but
it's solid enough to be committed as is and then further improved.

commit - 403c42204182515d7281d8c11084eef596f8a6ee
commit + efe7d180292726775fb3ae5e6af593490a264c60
blob - 8de518a5d56ffd3ae86e1ba763c8680dee0246ed
blob + c87a0e6cc824e0d70fe33dad887d221032bde85f
--- fcgi.c
+++ fcgi.c
@@ -223,16 +223,18 @@ fcgi_end_param(struct bufferevent *bev, int id)
 	return 0;
 }
 
-static int
-fcgi_abort_request(struct bufferevent *bev, int id)
+void
+fcgi_abort_request(struct client *c)
 {
+	struct fcgi *f;
 	struct fcgi_header h;
 
-	prepare_header(&h, FCGI_ABORT_REQUEST, id, 0, 0);
-	if (bufferevent_write(bev, &h, sizeof(h)) == -1)
-		return -1;
+	f = &fcgi[c->fcgi];
 
-	return 0;
+	prepare_header(&h, FCGI_ABORT_REQUEST, c->id, 0, 0);
+
+	if (bufferevent_write(f->bev, &h, sizeof(h)) == -1)
+		fcgi_close_backend(f);
 }
 
 static inline int
@@ -247,55 +249,6 @@ reclen(struct fcgi_header *h)
 	return h->content_len0 + (h->content_len1 << 8);
 }
 
-static void
-copy_mbuf(int fd, short ev, void *d)
-{
-	struct client	*c = d;
-	struct mbuf	*mbuf;
-	size_t		 len;
-	ssize_t		 r;
-	char		*data;
-
-        for (;;) {
-		mbuf = TAILQ_FIRST(&c->mbufhead);
-		if (mbuf == NULL)
-			break;
-
-		len = mbuf->len - mbuf->off;
-		data = mbuf->data + mbuf->off;
-		switch (r = tls_write(c->ctx, data, len)) {
-		case -1:
-			/*
-			 * Can't close_conn here.  The application
-			 * needs to be informed first, otherwise it
-			 * can interfere with future connections.
-			 * Check also that we're not doing recursion
-			 * (copy_mbuf -> handle_fcgi -> copy_mbuf ...)
-			 */
-			if (c->next != NULL)
-				goto end;
-			fcgi_abort_request(0, c->id);
-			return;
-		case TLS_WANT_POLLIN:
-			event_once(c->fd, EV_READ, &copy_mbuf, c, NULL);
-			return;
-		case TLS_WANT_POLLOUT:
-			event_once(c->fd, EV_WRITE, &copy_mbuf, c, NULL);
-			return;
-		}
-		mbuf->off += r;
-
-		if (mbuf->off == mbuf->len) {
-			TAILQ_REMOVE(&c->mbufhead, mbuf, mbufs);
-			free(mbuf);
-		}
-	}
-
-end:
-	if (c->next != NULL)
-		c->next(0, 0, c);
-}
-
 void
 fcgi_close_backend(struct fcgi *f)
 {
@@ -315,7 +268,6 @@ fcgi_read(struct bufferevent *bev, void *d)
 	struct fcgi_header	 hdr;
 	struct fcgi_end_req_body end;
 	struct client		*c;
-	struct mbuf		*mbuf;
 	size_t			 len;
 
 #if DEBUG_FCGI
@@ -372,8 +324,8 @@ fcgi_read(struct bufferevent *bev, void *d)
 			/* TODO: do something with the status? */
 			fcgi->pending--;
 			c->fcgi = -1;
-			c->next = close_conn;
-			event_once(c->fd, EV_WRITE, &copy_mbuf, c, NULL);
+			c->type = REQUEST_DONE;
+			client_write(c->bev, c);
 			break;
 
 		case FCGI_STDERR:
@@ -382,16 +334,7 @@ fcgi_read(struct bufferevent *bev, void *d)
 			break;
 
 		case FCGI_STDOUT:
-			if ((mbuf = calloc(1, sizeof(*mbuf) + len)) == NULL)
-				fatal("calloc");
-			mbuf->len = len;
-			bufferevent_read(bev, mbuf->data, len);
-
-			if (TAILQ_EMPTY(&c->mbufhead))
-				event_once(c->fd, EV_WRITE, &copy_mbuf,
-				    c, NULL);
-
-			TAILQ_INSERT_TAIL(&c->mbufhead, mbuf, mbufs);
+			bufferevent_write_buffer(c->bev, EVBUFFER_INPUT(bev));
 			break;
 
 		default:
@@ -439,7 +382,7 @@ fcgi_error(struct bufferevent *bev, short err, void *d
 			continue;
 
 		if (c->code != 0)
-			close_conn(0, 0, 0);
+			client_close(c);
 		else
 			start_reply(c, CGI_ERROR, "CGI error");
 	}
@@ -466,8 +409,6 @@ fcgi_req(struct fcgi *f, struct client *c)
 		fatal("getnameinfo failed: %s (%s)",
 		    gai_strerror(e), strerror(errno));
 
-	c->next = NULL;
-
 	fcgi_begin_request(f->bev, c->id);
 	fcgi_send_param(f->bev, c->id, "GATEWAY_INTERFACE", "CGI/1.1");
 	fcgi_send_param(f->bev, c->id, "GEMINI_URL_PATH", c->iri.path);
blob - d0b17bd334af3970950d093bbd0be97d7286b221
blob + 8e3f308c68e34bbb47f11452dbd7a62b4d632588
--- gmid.h
+++ gmid.h
@@ -185,61 +185,34 @@ struct parser {
 	const char	*err;
 };
 
-struct mbuf {
-	size_t			len;
-	size_t			off;
-	TAILQ_ENTRY(mbuf)	mbufs;
-	char			data[];
-};
-TAILQ_HEAD(mbufhead, mbuf);
-
 typedef void (imsg_handlerfn)(struct imsgbuf*, struct imsg*, size_t);
 
-typedef void (*statefn)(int, short, void*);
+enum {
+	REQUEST_UNDECIDED,
+	REQUEST_FILE,
+	REQUEST_DIR,
+	REQUEST_CGI,
+	REQUEST_FCGI,
+	REQUEST_DONE,
+};
 
-/*
- * DFA: handle_handshake is the initial state, close_conn the final.
- * Sometimes we have an enter_* function to handle the state switch.
- *
- * handle_handshake -> handle_open_conn
- * handle_handshake -> close_conn		// on err
- *
- * handle_open_conn -> handle_cgi_reply		// via open_file/dir/...
- * handle_open_conn -> send_fcgi_req		// via apply_fastcgi, IMSG_FCGI_FD
- * handle_open_conn -> handle_dirlist		// ...same
- * handle_open_conn -> send_file		// ...same
- * handle_open_conn -> start_reply		// on error
- *
- * handle_cgi_reply -> handle_cgi	// after logging the CGI reply
- * handle_cgi_reply -> start_reply	// on error
- *
- * handle_cgi -> close_conn
- *
- * send_fcgi_req -> copy_mbuf		// via handle_fcgi
- * handle_fcgi -> close_all		// on error
- * copy_mbuf -> close_conn		// on success/error
- *
- * handle_dirlist -> send_directory_listing
- * handle_dirlist -> close_conn			// on error
- *
- * send_directory_listing -> close_conn
- *
- * send_file -> close_conn
- */
+#define IS_INTERNAL_REQUEST(x)	((x) != REQUEST_CGI && (x) != REQUEST_FCGI)
+
 struct client {
 	int		 id;
 	struct tls	*ctx;
-	char		 req[GEMINI_URL_LEN];
+	char		*req;
 	struct iri	 iri;
 	char		 domain[DOMAIN_NAME_LEN];
 
-	/*
-	 * start_reply uses this to know what function call after the
-	 * reply.  It's also used as sentinel value in fastcgi to know
-	 * if the server has closed the request.
-	 */
-	statefn		 next;
+	struct bufferevent *bev;
 
+	int		 type;
+
+	struct bufferevent *cgibev;
+
+	char		*header;
+
 	int		 code;
 	const char	*meta;
 	int		 fd, pfd;
@@ -251,8 +224,6 @@ struct client {
 	char		 sbuf[1029];
 	ssize_t		 len, off;
 
-	struct mbufhead	 mbufhead;
-
 	struct sockaddr_storage	 addr;
 	struct vhost	*host;	/* host they're talking to */
 	size_t		 loc;	/* location matched */
@@ -356,8 +327,9 @@ X509_STORE	*vhost_require_ca(struct vhost*, const char
 int		 vhost_disable_log(struct vhost*, const char*);
 
 void		 mark_nonblock(int);
+void		 client_write(struct bufferevent *, void *);
 void		 start_reply(struct client*, int, const char*);
-void		 close_conn(int, short, void*);
+void		 client_close(struct client *);
 struct client	*try_client_by_id(int);
 void		 loop(struct tls*, int, int, struct imsgbuf*);
 
@@ -382,6 +354,7 @@ int		 recv_fd(int);
 int		 executor_main(struct imsgbuf*);
 
 /* fcgi.c */
+void		 fcgi_abort_request(struct client *);
 void		 fcgi_close_backend(struct fcgi *);
 void		 fcgi_read(struct bufferevent *, void *);
 void		 fcgi_write(struct bufferevent *, void *);
blob - 605ec6c139731b617f49551fc100762e1fdb24a7
blob + dec3898dcbcebcd8d573edc6d3c111f1b9c1a0cf
--- server.c
+++ server.c
@@ -19,6 +19,7 @@
 #include <sys/stat.h>
 
 #include <assert.h>
+#include <ctype.h>
 #include <errno.h>
 #include <event.h>
 #include <fcntl.h>
@@ -26,6 +27,8 @@
 #include <limits.h>
 #include <string.h>
 
+#define MIN(a, b)	((a) < (b) ? (a) : (b))
+
 int shutting_down;
 
 struct client	 clients[MAX_USERS];
@@ -39,9 +42,6 @@ int connected_clients;
 
 static inline int matches(const char*, const char*);
 
-static inline void yield_read(int, struct client*, statefn);
-static inline void yield_write(int, struct client*, statefn);
-
 static int	 check_path(struct client*, const char*, int*);
 static void	 open_file(struct client*);
 static void	 check_for_cgi(struct client*);
@@ -49,23 +49,33 @@ static void	 handle_handshake(int, short, void*);
 static const char *strip_path(const char*, int);
 static void	 fmt_sbuf(const char*, struct client*, const char*);
 static int	 apply_block_return(struct client*);
+static int	 apply_fastcgi(struct client*);
 static int	 apply_require_ca(struct client*);
-static void	 handle_open_conn(int, short, void*);
-static void	 handle_start_reply(int, short, void*);
 static size_t	 host_nth(struct vhost*);
 static void	 start_cgi(const char*, const char*, struct client*);
 static void	 open_dir(struct client*);
 static void	 redirect_canonical_dir(struct client*);
-static void	 enter_handle_dirlist(int, short, void*);
-static void	 handle_dirlist(int, short, void*);
-static int 	 read_next_dir_entry(struct client*);
-static void	 send_directory_listing(int, short, void*);
-static void	 handle_cgi_reply(int, short, void*);
-static void	 handle_copy(int, short, void*);
+
+static void	 client_tls_readcb(int, short, void *);
+static void	 client_tls_writecb(int, short, void *);
+
+static void	 client_read(struct bufferevent *, void *);
+void		 client_write(struct bufferevent *, void *);
+static void	 client_error(struct bufferevent *, short, void *);
+
+static void	 client_close_ev(int, short, void *);
+
+static void	 cgi_read(struct bufferevent *, void *);
+static void	 cgi_write(struct bufferevent *, void *);
+static void	 cgi_error(struct bufferevent *, short, void *);
+
 static void	 do_accept(int, short, void*);
+static struct client *client_by_id(int);
+
 static void	 handle_imsg_cgi_res(struct imsgbuf*, struct imsg*, size_t);
 static void	 handle_imsg_fcgi_fd(struct imsgbuf*, struct imsg*, size_t);
 static void	 handle_imsg_quit(struct imsgbuf*, struct imsg*, size_t);
+static void	 handle_dispatch_imsg(int, short, void *);
 static void	 handle_siginfo(int, short, void*);
 
 static imsg_handlerfn *handlers[] = {
@@ -80,18 +90,6 @@ matches(const char *pattern, const char *path)
 	if (*path == '/')
 		path++;
 	return !fnmatch(pattern, path, 0);
-}
-
-static inline void
-yield_read(int fd, struct client *c, statefn fn)
-{
-	event_once(fd, EV_READ, fn, c, NULL);
-}
-
-static inline void
-yield_write(int fd, struct client *c, statefn fn)
-{
-	event_once(fd, EV_WRITE, fn, c, NULL);
 }
 
 const char *
@@ -362,7 +360,7 @@ open_file(struct client *c)
 		/* fallthrough */
 
 	case FILE_EXISTS:
-		c->next = handle_copy;
+		c->type = REQUEST_FILE;
 		start_reply(c, SUCCESS, mime(c->host, c->iri.path));
 		return;
 
@@ -458,10 +456,10 @@ handle_handshake(int fd, short ev, void *d)
 	case -1: /* already handshaked */
 		break;
 	case TLS_WANT_POLLIN:
-		yield_read(fd, c, &handle_handshake);
+		event_once(c->fd, EV_READ, handle_handshake, c, NULL);
 		return;
 	case TLS_WANT_POLLOUT:
-		yield_write(fd, c, &handle_handshake);
+		event_once(c->fd, EV_WRITE, handle_handshake, c, NULL);
 		return;
 	default:
 		/* unreachable */
@@ -495,16 +493,24 @@ found:
 
 	if (h != NULL) {
 		c->host = h;
-		handle_open_conn(fd, ev, c);
+
+		c->bev = bufferevent_new(fd, client_read, client_write,
+		    client_error, c);
+		if (c->bev == NULL)
+			fatal("%s: failed to allocate client buffer: %s",
+			    __func__, strerror(errno));
+
+		event_set(&c->bev->ev_read, c->fd, EV_READ,
+		    client_tls_readcb, c->bev);
+		event_set(&c->bev->ev_write, c->fd, EV_WRITE,
+		    client_tls_writecb, c->bev);
+
+		bufferevent_enable(c->bev, EV_READ);
+
 		return;
 	}
 
 err:
-	if (servname != NULL)
-		strlcpy(c->req, servname, sizeof(c->req));
-	else
-		strlcpy(c->req, "null", sizeof(c->req));
-
 	start_reply(c, BAD_REQUEST, "Wrong/malformed host or missing SNI");
 }
 
@@ -647,110 +653,6 @@ apply_require_ca(struct client *c)
 	return 0;
 }
 
-static void
-handle_open_conn(int fd, short ev, void *d)
-{
-	struct client *c = d;
-	const char *parse_err = "invalid request";
-	char decoded[DOMAIN_NAME_LEN];
-
-	switch (tls_read(c->ctx, c->req, sizeof(c->req)-1)) {
-	case -1:
-		log_err(c, "tls_read: %s", tls_error(c->ctx));
-		close_conn(fd, ev, c);
-		return;
-
-	case TLS_WANT_POLLIN:
-		yield_read(fd, c, &handle_open_conn);
-		return;
-
-	case TLS_WANT_POLLOUT:
-		yield_write(fd, c, &handle_open_conn);
-		return;
-	}
-
-	if (!trim_req_iri(c->req, &parse_err) ||
-	    !parse_iri(c->req, &c->iri, &parse_err) ||
-	    !puny_decode(c->iri.host, decoded, sizeof(decoded), &parse_err)) {
-		log_info(c, "iri parse error: %s", parse_err);
-		start_reply(c, BAD_REQUEST, "invalid request");
-		return;
-	}
-
-	if (c->iri.port_no != conf.port ||
-	    strcmp(c->iri.schema, "gemini") ||
-	    strcmp(decoded, c->domain)) {
-		start_reply(c, PROXY_REFUSED, "won't proxy request");
-		return;
-	}
-
-	if (apply_require_ca(c))
-		return;
-
-	if (apply_block_return(c))
-		return;
-
-	if (apply_fastcgi(c))
-		return;
-
-	if (c->host->entrypoint != NULL) {
-		c->loc = 0;
-		start_cgi(c->host->entrypoint, c->iri.path, c);
-		return;
-	}
-
-	open_file(c);
-}
-
-void
-start_reply(struct client *c, int code, const char *meta)
-{
-	c->code = code;
-	c->meta = meta;
-	handle_start_reply(c->fd, 0, c);
-}
-
-static void
-handle_start_reply(int fd, short ev, void *d)
-{
-	struct client *c = d;
-	char buf[1030];		/* status + ' ' + max reply len + \r\n\0 */
-	const char *lang;
-	size_t len;
-
-	lang = vhost_lang(c->host, c->iri.path);
-
-	snprintf(buf, sizeof(buf), "%d ", c->code);
-	strlcat(buf, c->meta, sizeof(buf));
-	if (!strcmp(c->meta, "text/gemini") && lang != NULL) {
-		strlcat(buf, "; lang=", sizeof(buf));
-		strlcat(buf, lang, sizeof(buf));
-	}
-
-	len = strlcat(buf, "\r\n", sizeof(buf));
-	assert(len < sizeof(buf));
-
-	switch (tls_write(c->ctx, buf, len)) {
-	case -1:
-		close_conn(fd, ev, c);
-		return;
-	case TLS_WANT_POLLIN:
-		yield_read(fd, c, &handle_start_reply);
-		return;
-	case TLS_WANT_POLLOUT:
-		yield_write(fd, c, &handle_start_reply);
-		return;
-	}
-
-	if (!vhost_disable_log(c->host, c->iri.path))
-		log_request(c, buf, sizeof(buf));
-
-	if (c->code != SUCCESS)
-		close_conn(fd, ev, c);
-	else
-		c->next(fd, ev, c);
-}
-
 static size_t
 host_nth(struct vhost *h)
 {
@@ -774,6 +676,8 @@ start_cgi(const char *spath, const char *relpath, stru
 	struct cgireq req;
 	int e;
 
+	c->type = REQUEST_CGI;
+
 	e = getnameinfo((struct sockaddr*)&c->addr, sizeof(c->addr),
 	    addr, sizeof(addr),
 	    NULL, 0,
@@ -865,7 +769,7 @@ open_dir(struct client *c)
 		/* fallthrough */
 
 	case FILE_EXISTS:
-		c->next = handle_copy;
+		c->type = REQUEST_FILE;
 		start_reply(c, SUCCESS, mime(c->host, c->iri.path));
 		break;
 
@@ -881,7 +785,7 @@ open_dir(struct client *c)
 			break;
 		}
 
-		c->next = enter_handle_dirlist;
+		c->type = REQUEST_DIR;
 
 		c->dirlen = scandir_fd(dirfd, &c->dir,
 		    root ? select_non_dotdot : select_non_dot,
@@ -896,6 +800,8 @@ open_dir(struct client *c)
 		c->off = 0;
 
                 start_reply(c, SUCCESS, "text/gemini");
+		evbuffer_add_printf(EVBUFFER_OUTPUT(c->bev),
+		    "# Index of %s\n\n", c->iri.path);
 		return;
 
 	default:
@@ -924,216 +830,327 @@ redirect_canonical_dir(struct client *c)
 }
 
 static void
-enter_handle_dirlist(int fd, short ev, void *d)
+client_tls_readcb(int fd, short event, void *d)
 {
-	struct client *c = d;
-	char b[PATH_MAX];
-	size_t l;
+	struct bufferevent	*bufev = d;
+	struct client		*client = bufev->cbarg;
+	ssize_t			 ret;
+	size_t			 len;
+	int			 what = EVBUFFER_READ;
+	int			 howmuch = IBUF_READ_SIZE;
+	char			 buf[IBUF_READ_SIZE];
 
-	strlcpy(b, c->iri.path, sizeof(b));
-	l = snprintf(c->sbuf, sizeof(c->sbuf),
-	    "# Index of %s\n\n", b);
-	if (l >= sizeof(c->sbuf)) {
-		/*
-		 * This is impossible, given that we have enough space
-		 * in c->sbuf to hold the ancilliary string plus the
-		 * full path; but it wouldn't read nice without some
-		 * error checking, and I'd like to avoid a strlen.
-		 */
-		close_conn(fd, ev, c);
-		return;
+	if (event == EV_TIMEOUT) {
+		what |= EVBUFFER_TIMEOUT;
+		goto err;
 	}
 
-	c->len = l;
-	handle_dirlist(fd, ev, c);
-}
+	if (bufev->wm_read.high != 0)
+		howmuch = MIN(sizeof(buf), bufev->wm_read.high);
 
-static void
-handle_dirlist(int fd, short ev, void *d)
-{
-	struct client *c = d;
-	ssize_t r;
+	switch (ret = tls_read(client->ctx, buf, howmuch)) {
+	case TLS_WANT_POLLIN:
+	case TLS_WANT_POLLOUT:
+		goto retry;
+	case -1:
+		what |= EVBUFFER_ERROR;
+		goto err;
+	}
+	len = ret;
 
-	while (c->len > 0) {
-		switch (r = tls_write(c->ctx, c->sbuf + c->off, c->len)) {
-		case -1:
-			close_conn(fd, ev, c);
-			return;
-		case TLS_WANT_POLLOUT:
-			yield_read(fd, c, &handle_dirlist);
-			return;
-		case TLS_WANT_POLLIN:
-			yield_write(fd, c, &handle_dirlist);
-			return;
-		default:
-			c->off += r;
-			c->len -= r;
-		}
+	if (len == 0) {
+		what |= EVBUFFER_EOF;
+		goto err;
 	}
 
-	send_directory_listing(fd, ev, c);
-}
+	if (evbuffer_add(bufev->input, buf, len) == -1) {
+		what |= EVBUFFER_ERROR;
+		goto err;
+	}
 
-static int
-read_next_dir_entry(struct client *c)
-{
-	if (c->diroff == c->dirlen)
-		return 0;
+	event_add(&bufev->ev_read, NULL);
+	if (bufev->wm_read.low != 0 && len < bufev->wm_read.low)
+		return;
+	if (bufev->wm_read.high != 0 && len > bufev->wm_read.high) {
+		/*
+		 * here we could implement a read pressure policy.
+		 */
+	}
 
-	/* XXX: url escape */
-	snprintf(c->sbuf, sizeof(c->sbuf), "=> %s\n",
-	    c->dir[c->diroff]->d_name);
+	if (bufev->readcb != NULL)
+		(*bufev->readcb)(bufev, bufev->cbarg);
 
-	free(c->dir[c->diroff]);
-	c->diroff++;
+	return;
 
-	c->len = strlen(c->sbuf);
-	c->off = 0;
-	return 1;
+retry:
+	event_add(&bufev->ev_read, NULL);
+	return;
+
+err:
+	(*bufev->errorcb)(bufev, what, bufev->cbarg);
 }
 
 static void
-send_directory_listing(int fd, short ev, void *d)
+client_tls_writecb(int fd, short event, void *d)
 {
-	struct client *c = d;
-	ssize_t r;
+	struct bufferevent	*bufev = d;
+	struct client		*client = bufev->cbarg;
+	ssize_t			 ret;
+	size_t			 len;
+	short			 what = EVBUFFER_WRITE;
 
-	while (1) {
-		if (c->len == 0) {
-			if (!read_next_dir_entry(c))
-				goto end;
-		}
+	if (event == EV_TIMEOUT) {
+		what |= EVBUFFER_TIMEOUT;
+		goto err;
+	}
 
-		while (c->len > 0) {
-			switch (r = tls_write(c->ctx, c->sbuf + c->off, c->len)) {
-			case -1:
-				goto end;
-
-			case TLS_WANT_POLLOUT:
-				yield_read(fd, c, &send_directory_listing);
-				return;
-
-			case TLS_WANT_POLLIN:
-				yield_write(fd, c, &send_directory_listing);
-				return;
-
-			default:
-				c->off += r;
-				c->len -= r;
-				break;
-			}
+	if (EVBUFFER_LENGTH(bufev->output) != 0) {
+		ret = tls_write(client->ctx,
+		    EVBUFFER_DATA(bufev->output),
+		    EVBUFFER_LENGTH(bufev->output));
+		switch (ret) {
+		case TLS_WANT_POLLIN:
+		case TLS_WANT_POLLOUT:
+			goto retry;
+		case -1:
+			what |= EVBUFFER_ERROR;
+			goto err;
 		}
+		len = ret;
+		evbuffer_drain(bufev->output, len);
 	}
 
-end:
-	close_conn(fd, ev, d);
+	if (EVBUFFER_LENGTH(bufev->output) != 0)
+		event_add(&bufev->ev_write, NULL);
+
+	if (bufev->writecb != NULL &&
+	    EVBUFFER_LENGTH(bufev->output) <= bufev->wm_write.low)
+		(*bufev->writecb)(bufev, bufev->cbarg);
+	return;
+
+retry:
+	event_add(&bufev->ev_write, NULL);
+	return;
+err:
+	log_err(client, "tls error: %s", tls_error(client->ctx));
+	(*bufev->errorcb)(bufev, what, bufev->cbarg);
 }
 
-/* accumulate the meta line from the cgi script. */
 static void
-handle_cgi_reply(int fd, short ev, void *d)
+client_read(struct bufferevent *bev, void *d)
 {
-	struct client *c = d;
-	void	*buf, *e;
-	size_t	 len;
-	ssize_t	 r;
+	struct client	*c = d;
+	struct evbuffer	*src = EVBUFFER_INPUT(bev);
+	const char	*parse_err = "invalid request";
+	char		 decoded[DOMAIN_NAME_LEN];
+	size_t		 len;
 
+	bufferevent_disable(bev, EVBUFFER_READ);
 
-	buf = c->sbuf + c->len;
-	len = sizeof(c->sbuf) - c->len;
+	/* max url len + \r\n */
+	if (EVBUFFER_LENGTH(src) > 1024 + 2) {
+		log_err(c, "too much data received");
+		start_reply(c, BAD_REQUEST, "bad request");
+		return;
+	}
 
-	r = read(c->pfd, buf, len);
-	if (r == 0 || r == -1) {
-		start_reply(c, CGI_ERROR, "CGI error");
+	c->req = evbuffer_readln(src, &len, EVBUFFER_EOL_CRLF_STRICT);
+	if (c->req == NULL) {
+		/* not enough data yet. */
+		bufferevent_enable(bev, EVBUFFER_READ);
 		return;
 	}
 
-	c->len += r;
+	if (!parse_iri(c->req, &c->iri, &parse_err) ||
+	    !puny_decode(c->iri.host, decoded, sizeof(decoded), &parse_err)) {
+		log_err(c, "IRI parse error: %s", parse_err);
+                start_reply(c, BAD_REQUEST, "bad request");
+		return;
+	}
 
-	/* TODO: error if the CGI script don't reply correctly */
-	e = strchr(c->sbuf, '\n');
-	if (e != NULL || c->len == sizeof(c->sbuf)) {
-		log_request(c, c->sbuf, c->len);
+	if (c->iri.port_no != conf.port ||
+	    strcmp(c->iri.schema, "gemini") ||
+	    strcmp(decoded, c->domain)) {
+		start_reply(c, PROXY_REFUSED, "won't proxy request");
+		return;
+	}
 
-		c->off = 0;
-		handle_copy(fd, ev, c);
+	if (apply_require_ca(c) ||
+	    apply_block_return(c)||
+	    apply_fastcgi(c))
 		return;
+
+	if (c->host->entrypoint != NULL) {
+		c->loc = 0;
+		start_cgi(c->host->entrypoint, c->iri.path, c);
+		return;
 	}
 
-	yield_read(fd, c, &handle_cgi_reply);
+	open_file(c);
 }
 
-static void
-handle_copy(int fd, short ev, void *d)
+void
+client_write(struct bufferevent *bev, void *d)
 {
-	struct client *c = d;
-	ssize_t r;
+	struct client	*c = d;
+	struct evbuffer	*out = EVBUFFER_OUTPUT(bev);
+	char		 buf[BUFSIZ];
+	ssize_t		 r;
 
-	while (1) {
-		while (c->len > 0) {
-			switch (r = tls_write(c->ctx, c->sbuf + c->off, c->len)) {
-			case -1:
-				goto end;
+	switch (c->type) {
+	case REQUEST_UNDECIDED:
+		/*
+		 * Ignore spurious calls when we still don't have idea
+		 * what to do with the request.
+		 */
+		break;
 
-			case TLS_WANT_POLLOUT:
-				yield_write(c->fd, c, &handle_copy);
-				return;
+	case REQUEST_FILE:
+		if ((r = read(c->pfd, buf, sizeof(buf))) == -1) {
+			log_warn(c, "read: %s", strerror(errno));
+			client_error(bev, EVBUFFER_ERROR, c);
+			return;
+		} else if (r == 0) {
+			client_close(c);
+			return;
+		} else if (r != sizeof(buf))
+			c->type = REQUEST_DONE;
+		bufferevent_write(bev, buf, r);
+		break;
 
-			case TLS_WANT_POLLIN:
-				yield_read(c->fd, c, &handle_copy);
-				return;
-
-			default:
-                                c->off += r;
-				c->len -= r;
-				break;
-			}
+	case REQUEST_DIR:
+		/* TODO: handle big big directories better */
+		for (c->diroff = 0; c->diroff < c->dirlen; ++c->diroff) {
+			evbuffer_add_printf(out, "=> %s\n",
+			    c->dir[c->diroff]->d_name);
+			free(c->dir[c->diroff]);
 		}
+		free(c->dir);
+		c->dir = NULL;
 
-		switch (r = read(c->pfd, c->sbuf, sizeof(c->sbuf))) {
-		case 0:
-			goto end;
-		case -1:
-			if (errno == EAGAIN || errno == EWOULDBLOCK) {
-				yield_read(c->pfd, c, &handle_copy);
-				return;
-			}
-			goto end;
-		default:
-			c->len = r;
-			c->off = 0;
-		}
+		c->type = REQUEST_DONE;
+
+		event_add(&c->bev->ev_write, NULL);
+		break;
+
+	case REQUEST_CGI:
+	case REQUEST_FCGI:
+		/*
+		 *  Here we depend on on the cgi script or fastcgi
+		 *  connection to provide data.
+		 */
+		break;
+
+	case REQUEST_DONE:
+		if (EVBUFFER_LENGTH(out) == 0)
+			client_close(c);
+		break;
 	}
+}
 
-end:
-	close_conn(c->fd, ev, d);
+static void
+client_error(struct bufferevent *bev, short error, void *d)
+{
+	struct client	*c = d;
+
+	if (c->type == REQUEST_FCGI)
+		fcgi_abort_request(c);
+
+	c->type = REQUEST_DONE;
+
+	if (error & EVBUFFER_TIMEOUT) {
+		log_warn(c, "timeout reached, "
+		    "forcefully closing the connection");
+		if (c->code == 0)
+			start_reply(c, BAD_REQUEST, "timeout");
+		else
+			client_close(c);
+		return;
+	}
+
+	if (error & EVBUFFER_EOF) {
+		client_close(c);
+		return;
+	}
+
+	log_err(c, "unknown bufferevent error: %s", strerror(errno));
+	client_close(c);
 }
 
 void
-close_conn(int fd, short ev, void *d)
+start_reply(struct client *c, int code, const char *meta)
 {
+	struct evbuffer	*evb = EVBUFFER_OUTPUT(c->bev);
+	const char	*lang;
+	int		 r, rr;
+
+	bufferevent_enable(c->bev, EVBUFFER_WRITE);
+
+	c->code = code;
+	c->meta = meta;
+
+	r = evbuffer_add_printf(evb, "%d %s", code, meta);
+	if (r == -1)
+		goto err;
+
+	/* 2 digit status + space + 1024 max reply */
+	if (r > 1027)
+		goto overflow;
+
+	if (c->type != REQUEST_CGI &&
+	    c->type != REQUEST_FCGI &&
+	    !strcmp(meta, "text/gemini") &&
+	    (lang = vhost_lang(c->host, c->iri.path)) != NULL) {
+                rr = evbuffer_add_printf(evb, ";lang=%s", lang);
+		if (rr == -1)
+			goto err;
+		if (r + rr > 1027)
+			goto overflow;
+	}
+
+	bufferevent_write(c->bev, "\r\n", 2);
+
+	if (!vhost_disable_log(c->host, c->iri.path))
+		log_request(c, EVBUFFER_DATA(evb), EVBUFFER_LENGTH(evb));
+
+	if (code != 20 && IS_INTERNAL_REQUEST(c->type))
+		c->type = REQUEST_DONE;
+
+	return;
+
+err:
+	log_err(c, "evbuffer_add_printf error: no memory");
+	evbuffer_drain(evb, EVBUFFER_LENGTH(evb));
+	client_close(c);
+	return;
+
+overflow:
+	log_warn(c, "reply header overflow");
+	evbuffer_drain(evb, EVBUFFER_LENGTH(evb));
+	start_reply(c, TEMP_FAILURE, "internal error");
+}
+
+static void
+client_close_ev(int fd, short event, void *d)
+{
 	struct client	*c = d;
-	struct mbuf	*mbuf;
 
 	switch (tls_close(c->ctx)) {
 	case TLS_WANT_POLLIN:
-		yield_read(c->fd, c, &close_conn);
-		return;
+		event_once(c->fd, EV_READ, client_close_ev, c, NULL);
+		break;
 	case TLS_WANT_POLLOUT:
-		yield_read(c->fd, c, &close_conn);
-		return;
+		event_once(c->fd, EV_WRITE, client_close_ev, c, NULL);
+		break;
 	}
 
 	connected_clients--;
 
-	while ((mbuf = TAILQ_FIRST(&c->mbufhead)) != NULL) {
-		TAILQ_REMOVE(&c->mbufhead, mbuf, mbufs);
-		free(mbuf);
-	}
-
 	tls_free(c->ctx);
 	c->ctx = NULL;
 
+	free(c->header);
+
 	if (c->pfd != -1)
 		close(c->pfd);
 
@@ -1144,7 +1161,123 @@ close_conn(int fd, short ev, void *d)
 	c->fd = -1;
 }
 
+void
+client_close(struct client *c)
+{
+	/*
+	 * We may end up calling client_close in various situations
+	 * and for the most unexpected reasons.  Therefore, we need to
+	 * ensure that everything is properly released once we reach
+	 * this point.
+	 */
+
+	if (c->type == REQUEST_FCGI)
+		fcgi_abort_request(c);
+
+	if (c->cgibev != NULL) {
+		bufferevent_disable(c->cgibev, EVBUFFER_READ|EVBUFFER_WRITE);
+		bufferevent_free(c->cgibev);
+		c->cgibev = NULL;
+		close(c->pfd);
+		c->pfd = -1;
+	}
+
+	bufferevent_disable(c->bev, EVBUFFER_READ|EVBUFFER_WRITE);
+	bufferevent_free(c->bev);
+	c->bev = NULL;
+
+	client_close_ev(c->fd, 0, c);
+}
+
 static void
+cgi_read(struct bufferevent *bev, void *d)
+{
+	struct client	*client = d;
+	struct evbuffer	*src = EVBUFFER_INPUT(bev);
+	char		*header;
+	size_t		 len;
+	int		 code;
+
+	/* intercept the header */
+	if (client->code == 0) {
+		header = evbuffer_readln(src, &len, EVBUFFER_EOL_CRLF_STRICT);
+		if (header == NULL) {
+			/* max reply + \r\n */
+			if (EVBUFFER_LENGTH(src) > 1026) {
+				log_warn(client, "CGI script is trying to "
+				    "send a header too long.");
+				cgi_error(bev, EVBUFFER_READ, client);
+			}
+
+			/* wait a bit */
+			return;
+		}
+
+		if (len < 3 || len > 1029 ||
+		    !isdigit(header[0]) ||
+		    !isdigit(header[1]) ||
+		    !isspace(header[2])) {
+			free(header);
+			log_warn(client, "CGI script is trying to send a "
+			    "malformed header");
+			cgi_error(bev, EVBUFFER_READ, client);
+			return;
+		}
+
+		client->header = header;
+		code = (header[0] - '0') * 10 + (header[1] - '0');
+
+		if (code < 10 || code >= 70) {
+			log_warn(client, "CGI script is trying to send an "
+			    "invalid reply code (%d)", code);
+			cgi_error(bev, EVBUFFER_READ, client);
+			return;
+		}
+
+		start_reply(client, code, header + 3);
+
+		if (client->code < 20 || client->code > 29) {
+			cgi_error(client->cgibev, EVBUFFER_EOF, client);
+			return;
+		}
+	}
+
+	bufferevent_write_buffer(client->bev, src);
+}
+
+static void
+cgi_write(struct bufferevent *bev, void *d)
+{
+	/*
+	 * Never called.  We don't send data to a CGI script.
+	 */
+	abort();
+}
+
+static void
+cgi_error(struct bufferevent *bev, short error, void *d)
+{
+	struct client	*client = d;
+
+	if (error & EVBUFFER_ERROR)
+		log_err(client, "%s: evbuffer error (%x): %s",
+		    __func__, error, strerror(errno));
+
+	bufferevent_disable(bev, EVBUFFER_READ|EVBUFFER_WRITE);
+	bufferevent_free(bev);
+	client->cgibev = NULL;
+
+	close(client->pfd);
+	client->pfd = -1;
+
+	client->type = REQUEST_DONE;
+	if (client->code != 0)
+		client_write(client->bev, client);
+	else
+		start_reply(client, CGI_ERROR, "CGI error");
+}
+
+static void
 do_accept(int sock, short et, void *d)
 {
 	struct client *c;
@@ -1179,9 +1312,9 @@ do_accept(int sock, short et, void *d)
 			c->addr = addr;
 			c->fcgi = -1;
 
-			TAILQ_INIT(&c->mbufhead);
+			event_once(c->fd, EV_READ|EV_WRITE, handle_handshake,
+			    c, NULL);
 
-			yield_read(fd, c, &handle_handshake);
 			connected_clients++;
 			return;
 		}
@@ -1213,10 +1346,17 @@ handle_imsg_cgi_res(struct imsgbuf *ibuf, struct imsg 
 
 	c = client_by_id(imsg->hdr.peerid);
 
-	if ((c->pfd = imsg->fd) == -1)
+	if ((c->pfd = imsg->fd) == -1) {
 		start_reply(c, TEMP_FAILURE, "internal server error");
-	else
-		yield_read(c->pfd, c, &handle_cgi_reply);
+		return;
+	}
+
+	c->type = REQUEST_CGI;
+
+	c->cgibev = bufferevent_new(c->pfd, cgi_read, cgi_write,
+	    cgi_error, c);
+
+	bufferevent_enable(c->cgibev, EV_READ);
 }
 
 static void