Commit Diff

Commit:
baef953da253314657be9adea8f371bfbf4ba09e
From:
Russ Cox <rsc@swtch.com>
Date:
Message:
libthread: add pthreadperthread mode and use under ASAN ASAN can't deal with the coroutine stacks. In theory we can call into ASAN runtime to let it know about them, but ASAN still has problems with fork or exit happening from a non-system stack. Bypass all possible problems by just having a full OS thread for each libthread thread. The threads are still cooperatively scheduled within a proc (in thos mode, a group of OS threads). Setting the environment variable LIBTHREAD=pthreadperthread will enable the pthreadperthread mode, as will building with CC9FLAGS='-fsanitize=address' in $PLAN9/config. This solution is much more general than ASAN - for example if you are trying to find all the thread stacks in a reproducible crash you can use pthreadperthread mode with any debugger that knows only about OS threads.
Actions:
Patch | Tree
commit - 06687f70ba7a5836c2e872648a85a724a5a1d486
commit + baef953da253314657be9adea8f371bfbf4ba09e
blob - 53af86e6b832b552f64d914109d90b188ab26685
blob + 9efb7a62c92b5714126934d8162c894cf5ce7136
--- src/libthread/channel.c
+++ src/libthread/channel.c
@@ -141,7 +141,7 @@ altdequeue(Alt *a)
 			delarray(ar, i);
 			return;
 		}
-	fprint(2, "cannot find self in altdq\n");
+	fprint(2, "cannot find self in altdequeue\n");
 	abort();
 }
 
blob - 46bb396a28b04b1b5aef43f5a8450c32ff2db5d2
blob + 5e022f0b145c452fe684d23bc1c73f012bcb096b
--- src/libthread/pthread.c
+++ src/libthread/pthread.c
@@ -99,6 +99,23 @@ startprocfn(void *v)
 	pthread_exit(0);
 }
 
+static void
+startpthreadfn(void *v)
+{
+	void **a;
+	Proc *p;
+	_Thread *t;
+
+	a = (void**)v;
+	p = a[0];
+	t = a[1];
+	free(a);
+	t->osprocid = pthread_self();
+	pthread_detach(t->osprocid);
+	_threadpthreadmain(p, t);
+	pthread_exit(0);
+}
+
 void
 _procstart(Proc *p, void (*fn)(Proc*))
 {
@@ -116,6 +133,22 @@ _procstart(Proc *p, void (*fn)(Proc*))
 	}
 }
 
+void
+_threadpthreadstart(Proc *p, _Thread *t)
+{
+	void **a;
+
+	a = malloc(3*sizeof a[0]);
+	if(a == nil)
+		sysfatal("_pthreadstart malloc: %r");
+	a[0] = p;
+	a[1] = t;
+	if(pthread_create(&t->osprocid, nil, (void*(*)(void*))startpthreadfn, (void*)a) < 0){
+		fprint(2, "pthread_create: %r\n");
+		abort();
+	}
+}
+
 static pthread_key_t prockey;
 
 Proc*
blob - f657b5b24d89c3dbd1fd347250457340e86320bf
blob + f579b5675b47fcfb2596039ace5318e47b8c8129
--- src/libthread/thread.c
+++ src/libthread/thread.c
@@ -7,6 +7,7 @@ static	uint		threadnsysproc;
 static	Lock		threadnproclock;
 static	Ref		threadidref;
 static	Proc		*threadmainproc;
+static	int		pthreadperthread;
 
 static	void		addproc(Proc*);
 static	void		delproc(Proc*);
@@ -176,12 +177,16 @@ _threadcreate(Proc *p, void (*fn)(void*), void *arg, u
 	if(stack < (256<<10))
 		stack = 256<<10;
 
-	if(p->nthread == 0)
+	if(p->nthread == 0 || pthreadperthread)
 		stack = 0; // not using it
 	t = threadalloc(fn, arg, stack);
 	t->proc = p;
-	addthreadinproc(p, t);
+	if(p->nthread == 0)
+		p->thread0 = t;
+	else if(pthreadperthread)
+		_threadpthreadstart(p, t);
 	p->nthread++;
+	addthreadinproc(p, t);
 	_threadready(t);
 	return t;
 }
@@ -209,6 +214,29 @@ proccreate(void (*fn)(void*), void *arg, uint stack)
 	return id;
 }
 
+// For pthreadperthread mode, procswitch flips
+// between the threads.
+static void
+procswitch(Proc *p, _Thread *from, _Thread *to)
+{
+// fprint(2, "procswitch %p %d %d\n", p, from?from->id:-1, to?to->id:-1);
+	lock(&p->schedlock);
+	from->schedrend.l = &p->schedlock;
+	if(to) {
+		p->schedthread = to;
+		to->schedrend.l = &p->schedlock;
+		_procwakeup(&to->schedrend);
+	}
+	if(p->schedthread != from) {
+		if(from->exiting) {
+			unlock(&p->schedlock);
+			_threadpexit();
+		}
+		_procsleep(&from->schedrend);
+	}
+	unlock(&p->schedlock);
+}
+
 void
 _threadswitch(void)
 {
@@ -216,9 +244,13 @@ _threadswitch(void)
 
 	needstack(0);
 	p = proc();
+
 /*print("threadswtch %p\n", p); */
-	if(p->thread->stk == nil)
+
+	if(p->thread == p->thread0)
 		procscheduler(p);
+	else if(pthreadperthread)
+		procswitch(p, p->thread, p->thread0);
 	else
 		contextswitch(&p->thread->context, &p->schedcontext);
 }
@@ -346,6 +378,15 @@ procmain(Proc *p)
 		threadexits(nil);
 }
 
+void
+_threadpthreadmain(Proc *p, _Thread *t)
+{
+	_threadsetproc(p);
+	procswitch(p, t, nil);
+	t->startfn(t->startarg);
+	threadexits(nil);
+}
+
 static void
 procscheduler(Proc *p)
 {
@@ -401,9 +442,12 @@ Top:
 		p->nswitch++;
 		_threaddebug("run %d (%s)", t->id, t->name);
 //print("run %p %p %p %p\n", t, *(uintptr*)(t->context.uc.mc.sp), t->context.uc.mc.di, t->context.uc.mc.si);
-		if(t->stk == nil)
+		if(t == p->thread0)
 			return;
-		contextswitch(&p->schedcontext, &t->context);
+		if(pthreadperthread)
+			procswitch(p, p->thread0, t);
+		else
+			contextswitch(&p->schedcontext, &t->context);
 /*print("back in scheduler\n"); */
 		goto Top;
 	}
@@ -757,10 +801,24 @@ int
 main(int argc, char **argv)
 {
 	Proc *p;
+	char *opts;
 
 	argv0 = argv[0];
 
-	if(getenv("NOLIBTHREADDAEMONIZE") == nil)
+	opts = getenv("LIBTHREAD");
+	if(opts == nil)
+		opts = "";
+
+	pthreadperthread = (strstr(opts, "pthreadperthread") != nil);
+#ifdef PLAN9PORT_ASAN
+	// ASAN can't deal with the coroutine stack switches.
+	// In theory it has support for informing it about stack switches,
+	// but even with those calls added it can't deal with things
+	// like fork or exit from a coroutine stack.
+	// Easier to just run in pthread-per-thread mode.
+	pthreadperthread = 1;
+#endif
+	if(strstr(opts, "nodaemon") || getenv("NOLIBTHREADDAEMONIZE") == nil)
 		_threadsetupdaemonize();
 
 	threadargc = argc;
blob - cceb1b8e7cc0628c4aeb3f23f36fb7e582c97e33
blob + c737384363b6a4cb708c34763c588286e5f47bf0
--- src/libthread/threadimpl.h
+++ src/libthread/threadimpl.h
@@ -102,6 +102,17 @@ struct Execjob
 	Channel *c;
 };
 
+struct _Procrendez
+{
+	Lock		*l;
+	int		asleep;
+#ifdef PLAN9PORT_USING_PTHREADS
+	pthread_cond_t	cond;
+#else
+	int		pid;
+#endif
+};
+
 struct _Thread
 {
 	_Thread	*next;
@@ -112,6 +123,11 @@ struct _Thread
 	void	(*startfn)(void*);
 	void	*startarg;
 	uint	id;
+#ifdef PLAN9PORT_USING_PTHREADS
+	pthread_t	osprocid;
+#else
+	int		osprocid;
+#endif
 	uchar	*stk;
 	uint	stksize;
 	int		exiting;
@@ -120,19 +136,9 @@ struct _Thread
 	char	state[256];
 	void *udata;
 	Alt	*alt;
+	_Procrendez schedrend;
 };
 
-struct _Procrendez
-{
-	Lock		*l;
-	int		asleep;
-#ifdef PLAN9PORT_USING_PTHREADS
-	pthread_cond_t	cond;
-#else
-	int		pid;
-#endif
-};
-
 extern	void	_procsleep(_Procrendez*);
 extern	void	_procwakeup(_Procrendez*);
 extern	void	_procwakeupandunlock(_Procrendez*);
@@ -149,6 +155,7 @@ struct Proc
 #endif
 	Lock		lock;
 	int			nswitch;
+	_Thread		*thread0;
 	_Thread		*thread;
 	_Thread		*pinthread;
 	_Threadlist	runqueue;
@@ -157,6 +164,8 @@ struct Proc
 	uint		nthread;
 	uint		sysproc;
 	_Procrendez	runrend;
+	Lock		schedlock;
+	_Thread	*schedthread;
 	Context	schedcontext;
 	void		*udata;
 	Jmp		sigjmp;
@@ -188,6 +197,8 @@ extern void _threadpexit(void);
 extern void _threaddaemonize(void);
 extern void *_threadstkalloc(int);
 extern void _threadstkfree(void*, int);
+extern void _threadpthreadmain(Proc*, _Thread*);
+extern void _threadpthreadstart(Proc*, _Thread*);
 
 #define USPALIGN(ucp, align) \
 	(void*)((((uintptr)(ucp)->uc_stack.ss_sp+(ucp)->uc_stack.ss_size)-(align))&~((align)-1))