runtime (gc_blocks.go): implement thread-local stack scanning

This change allows all thread stacks to be scanned in parallel.
Each thread marks all objects reachable from its stack, using a thread-local scan list.

The marking code now uses atomics to access GC metadata.
If multiple threads try to mark the same object, the first one will responsible for scanning it.

Author: niaow

compiler/atomic.go

@@ -20,7 +20,7 @@ func (b *builder) createAtomicOp(name string) llvm.Value {
 		val := b.getValue(b.fn.Params[1], getPos(b.fn))
 		oldVal := b.CreateAtomicRMW(llvm.AtomicRMWBinOpAnd, ptr, val, llvm.AtomicOrderingSequentiallyConsistent, true)
 		return oldVal
-	case "OrInt32", "OrInt64", "OrUint32", "OrUint64", "OrUintptr":
+	case "atomicOr8", "OrInt32", "OrInt64", "OrUint32", "OrUint64", "OrUintptr":
 		ptr := b.getValue(b.fn.Params[0], getPos(b.fn))
 		val := b.getValue(b.fn.Params[1], getPos(b.fn))
 		oldVal := b.CreateAtomicRMW(llvm.AtomicRMWBinOpOr, ptr, val, llvm.AtomicOrderingSequentiallyConsistent, true)
@@ -37,7 +37,7 @@ func (b *builder) createAtomicOp(name string) llvm.Value {
 		tuple := b.CreateAtomicCmpXchg(ptr, old, newVal, llvm.AtomicOrderingSequentiallyConsistent, llvm.AtomicOrderingSequentiallyConsistent, true)
 		swapped := b.CreateExtractValue(tuple, 1, "")
 		return swapped
-	case "LoadInt32", "LoadInt64", "LoadUint32", "LoadUint64", "LoadUintptr", "LoadPointer":
+	case "atomicLoad8", "LoadInt32", "LoadInt64", "LoadUint32", "LoadUint64", "LoadUintptr", "LoadPointer":
 		ptr := b.getValue(b.fn.Params[0], getPos(b.fn))
 		val := b.CreateLoad(b.getLLVMType(b.fn.Signature.Results().At(0).Type()), ptr, "")
 		val.SetOrdering(llvm.AtomicOrderingSequentiallyConsistent)

compiler/intrinsics.go

@@ -33,7 +33,7 @@ func (b *builder) defineIntrinsicFunction() {
 		b.createVolatileLoad()
 	case strings.HasPrefix(name, "runtime/volatile.Store"):
 		b.createVolatileStore()
-	case strings.HasPrefix(name, "sync/atomic.") && token.IsExported(b.fn.Name()):
+	case (strings.HasPrefix(name, "sync/atomic.") && token.IsExported(b.fn.Name())) || strings.HasPrefix(name, "runtime.atomic"):
 		b.createFunctionStart(true)
 		returnValue := b.createAtomicOp(b.fn.Name())
 		if !returnValue.IsNil() {

src/internal/task/task_threads.c

@@ -147,3 +147,10 @@ void* tinygo_task_current(void) {
 void tinygo_task_send_gc_signal(pthread_t thread) {
     pthread_kill(thread, taskPauseSignal);
 }
+
+// The local gc scan list.
+static __thread void *gcScanList;
+
+void* tinygo_scan_list(void) {
+	return &gcScanList;
+}

src/internal/task/task_threads.go

@@ -147,9 +147,6 @@ func taskExited(t *Task) {
 	}
 }
 
-// scanWaitGroup is used to wait on until all threads have finished the current state transition.
-var scanWaitGroup waitGroup
-
 type waitGroup struct {
 	f Futex
 }
@@ -176,109 +173,9 @@ func (wg *waitGroup) wait() {
 	}
 }
 
-// gcState is used to track and notify threads when the GC is stopping/resuming.
-var gcState Futex
-
-const (
-	gcStateResumed = iota
-	gcStateStopped
-)
-
-// GC scan phase. Because we need to stop the world while scanning, this kinda
-// needs to be done in the tasks package.
-//
-// After calling this function, GCResumeWorld needs to be called once to resume
-// all threads again.
-func GCStopWorldAndScan() {
-	current := Current()
-
-	// NOTE: This does not need to be atomic.
-	if gcState.Load() == gcStateResumed {
-		// Don't allow new goroutines to be started while pausing/resuming threads
-		// in the stop-the-world phase.
-		activeTaskLock.Lock()
-
-		// Wait for threads to finish resuming.
-		scanWaitGroup.wait()
-
-		// Change the gc state to stopped.
-		// NOTE: This does not need to be atomic.
-		gcState.Store(gcStateStopped)
-
-		// Set the number of threads to wait for.
-		scanWaitGroup = initWaitGroup(otherGoroutines)
-
-		// Pause all other threads.
-		for t := activeTasks; t != nil; t = t.state.QueueNext {
-			if t != current {
-				tinygo_task_send_gc_signal(t.state.thread)
-			}
-		}
-
-		// Wait for the threads to finish stopping.
-		scanWaitGroup.wait()
-	}
-
-	// Scan other thread stacks.
-	for t := activeTasks; t != nil; t = t.state.QueueNext {
-		if t != current {
-			markRoots(t.state.stackBottom, t.state.stackTop)
-		}
-	}
-
-	// Scan the current stack, and all current registers.
-	scanCurrentStack()
-
-	// Scan all globals (implemented in the runtime).
-	gcScanGlobals()
-}
-
-// After the GC is done scanning, resume all other threads.
-func GCResumeWorld() {
-	// NOTE: This does not need to be atomic.
-	if gcState.Load() == gcStateResumed {
-		// This is already resumed.
-		return
-	}
-
-	// Set the wait group to track resume progress.
-	scanWaitGroup = initWaitGroup(otherGoroutines)
-
-	// Set the state to resumed.
-	gcState.Store(gcStateResumed)
-
-	// Wake all of the stopped threads.
-	gcState.WakeAll()
-
-	// Allow goroutines to start and exit again.
-	activeTaskLock.Unlock()
-}
-
-//go:linkname markRoots runtime.markRoots
-func markRoots(start, end uintptr)
-
 // Scan globals, implemented in the runtime package.
 func gcScanGlobals()
 
-var stackScanLock PMutex
-
-//export tinygo_task_gc_pause
-func tingyo_task_gc_pause(sig int32) {
-	// Write the entrty stack pointer to the state.
-	Current().state.stackBottom = uintptr(stacksave())
-
-	// Notify the GC that we are stopped.
-	scanWaitGroup.done()
-
-	// Wait for the GC to resume.
-	for gcState.Load() == gcStateStopped {
-		gcState.Wait(gcStateStopped)
-	}
-
-	// Notify the GC that we have resumed.
-	scanWaitGroup.done()
-}
-
 //go:export tinygo_scanCurrentStack
 func scanCurrentStack()
 

src/internal/task/task_threads_blocks.go

@@ -0,0 +1,88 @@
+//go:build scheduler.threads && (gc.conservative || gc.precise)
+
+package task
+
+// stopWaitGroup is used to wait until all threads have stopped.
+var stopWaitGroup waitGroup
+
+// scanWaitGroup is used to wait until all threads have finished scanning their stacks.
+var scanWaitGroup waitGroup
+
+// resumeWaitGroup is used to wait until all threads have resumed.
+var resumeWaitGroup waitGroup
+
+// GC scan phase. Because we need to stop the world while scanning, this kinda
+// needs to be done in the tasks package.
+//
+// After calling this function, GCResumeWorld needs to be called once to resume
+// all threads again.
+func GCStopWorldAndScan() {
+	// Wait for threads to resume from the previous scan.
+	resumeWaitGroup.wait()
+
+	// Don't allow new goroutines to be started while pausing/resuming threads
+	// in the stop-the-world phase.
+	activeTaskLock.Lock()
+
+	// Set the number of threads to wait for.
+	otherGoroutines := otherGoroutines
+	stopWaitGroup = initWaitGroup(otherGoroutines)
+	scanWaitGroup = initWaitGroup(otherGoroutines + 1)
+	resumeWaitGroup = initWaitGroup(otherGoroutines)
+
+	// Pause all other threads.
+	current := Current()
+	for t := activeTasks; t != nil; t = t.state.QueueNext {
+		if t != current {
+			tinygo_task_send_gc_signal(t.state.thread)
+		}
+	}
+
+	// Wait for everything to stop.
+	stopWaitGroup.wait()
+
+	// Scan all globals (implemented in the runtime).
+	gcScanGlobals()
+
+	// Scan our stack and wait for everything else to complete.
+	localScan()
+}
+
+//export tinygo_task_gc_pause
+func tingyo_task_gc_pause(sig int32) {
+	// We have stopped.
+	stopWaitGroup.done()
+
+	// Wait for all other threads to stop.
+	stopWaitGroup.wait()
+
+	// Scan the local stack.
+	localScan()
+
+	// We are resuming.
+	resumeWaitGroup.done()
+}
+
+func localScan() {
+	// Scan the current stack, and all current registers.
+	scanCurrentStack()
+
+	// Assist scanning of heap objects.
+	finishMark()
+
+	// We are done scanning.
+	scanWaitGroup.done()
+
+	// Wait for all other threads to finish scanning.
+	scanWaitGroup.wait()
+}
+
+//go:linkname finishMark runtime.finishMark
+func finishMark()
+
+// GCResumeWorld does not resume anything with the blocks collector.
+// The threads will resume as soon as the scan completes.
+func GCResumeWorld() {
+	// Allow goroutines to start and exit again.
+	activeTaskLock.Unlock()
+}

src/internal/task/task_threads_other.go

@@ -0,0 +1,104 @@
+//go:build scheduler.threads && !(gc.conservative || gc.precise)
+
+package task
+
+// scanWaitGroup is used to wait on until all threads have finished the current state transition.
+var scanWaitGroup waitGroup
+
+// gcState is used to track and notify threads when the GC is stopping/resuming.
+var gcState Futex
+
+const (
+	gcStateResumed = iota
+	gcStateStopped
+)
+
+// GC scan phase. Because we need to stop the world while scanning, this kinda
+// needs to be done in the tasks package.
+//
+// After calling this function, GCResumeWorld needs to be called once to resume
+// all threads again.
+func GCStopWorldAndScan() {
+	current := Current()
+
+	// NOTE: This does not need to be atomic.
+	if gcState.Load() == gcStateResumed {
+		// Don't allow new goroutines to be started while pausing/resuming threads
+		// in the stop-the-world phase.
+		activeTaskLock.Lock()
+
+		// Wait for threads to finish resuming.
+		scanWaitGroup.wait()
+
+		// Change the gc state to stopped.
+		// NOTE: This does not need to be atomic.
+		gcState.Store(gcStateStopped)
+
+		// Set the number of threads to wait for.
+		scanWaitGroup = initWaitGroup(otherGoroutines)
+
+		// Pause all other threads.
+		for t := activeTasks; t != nil; t = t.state.QueueNext {
+			if t != current {
+				tinygo_task_send_gc_signal(t.state.thread)
+			}
+		}
+
+		// Wait for the threads to finish stopping.
+		scanWaitGroup.wait()
+	}
+
+	// Scan other thread stacks.
+	for t := activeTasks; t != nil; t = t.state.QueueNext {
+		if t != current {
+			markRoots(t.state.stackBottom, t.state.stackTop)
+		}
+	}
+
+	// Scan the current stack, and all current registers.
+	scanCurrentStack()
+
+	// Scan all globals (implemented in the runtime).
+	gcScanGlobals()
+}
+
+// After the GC is done scanning, resume all other threads.
+func GCResumeWorld() {
+	// NOTE: This does not need to be atomic.
+	if gcState.Load() == gcStateResumed {
+		// This is already resumed.
+		return
+	}
+
+	// Set the wait group to track resume progress.
+	scanWaitGroup = initWaitGroup(otherGoroutines)
+
+	// Set the state to resumed.
+	gcState.Store(gcStateResumed)
+
+	// Wake all of the stopped threads.
+	gcState.WakeAll()
+
+	// Allow goroutines to start and exit again.
+	activeTaskLock.Unlock()
+}
+
+//go:linkname markRoots runtime.markRoots
+func markRoots(start, end uintptr)
+
+//export tinygo_task_gc_pause
+func tingyo_task_gc_pause(sig int32) {
+	// Write the entrty stack pointer to the state.
+	Current().state.stackBottom = uintptr(stacksave())
+
+	// Notify the GC that we are stopped.
+	scanWaitGroup.done()
+
+	// Wait for the GC to resume.
+	for gcState.Load() == gcStateStopped {
+		gcState.Wait(gcStateStopped)
+	}
+
+	// Notify the GC that we have resumed.
+	scanWaitGroup.done()
+}