runtime (gc_blocks.go): implement thread-local stack scanning

compiler/atomic.go

@@ -20,7 +20,7 @@ func (b *builder) createAtomicOp(name string) llvm.Value {
 		val := b.getValue(b.fn.Params[1], getPos(b.fn))
 		oldVal := b.CreateAtomicRMW(llvm.AtomicRMWBinOpAnd, ptr, val, llvm.AtomicOrderingSequentiallyConsistent, true)
 		return oldVal
-	case "OrInt32", "OrInt64", "OrUint32", "OrUint64", "OrUintptr":
+	case "atomicOr8", "OrInt32", "OrInt64", "OrUint32", "OrUint64", "OrUintptr":
 		ptr := b.getValue(b.fn.Params[0], getPos(b.fn))
 		val := b.getValue(b.fn.Params[1], getPos(b.fn))
 		oldVal := b.CreateAtomicRMW(llvm.AtomicRMWBinOpOr, ptr, val, llvm.AtomicOrderingSequentiallyConsistent, true)
@@ -37,7 +37,7 @@ func (b *builder) createAtomicOp(name string) llvm.Value {
 		tuple := b.CreateAtomicCmpXchg(ptr, old, newVal, llvm.AtomicOrderingSequentiallyConsistent, llvm.AtomicOrderingSequentiallyConsistent, true)
 		swapped := b.CreateExtractValue(tuple, 1, "")
 		return swapped
-	case "LoadInt32", "LoadInt64", "LoadUint32", "LoadUint64", "LoadUintptr", "LoadPointer":
+	case "atomicLoad8", "LoadInt32", "LoadInt64", "LoadUint32", "LoadUint64", "LoadUintptr", "LoadPointer":
 		ptr := b.getValue(b.fn.Params[0], getPos(b.fn))
 		val := b.CreateLoad(b.getLLVMType(b.fn.Signature.Results().At(0).Type()), ptr, "")
 		val.SetOrdering(llvm.AtomicOrderingSequentiallyConsistent)

compiler/intrinsics.go

@@ -33,7 +33,7 @@ func (b *builder) defineIntrinsicFunction() {
 		b.createVolatileLoad()
 	case strings.HasPrefix(name, "runtime/volatile.Store"):
 		b.createVolatileStore()
-	case strings.HasPrefix(name, "sync/atomic.") && token.IsExported(b.fn.Name()):
+	case (strings.HasPrefix(name, "sync/atomic.") && token.IsExported(b.fn.Name())) || strings.HasPrefix(name, "runtime.atomic"):
 		b.createFunctionStart(true)
 		returnValue := b.createAtomicOp(b.fn.Name())
 		if !returnValue.IsNil() {

src/internal/task/task_threads.c

@@ -147,3 +147,10 @@ void* tinygo_task_current(void) {
 void tinygo_task_send_gc_signal(pthread_t thread) {
     pthread_kill(thread, taskPauseSignal);
 }
+
+// The local gc scan list.
+static __thread void *gcScanList;
+
+void* tinygo_scan_list(void) {
+	return &gcScanList;
+}

src/internal/task/task_threads.go

@@ -147,9 +147,6 @@ func taskExited(t *Task) {
 	}
 }
 
-// scanWaitGroup is used to wait on until all threads have finished the current state transition.
-var scanWaitGroup waitGroup
-
 type waitGroup struct {
 	f Futex
 }
@@ -176,109 +173,9 @@ func (wg *waitGroup) wait() {
 	}
 }
 
-// gcState is used to track and notify threads when the GC is stopping/resuming.
-var gcState Futex
-
-const (
-	gcStateResumed = iota
-	gcStateStopped
-)
-
-// GC scan phase. Because we need to stop the world while scanning, this kinda
-// needs to be done in the tasks package.
-//
-// After calling this function, GCResumeWorld needs to be called once to resume
-// all threads again.
-func GCStopWorldAndScan() {
-	current := Current()
-
-	// NOTE: This does not need to be atomic.
-	if gcState.Load() == gcStateResumed {
-		// Don't allow new goroutines to be started while pausing/resuming threads
-		// in the stop-the-world phase.
-		activeTaskLock.Lock()
-
-		// Wait for threads to finish resuming.
-		scanWaitGroup.wait()
-
-		// Change the gc state to stopped.
-		// NOTE: This does not need to be atomic.
-		gcState.Store(gcStateStopped)
-
-		// Set the number of threads to wait for.
-		scanWaitGroup = initWaitGroup(otherGoroutines)
-
-		// Pause all other threads.
-		for t := activeTasks; t != nil; t = t.state.QueueNext {
-			if t != current {
-				tinygo_task_send_gc_signal(t.state.thread)
-			}
-		}
-
-		// Wait for the threads to finish stopping.
-		scanWaitGroup.wait()
-	}
-
-	// Scan other thread stacks.
-	for t := activeTasks; t != nil; t = t.state.QueueNext {
-		if t != current {
-			markRoots(t.state.stackBottom, t.state.stackTop)
-		}
-	}
-
-	// Scan the current stack, and all current registers.
-	scanCurrentStack()
-
-	// Scan all globals (implemented in the runtime).
-	gcScanGlobals()
-}
-
-// After the GC is done scanning, resume all other threads.
-func GCResumeWorld() {
-	// NOTE: This does not need to be atomic.
-	if gcState.Load() == gcStateResumed {
-		// This is already resumed.
-		return
-	}
-
-	// Set the wait group to track resume progress.
-	scanWaitGroup = initWaitGroup(otherGoroutines)
-
-	// Set the state to resumed.
-	gcState.Store(gcStateResumed)
-
-	// Wake all of the stopped threads.
-	gcState.WakeAll()
-
-	// Allow goroutines to start and exit again.
-	activeTaskLock.Unlock()
-}
-
-//go:linkname markRoots runtime.markRoots
-func markRoots(start, end uintptr)
-
 // Scan globals, implemented in the runtime package.
 func gcScanGlobals()
 
-var stackScanLock PMutex
-
-//export tinygo_task_gc_pause
-func tingyo_task_gc_pause(sig int32) {
-	// Write the entrty stack pointer to the state.
-	Current().state.stackBottom = uintptr(stacksave())
-
-	// Notify the GC that we are stopped.
-	scanWaitGroup.done()
-
-	// Wait for the GC to resume.
-	for gcState.Load() == gcStateStopped {
-		gcState.Wait(gcStateStopped)
-	}
-
-	// Notify the GC that we have resumed.
-	scanWaitGroup.done()
-}
-
 //go:export tinygo_scanCurrentStack
 func scanCurrentStack()
 

src/internal/task/task_threads_blocks.go

@@ -0,0 +1,88 @@
+//go:build scheduler.threads && (gc.conservative || gc.precise)
+
+package task
+
+// stopWaitGroup is used to wait until all threads have stopped.
+var stopWaitGroup waitGroup
+
+// scanWaitGroup is used to wait until all threads have finished scanning their stacks.
+var scanWaitGroup waitGroup
+
+// resumeWaitGroup is used to wait until all threads have resumed.
+var resumeWaitGroup waitGroup
+
+// GC scan phase. Because we need to stop the world while scanning, this kinda
+// needs to be done in the tasks package.
+//
+// After calling this function, GCResumeWorld needs to be called once to resume
+// all threads again.
+func GCStopWorldAndScan() {
+	// Wait for threads to resume from the previous scan.
+	resumeWaitGroup.wait()
+
+	// Don't allow new goroutines to be started while pausing/resuming threads
+	// in the stop-the-world phase.
+	activeTaskLock.Lock()
+
+	// Set the number of threads to wait for.
+	otherGoroutines := otherGoroutines
+	stopWaitGroup = initWaitGroup(otherGoroutines)
+	scanWaitGroup = initWaitGroup(otherGoroutines + 1)
+	resumeWaitGroup = initWaitGroup(otherGoroutines)
+
+	// Pause all other threads.
+	current := Current()
+	for t := activeTasks; t != nil; t = t.state.QueueNext {
+		if t != current {
+			tinygo_task_send_gc_signal(t.state.thread)
+		}
+	}
+
+	// Wait for everything to stop.
+	stopWaitGroup.wait()
+
+	// Scan all globals (implemented in the runtime).
+	gcScanGlobals()
+
+	// Scan our stack and wait for everything else to complete.
+	localScan()
+}
+
+//export tinygo_task_gc_pause
+func tingyo_task_gc_pause(sig int32) {
+	// We have stopped.
+	stopWaitGroup.done()
+
+	// Wait for all other threads to stop.
+	stopWaitGroup.wait()
+
+	// Scan the local stack.
+	localScan()
+
+	// We are resuming.
+	resumeWaitGroup.done()
+}
+
+func localScan() {
+	// Scan the current stack, and all current registers.
+	scanCurrentStack()
+
+	// Assist scanning of heap objects.
+	finishMark()
+
+	// We are done scanning.
+	scanWaitGroup.done()
+
+	// Wait for all other threads to finish scanning.
+	scanWaitGroup.wait()
+}
+
+//go:linkname finishMark runtime.finishMark
+func finishMark()
+
+// GCResumeWorld does not resume anything with the blocks collector.
+// The threads will resume as soon as the scan completes.
+func GCResumeWorld() {
+	// Allow goroutines to start and exit again.
+	activeTaskLock.Unlock()
+}

src/internal/task/task_threads_other.go

@@ -0,0 +1,104 @@
+//go:build scheduler.threads && !(gc.conservative || gc.precise)
+
+package task
+
+// scanWaitGroup is used to wait on until all threads have finished the current state transition.
+var scanWaitGroup waitGroup
+
+// gcState is used to track and notify threads when the GC is stopping/resuming.
+var gcState Futex
+
+const (
+	gcStateResumed = iota
+	gcStateStopped
+)
+
+// GC scan phase. Because we need to stop the world while scanning, this kinda
+// needs to be done in the tasks package.
+//
+// After calling this function, GCResumeWorld needs to be called once to resume
+// all threads again.
+func GCStopWorldAndScan() {
+	current := Current()
+
+	// NOTE: This does not need to be atomic.
+	if gcState.Load() == gcStateResumed {
+		// Don't allow new goroutines to be started while pausing/resuming threads
+		// in the stop-the-world phase.
+		activeTaskLock.Lock()
+
+		// Wait for threads to finish resuming.
+		scanWaitGroup.wait()
+
+		// Change the gc state to stopped.
+		// NOTE: This does not need to be atomic.
+		gcState.Store(gcStateStopped)
+
+		// Set the number of threads to wait for.
+		scanWaitGroup = initWaitGroup(otherGoroutines)
+
+		// Pause all other threads.
+		for t := activeTasks; t != nil; t = t.state.QueueNext {
+			if t != current {
+				tinygo_task_send_gc_signal(t.state.thread)
+			}
+		}
+
+		// Wait for the threads to finish stopping.
+		scanWaitGroup.wait()
+	}
+
+	// Scan other thread stacks.
+	for t := activeTasks; t != nil; t = t.state.QueueNext {
+		if t != current {
+			markRoots(t.state.stackBottom, t.state.stackTop)
+		}
+	}
+
+	// Scan the current stack, and all current registers.
+	scanCurrentStack()
+
+	// Scan all globals (implemented in the runtime).
+	gcScanGlobals()
+}
+
+// After the GC is done scanning, resume all other threads.
+func GCResumeWorld() {
+	// NOTE: This does not need to be atomic.
+	if gcState.Load() == gcStateResumed {
+		// This is already resumed.
+		return
+	}
+
+	// Set the wait group to track resume progress.
+	scanWaitGroup = initWaitGroup(otherGoroutines)
+
+	// Set the state to resumed.
+	gcState.Store(gcStateResumed)
+
+	// Wake all of the stopped threads.
+	gcState.WakeAll()
+
+	// Allow goroutines to start and exit again.
+	activeTaskLock.Unlock()
+}
+
+//go:linkname markRoots runtime.markRoots
+func markRoots(start, end uintptr)
+
+//export tinygo_task_gc_pause
+func tingyo_task_gc_pause(sig int32) {
+	// Write the entrty stack pointer to the state.
+	Current().state.stackBottom = uintptr(stacksave())
+
+	// Notify the GC that we are stopped.
+	scanWaitGroup.done()
+
+	// Wait for the GC to resume.
+	for gcState.Load() == gcStateStopped {
+		gcState.Wait(gcStateStopped)
+	}
+
+	// Notify the GC that we have resumed.
+	scanWaitGroup.done()
+}