Evaluation of two concurrent linked lists: QSBR and lock-based

Author: jserv

.gitignore

@@ -18,6 +18,8 @@ rcu_queue/rcu_queue
 redirect/redirect
 tinync/tinync
 tpool/tpool
+list-move/bench-lock
+list-move/bench-lockfree
 
 # external source files
 preempt_sched/list.h

README.md

@@ -23,6 +23,7 @@ purpose of these programs is to be illustrative and educational.
 * [Synchronization](https://en.wikipedia.org/wiki/Synchronization_(computer_science))
     - [qsbr](qsbr/): An implementation of Quiescent state based reclamation (QSBR).
     - [rcu\_list](rcu_list/): A concurrent linked list utilizing the simplified RCU algorithm.
+    - [list-move](list-move/): Evaluation of two concurrent linked lists: QSBR and lock-based.
     - [rcu\_queue](rcu_queue/): An efficient concurrent queue based on QSBR.
 * Applications
     - [httpd](httpd/): A multi-threaded web server.

list-move/Makefile

@@ -0,0 +1,23 @@
+CFLAGS = -Wall -Wextra
+LDFLAGS = -lpthread
+
+BINS = bench-lock bench-lockfree
+
+all: $(BINS)
+
+bench-lock: bench.c lock.c
+	$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
+
+bench-lockfree: bench.c lockfree.c
+	$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
+
+clean:
+	rm -f $(BINS)
+
+check: $(BINS)
+	./bench-lock
+	@echo
+	./bench-lockfree
+
+indent:
+	clang-format -i *.[ch]

list-move/bench.c

@@ -0,0 +1,150 @@
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include "bench.h"
+
+static void barrier_init(barrier_t *b, int n)
+{
+    pthread_cond_init(&b->complete, NULL);
+    pthread_mutex_init(&b->mutex, NULL);
+    b->count = n;
+    b->crossing = 0;
+}
+
+static void barrier_cross(barrier_t *b)
+{
+    pthread_mutex_lock(&b->mutex);
+    b->crossing++;
+    if (b->crossing < b->count)
+        pthread_cond_wait(&b->complete, &b->mutex);
+    else {
+        pthread_cond_broadcast(&b->complete);
+        b->crossing = 0;
+    }
+    pthread_mutex_unlock(&b->mutex);
+}
+
+#define DEFAULT_DURATION 1000
+#define DEFAULT_NTHREADS 64
+#define DEFAULT_ISIZE 256
+#define DEFAULT_VRANGE 512
+
+static volatile bool should_stop = false;
+
+static void *bench_thread(void *data)
+{
+    pthread_data_t *d = (pthread_data_t *) data;
+
+    barrier_cross(d->barrier);
+    while (!should_stop) {
+        int from = rand_r(&d->seed) & 0x1;
+        int key = rand_r(&d->seed) % d->range;
+        list_move(key, d, from);
+        d->n_move++;
+    }
+
+    return NULL;
+}
+
+int main(void)
+{
+    int duration = DEFAULT_DURATION;
+    int n_threads = DEFAULT_NTHREADS;
+    int init_size = DEFAULT_ISIZE;
+    int value_range = DEFAULT_VRANGE;
+
+    printf("List move benchmark\n");
+    printf("Test time:     %d\n", duration);
+    printf("Thread number: %d\n", n_threads);
+    printf("Initial size:  %d\n", init_size);
+    printf("Value range:   %d\n", value_range);
+
+    struct timespec timeout = {.tv_sec = duration / 1000,
+                               .tv_nsec = (duration % 1000) * 1000000};
+
+    pthread_t *threads;
+    if (!(threads = malloc(n_threads * sizeof(pthread_t)))) {
+        printf("failed to malloc pthread_t\n");
+        goto out;
+    }
+
+    pthread_data_t **data;
+    if (!(data = malloc(n_threads * sizeof(pthread_data_t *)))) {
+        printf("failed to malloc pthread_data_t\n");
+        goto out;
+    }
+    for (int i = 0; i < n_threads; i++) {
+        if ((data[i] = alloc_pthread_data()) == NULL) {
+            printf("failed to malloc pthread_data_t %d\n", i);
+            goto out;
+        }
+    }
+
+    srand(getpid() ^ (uintptr_t) main);
+
+    void *list;
+    if (!(list = list_global_init(init_size, value_range))) {
+        printf("failed to do list_global_init\n");
+        goto out;
+    }
+
+    barrier_t barrier;
+    pthread_attr_t attr;
+    barrier_init(&barrier, n_threads + 1);
+    pthread_attr_init(&attr);
+    pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE);
+    for (int i = 0; i < n_threads; i++) {
+        data[i]->id = i;
+        data[i]->n_move = 0;
+        data[i]->range = value_range;
+        data[i]->seed = rand();
+        data[i]->barrier = &barrier;
+        data[i]->list = list;
+        if (list_thread_init(data[i])) {
+            printf("Failed to do list_thread_init\n");
+            goto out;
+        }
+        if (pthread_create(&threads[i], &attr, bench_thread,
+                           (void *) (data[i])) != 0) {
+            printf("Failed to create thread %d\n", i);
+            goto out;
+        }
+    }
+    pthread_attr_destroy(&attr);
+
+    barrier_cross(&barrier);
+
+    struct timeval start, end;
+    gettimeofday(&start, NULL);
+    nanosleep(&timeout, NULL);
+    should_stop = true;
+    gettimeofday(&end, NULL);
+
+    for (int i = 0; i < n_threads; i++) {
+        if (pthread_join(threads[i], NULL) != 0) {
+            printf("Failed to join child thread %d\n", i);
+            goto out;
+        }
+    }
+
+    duration = (end.tv_sec * 1000 + end.tv_usec / 1000) -
+               (start.tv_sec * 1000 + start.tv_usec / 1000);
+    unsigned long n_move = 0;
+    for (int i = 0; i < n_threads; i++)
+        n_move += (data[i]->n_move);
+
+    printf("\tduration:     %d ms\n", duration);
+    printf("\tops/second    %lu (%f/s)\n", n_move,
+           n_move * (1000.0) / duration);
+
+    for (int i = 0; i < n_threads; i++)
+        free_pthread_data(data[i]);
+    list_global_exit(list);
+    free(data);
+    free(threads);
+
+out:
+    return 0;
+}

list-move/bench.h

@@ -0,0 +1,39 @@
+#ifndef BENCH_MOVE_LIST_H
+#define BENCH_MOVE_LIST_H
+
+#include <limits.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+typedef struct {
+    pthread_cond_t complete;
+    pthread_mutex_t mutex;
+    int count, crossing;
+} barrier_t;
+
+#define CACHE_LINE (64)
+#define CACHE_ALIGN(size) ((((size - 1) / CACHE_LINE) + 1) * CACHE_LINE)
+
+#define PTHREAD_PADDING (16)
+
+typedef struct {
+    long pthread_padding[PTHREAD_PADDING];
+    long id;
+    unsigned long n_move;
+    int range;
+    unsigned int seed;
+    barrier_t *barrier;
+    void *list;
+    void *ds_data; /* data structure specific data */
+} pthread_data_t;
+
+pthread_data_t *alloc_pthread_data(void);
+
+void free_pthread_data(pthread_data_t *d);
+void *list_global_init(int init_size, int value_range);
+int list_thread_init(pthread_data_t *data);
+void list_global_exit(void *list);
+int list_move(int key, pthread_data_t *data, int from);
+
+#endif

list-move/lock.c

@@ -0,0 +1,107 @@
+#include <assert.h>
+
+#include "bench.h"
+
+typedef struct node {
+    int val;
+    struct node *next;
+} node_t;
+
+typedef struct spinlock_list {
+    pthread_spinlock_t spinlock;
+    node_t *head[2];
+} spinlock_list_t;
+
+pthread_data_t *alloc_pthread_data(void)
+{
+    size_t size = sizeof(pthread_data_t);
+    size = CACHE_ALIGN(size);
+
+    pthread_data_t *d = malloc(size);
+    if (d)
+        d->ds_data = NULL;
+
+    return d;
+}
+
+void free_pthread_data(pthread_data_t *d)
+{
+    free(d);
+}
+
+void *list_global_init(int size, int value_range)
+{
+    spinlock_list_t *list = malloc(sizeof(spinlock_list_t));
+    if (!list)
+        return NULL;
+
+    pthread_spin_init(&list->spinlock, PTHREAD_PROCESS_PRIVATE);
+    list->head[0] = malloc(sizeof(node_t));
+    list->head[1] = malloc(sizeof(node_t));
+    if (!list->head[0] || !list->head[1])
+        return NULL;
+    node_t *node[2] = {list->head[0], list->head[1]};
+    node[0]->val = INT_MIN;
+    node[1]->val = INT_MIN;
+
+    for (int i = 0; i < value_range; i += value_range / size) {
+        node[0]->next = malloc(sizeof(node_t));
+        node[1]->next = malloc(sizeof(node_t));
+        if (!node[0]->next || !node[1]->next)
+            return NULL;
+        node[0] = node[0]->next;
+        node[1] = node[1]->next;
+        node[0]->val = i;
+        node[1]->val = i + 1;
+    }
+    node[0]->next = malloc(sizeof(node_t));
+    node[1]->next = malloc(sizeof(node_t));
+    if (!node[0]->next || !node[1]->next)
+        return NULL;
+    node[0]->val = node[1]->val = INT_MAX;
+
+    return list;
+}
+
+int list_thread_init(pthread_data_t *data)
+{
+    return 0;
+}
+
+void list_global_exit(void *list)
+{
+    spinlock_list_t *l = (spinlock_list_t *) list;
+    pthread_spin_destroy(&l->spinlock);
+}
+
+int list_move(int key, pthread_data_t *data, int from)
+{
+    spinlock_list_t *list = (spinlock_list_t *) data->list;
+    node_t *prev_src, *cur, *prev_dst, *next_dst;
+    int val;
+
+    pthread_spin_lock(&list->spinlock);
+    for (prev_src = list->head[from], cur = prev_src->next; cur;
+         prev_src = cur, cur = cur->next)
+        if ((val = cur->val) >= key)
+            break;
+
+    int ret = (val == key);
+    if (!ret)
+        goto out;
+    for (prev_dst = list->head[1 - from], next_dst = prev_dst->next; next_dst;
+         prev_dst = next_dst, next_dst = next_dst->next)
+        if ((val = next_dst->val) >= key)
+            break;
+    ret = (val != key);
+    if (!ret)
+        goto out;
+
+    prev_src->next = cur->next;
+    prev_dst->next = cur;
+    cur->next = next_dst;
+out:
+    pthread_spin_unlock(&list->spinlock);
+
+    return ret;
+}