change: slaves only sync from masters

Author: vanhauser-thc

TODO.md

@@ -2,8 +2,6 @@
 
 ## Roadmap 2.65+
 
- - sync_fuzzers(): only masters sync from all, slaves only sync from master
-   (@andrea: be careful, often people run all slaves)
  - AFL_MAP_SIZE for qemu_mode and unicorn_mode
  - random crc32 HASH_CONST per run? because with 65536 paths we have collisions
  - namespace for targets? e.g. network

docs/Changelog.md

@@ -9,8 +9,11 @@ Want to stay in the loop on major new features? Join our mailing list by
 sending a mail to <[email protected]>.
 
 
-### Version ++2.6d (dev)
-  - ...
+### Version ++2.65d (dev)
+  - afl-fuzz:
+     - -S slaves now only sync from the master to increase performance,
+       the -M master stilly syncs from everyone. Added checks that exactly
+       one master is present
 
 
 ### Version ++2.65c (release):

docs/parallel_fuzzing.md

@@ -57,12 +57,14 @@ Each fuzzer will keep its state in a separate subdirectory, like so:
 Each instance will also periodically rescan the top-level sync directory
 for any test cases found by other fuzzers - and will incorporate them into
 its own fuzzing when they are deemed interesting enough.
+For performance reasons only -M masters sync the queue with everyone, the
+-S slaves will only sync from the master.
 
 The difference between the -M and -S modes is that the master instance will
 still perform deterministic checks; while the secondary instances will
-proceed straight to random tweaks. If you don't want to do deterministic
-fuzzing at all, it's OK to run all instances with -S. With very slow or complex
-targets, or when running heavily parallelized jobs, this is usually a good plan.
+proceed straight to random tweaks.
+
+Note that you must always have one -M master instance!
 
 Note that running multiple -M instances is wasteful, although there is an
 experimental support for parallelizing the deterministic checks. To leverage

include/afl-fuzz.h

@@ -913,6 +913,7 @@ u32    find_start_position(afl_state_t *);
 void   find_timeout(afl_state_t *);
 double get_runnable_processes(void);
 void   nuke_resume_dir(afl_state_t *);
+int    check_master_exists(afl_state_t *);
 void   setup_dirs_fds(afl_state_t *);
 void   setup_cmdline_file(afl_state_t *, char **);
 void   setup_stdio_file(afl_state_t *);

llvm_mode/LLVMInsTrim.so.cc

@@ -160,21 +160,23 @@ struct InsTrim : public ModulePass {
     else
 #else
     if (ngram_size_str)
-#ifdef LLVM_VERSION_STRING
+  #ifdef LLVM_VERSION_STRING
       FATAL(
           "Sorry, NGRAM branch coverage is not supported with llvm version %s!",
           LLVM_VERSION_STRING);
-#else
-#ifndef LLVM_VERSION_PATCH
+  #else
+    #ifndef LLVM_VERSION_PATCH
       FATAL(
-          "Sorry, NGRAM branch coverage is not supported with llvm version %d.%d.%d!",
+          "Sorry, NGRAM branch coverage is not supported with llvm version "
+          "%d.%d.%d!",
           LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR, 0);
-#else
+    #else
       FATAL(
-          "Sorry, NGRAM branch coverage is not supported with llvm version %d.%d.%d!",
+          "Sorry, NGRAM branch coverage is not supported with llvm version "
+          "%d.%d.%d!",
           LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR, LLVM_VERISON_PATCH);
-#endif
-#endif
+    #endif
+  #endif
 #endif
       PrevLocSize = 1;
 

llvm_mode/afl-llvm-pass.so.cc

@@ -211,15 +211,17 @@ bool AFLCoverage::runOnModule(Module &M) {
   else
 #else
   if (ngram_size_str)
-#ifndef LLVM_VERSION_PATCH
-    FATAL("Sorry, NGRAM branch coverage is not supported with llvm version %d.%d.%d!",
-          LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR,
-          0);
-#else
-    FATAL("Sorry, NGRAM branch coverage is not supported with llvm version %d.%d.%d!",
-          LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR,
-          LLVM_VERSION_PATCH);
-#endif
+  #ifndef LLVM_VERSION_PATCH
+    FATAL(
+        "Sorry, NGRAM branch coverage is not supported with llvm version "
+        "%d.%d.%d!",
+        LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR, 0);
+  #else
+    FATAL(
+        "Sorry, NGRAM branch coverage is not supported with llvm version "
+        "%d.%d.%d!",
+        LLVM_VERSION_MAJOR, LLVM_VERSION_MINOR, LLVM_VERSION_PATCH);
+  #endif
 #endif
     PrevLocSize = 1;
 

src/afl-fuzz-init.c

@@ -1315,6 +1315,36 @@ static void handle_existing_out_dir(afl_state_t *afl) {
 
 }
 
+/* If this is a -S slave, ensure a -M master is running */
+
+int check_master_exists(afl_state_t *afl) {
+
+  DIR *          sd;
+  struct dirent *sd_ent;
+  u8 *           fn;
+  sd = opendir(afl->sync_dir);
+  if (!sd) { PFATAL("Unable to open '%s'", afl->sync_dir); }
+  while ((sd_ent = readdir(sd))) {
+
+    /* Skip dot files and our own output directory. */
+
+    if (sd_ent->d_name[0] == '.' || !strcmp(afl->sync_id, sd_ent->d_name)) {
+
+      continue;
+
+    }
+
+    fn = alloc_printf("%s/%s/is_master", afl->sync_dir, sd_ent->d_name);
+    int res = access(fn, F_OK);
+    free(fn);
+    if (res == 0) return 1;
+
+  }
+
+  return 0;
+
+}
+
 /* Prepare output directories and fds. */
 
 void setup_dirs_fds(afl_state_t *afl) {
@@ -1330,18 +1360,15 @@ void setup_dirs_fds(afl_state_t *afl) {
 
   }
 
-  /*
-    if (afl->is_master) {
+  if (afl->is_master) {
 
-      u8 *x = alloc_printf("%s/%s/is_master", afl->sync_dir, afl->sync_id);
-      int fd = open(x, O_CREAT | O_RDWR, 0644);
-      if (fd < 0) FATAL("cannot create %s", x);
-      free(x);
-      close(fd);
-
-    }
+    u8 *x = alloc_printf("%s/%s/is_master", afl->sync_dir, afl->sync_id);
+    int fd = open(x, O_CREAT | O_RDWR, 0644);
+    if (fd < 0) FATAL("cannot create %s", x);
+    free(x);
+    close(fd);
 
-  */
+  }
 
   if (mkdir(afl->out_dir, 0700)) {
 

src/afl-fuzz-run.c

@@ -401,19 +401,15 @@ void sync_fuzzers(afl_state_t *afl) {
 
     }
 
-    /*
-        // a slave only syncs from a master, a master syncs from everyone
-        if (likely(afl->is_slave)) {
+    // a slave only syncs from a master, a master syncs from everyone
+    if (likely(afl->is_slave)) {
 
-          u8 x = alloc_printf("%s/%s/is_master", afl->sync_dir, sd_ent->d_name);
-          int res = access(x, F_OK);
-          free(x);
-          if (res != 0)
-            continue;
+      u8 *x = alloc_printf("%s/%s/is_master", afl->sync_dir, sd_ent->d_name);
+      int res = access(x, F_OK);
+      free(x);
+      if (likely(res != 0)) continue;
 
-        }
-
-    */
+    }
 
     /* Skip anything that doesn't have a queue/ subdirectory. */
 

src/afl-fuzz.c

@@ -1065,8 +1065,21 @@ int main(int argc, char **argv_orig, char **envp) {
 
   init_count_class16();
 
+  if (afl->is_master && check_master_exists(afl) == 1) {
+
+    WARNF("It is wasteful to run more than one master!");
+
+  }
+
   setup_dirs_fds(afl);
 
+  if (afl->is_slave && check_master_exists(afl) == 0) {
+
+    WARNF("no -M master found. You need to run one master!");
+    sleep(5);
+
+  }
+
   setup_custom_mutators(afl);
 
   setup_cmdline_file(afl, argv + optind);