split up testcases

Author: domenukk

GNUmakefile

@@ -270,7 +270,7 @@ all:	test_x86 test_shm test_python ready $(PROGS) afl-as test_build all_done
 man:    $(MANPAGES)
 
 tests:	source-only
-	@cd test ; ./test.sh
+	@cd test ; ./test-all.sh
 	@rm -f test/errors
 
 performance-tests:	performance-test

test/test-all.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+source ./test-pre.sh
+
+source ./test-basic.sh
+
+source ./test-llvm.sh
+
+source ./test-llvm-lto.sh
+
+source ./test-gcc-plugin.sh
+
+source ./test-compcov.sh
+
+source ./test-qemu-mode.sh
+
+source ./test-unicorn-mode.sh
+
+source ./test-custom-mutators.sh
+
+source ./test-unittests.sh
+
+source ./test-post.sh

test/test-basic.sh

@@ -0,0 +1,125 @@
+#!/bin/sh
+
+source ./test-pre.sh
+
+$ECHO "$BLUE[*] Testing: ${AFL_GCC}, afl-showmap, afl-fuzz, afl-cmin and afl-tmin"
+test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc" -o "$SYS" = "i386" && {
+ test -e ../${AFL_GCC} -a -e ../afl-showmap -a -e ../afl-fuzz && {
+  ../${AFL_GCC} -o test-instr.plain ../test-instr.c > /dev/null 2>&1
+  AFL_HARDEN=1 ../${AFL_GCC} -o test-compcov.harden test-compcov.c > /dev/null 2>&1
+  test -e test-instr.plain && {
+    $ECHO "$GREEN[+] ${AFL_GCC} compilation succeeded"
+    echo 0 | ../afl-showmap -m ${MEM_LIMIT} -o test-instr.plain.0 -r -- ./test-instr.plain > /dev/null 2>&1
+    ../afl-showmap -m ${MEM_LIMIT} -o test-instr.plain.1 -r -- ./test-instr.plain < /dev/null > /dev/null 2>&1
+    test -e test-instr.plain.0 -a -e test-instr.plain.1 && {
+      diff test-instr.plain.0 test-instr.plain.1 > /dev/null 2>&1 && {
+        $ECHO "$RED[!] ${AFL_GCC} instrumentation should be different on different input but is not"
+        CODE=1
+      } || {
+        $ECHO "$GREEN[+] ${AFL_GCC} instrumentation present and working correctly"
+      }
+    } || {
+      $ECHO "$RED[!] ${AFL_GCC} instrumentation failed"
+      CODE=1
+    }
+    rm -f test-instr.plain.0 test-instr.plain.1
+    TUPLES=`echo 0|../afl-showmap -m ${MEM_LIMIT} -o /dev/null -- ./test-instr.plain 2>&1 | grep Captur | awk '{print$3}'`
+    test "$TUPLES" -gt 3 -a "$TUPLES" -lt 11 && {
+      $ECHO "$GREEN[+] ${AFL_GCC} run reported $TUPLES instrumented locations which is fine"
+    } || {
+      $ECHO "$RED[!] ${AFL_GCC} instrumentation produces weird numbers: $TUPLES"
+      CODE=1
+    }
+  } || {
+    $ECHO "$RED[!] ${AFL_GCC} failed"
+    echo CUT------------------------------------------------------------------CUT
+    uname -a
+    ../${AFL_GCC} -o test-instr.plain ../test-instr.c
+    echo CUT------------------------------------------------------------------CUT
+    CODE=1
+  }
+  test -e test-compcov.harden && {
+    grep -Eq$GREPAOPTION 'stack_chk_fail|fstack-protector-all|fortified' test-compcov.harden > /dev/null 2>&1 && {
+      $ECHO "$GREEN[+] ${AFL_GCC} hardened mode succeeded and is working"
+    } || {
+      $ECHO "$RED[!] ${AFL_GCC} hardened mode is not hardened"
+      CODE=1
+    }
+    rm -f test-compcov.harden
+  } || {
+    $ECHO "$RED[!] ${AFL_GCC} hardened mode compilation failed"
+    CODE=1
+  }
+  # now we want to be sure that afl-fuzz is working
+  # make sure core_pattern is set to core on linux
+  (test "$(uname -s)" = "Linux" && test "$(sysctl kernel.core_pattern)" != "kernel.core_pattern = core" && {
+    $ECHO "$YELLOW[-] we should not run afl-fuzz with enabled core dumps. Run 'sudo sh afl-system-config'.$RESET"
+    true
+  }) ||
+  # make sure crash reporter is disabled on Mac OS X
+  (test "$(uname -s)" = "Darwin" && test $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') && {
+    $ECHO "$RED[!] we cannot run afl-fuzz with enabled crash reporter. Run 'sudo sh afl-system-config'.$RESET"
+    true
+  }) || {
+    mkdir -p in
+    echo 0 > in/in
+    $ECHO "$GREY[*] running afl-fuzz for ${AFL_GCC}, this will take approx 10 seconds"
+    {
+      ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -- ./test-instr.plain >>errors 2>&1
+    } >>errors 2>&1
+    test -n "$( ls out/queue/id:000002* 2>/dev/null )" && {
+      $ECHO "$GREEN[+] afl-fuzz is working correctly with ${AFL_GCC}"
+    } || {
+      echo CUT------------------------------------------------------------------CUT
+      cat errors
+      echo CUT------------------------------------------------------------------CUT
+      $ECHO "$RED[!] afl-fuzz is not working correctly with ${AFL_GCC}"
+      CODE=1
+    }
+    echo 000000000000000000000000 > in/in2
+    echo 111 > in/in3
+    mkdir -p in2
+    ../afl-cmin -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null 2>&1 # why is afl-forkserver writing to stderr?
+    CNT=`ls in2/* 2>/dev/null | wc -l`
+    case "$CNT" in
+      *2) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;;
+      *)  $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases ($CNT)"
+          CODE=1
+          ;;
+    esac
+    rm -f in2/in*
+    export AFL_QUIET=1
+    if command -v bash >/dev/null ; then {
+      ../afl-cmin.bash -m ${MEM_LIMIT} -i in -o in2 -- ./test-instr.plain >/dev/null
+      CNT=`ls in2/* 2>/dev/null | wc -l`
+      case "$CNT" in
+        *2) $ECHO "$GREEN[+] afl-cmin.bash correctly minimized the number of testcases" ;;
+        *)  $ECHO "$RED[!] afl-cmin.bash did not correctly minimize the number of testcases ($CNT)"
+            CODE=1
+            ;;
+        esac
+    } else {
+      $ECHO "$YELLOW[-] no bash available, cannot test afl-cmin.bash"
+      INCOMPLETE=1
+    }
+    fi
+    ../afl-tmin -m ${MEM_LIMIT} -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1
+    SIZE=`ls -l in2/in2 2>/dev/null | awk '{print$5}'`
+    test "$SIZE" = 1 && $ECHO "$GREEN[+] afl-tmin correctly minimized the testcase"
+    test "$SIZE" = 1 || {
+       $ECHO "$RED[!] afl-tmin did incorrectly minimize the testcase to $SIZE"
+       CODE=1
+    }
+    rm -rf in out errors in2
+    unset AFL_QUIET
+  }
+  rm -f test-instr.plain
+ } || {
+  $ECHO "$YELLOW[-] afl is not compiled, cannot test"
+  INCOMPLETE=1
+ }
+} || {
+ $ECHO "$YELLOW[-] not an intel platform, cannot test afl-gcc"
+}
+
+source ./test-post.sh

test/test-compcov.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+source ./test-pre.sh
+
+test -z "$AFL_CC" && unset AFL_CC
+
+$ECHO "$BLUE[*] Testing: shared library extensions"
+cc $CFLAGS -o test-compcov test-compcov.c > /dev/null 2>&1
+test -e ../libtokencap.so && {
+  AFL_TOKEN_FILE=token.out LD_PRELOAD=../libtokencap.so DYLD_INSERT_LIBRARIES=../libtokencap.so DYLD_FORCE_FLAT_NAMESPACE=1 ./test-compcov foobar > /dev/null 2>&1
+  grep -q BUGMENOT token.out > /dev/null 2>&1 && {
+    $ECHO "$GREEN[+] libtokencap did successfully capture tokens"
+  } || {
+    $ECHO "$RED[!] libtokencap did not capture tokens"
+    CODE=1
+  }
+  rm -f token.out
+} || {
+  $ECHO "$YELLOW[-] libtokencap is not compiled, cannot test"
+  INCOMPLETE=1
+}
+test -e ../libdislocator.so && {
+  {
+    ulimit -c 1
+    # DYLD_INSERT_LIBRARIES and DYLD_FORCE_FLAT_NAMESPACE is used on Darwin/MacOSX
+    LD_PRELOAD=../libdislocator.so DYLD_INSERT_LIBRARIES=../libdislocator.so DYLD_FORCE_FLAT_NAMESPACE=1 ./test-compcov BUFFEROVERFLOW > test.out 2>/dev/null
+  } > /dev/null 2>&1
+  grep -q BUFFEROVERFLOW test.out > /dev/null 2>&1 && {
+    $ECHO "$RED[!] libdislocator did not detect the memory corruption"
+    CODE=1
+  } || {
+    $ECHO "$GREEN[+] libdislocator did successfully detect the memory corruption"
+  }
+  rm -f test.out core test-compcov.core core.test-compcov
+} || {
+  $ECHO "$YELLOW[-] libdislocator is not compiled, cannot test"
+  INCOMPLETE=1
+}
+rm -f test-compcov
+
+test -z "$AFL_CC" && {
+  if type gcc >/dev/null; then
+    export AFL_CC=gcc
+  else
+    if type clang >/dev/null; then
+      export AFL_CC=clang
+    fi
+  fi
+}
+
+source ./test-post.sh

test/test-custom-mutators.sh

@@ -0,0 +1,125 @@
+#!/bin/sh
+
+source ./test-pre.sh
+
+$ECHO "$BLUE[*] Testing: custom mutator"
+test "1" = "`../afl-fuzz | grep -i 'without python' >/dev/null; echo $?`" && {
+  # normalize path
+  CUSTOM_MUTATOR_PATH=$(cd $(pwd)/../examples/custom_mutators;pwd)
+  test -e test-custom-mutator.c -a -e ${CUSTOM_MUTATOR_PATH}/example.c -a -e ${CUSTOM_MUTATOR_PATH}/example.py && {
+    unset AFL_CC
+    # Compile the vulnerable program for single mutator
+    test -e ../afl-clang-fast && {
+      ../afl-clang-fast -o test-custom-mutator test-custom-mutator.c > /dev/null 2>&1
+    } || {
+      test -e ../afl-gcc-fast && {
+        ../afl-gcc-fast -o test-custom-mutator test-custom-mutator.c > /dev/null 2>&1
+      } || {
+        ../afl-gcc -o test-custom-mutator test-custom-mutator.c > /dev/null 2>&1
+      }
+    }
+    # Compile the vulnerable program for multiple mutators
+    test -e ../afl-clang-fast && {
+      ../afl-clang-fast -o test-multiple-mutators test-multiple-mutators.c > /dev/null 2>&1
+    } || {
+      test -e ../afl-gcc-fast && {
+        ../afl-gcc-fast -o test-multiple-mutators test-multiple-mutators.c > /dev/null 2>&1
+      } || {
+        ../afl-gcc -o test-multiple-mutators test-multiple-mutators.c > /dev/null 2>&1
+      }
+    }
+    # Compile the custom mutator
+    cc -D_FIXED_CHAR=0x41 -g -fPIC -shared -I../include ../examples/custom_mutators/simple_example.c -o libexamplemutator.so > /dev/null 2>&1
+    cc -D_FIXED_CHAR=0x42 -g -fPIC -shared -I../include ../examples/custom_mutators/simple_example.c -o libexamplemutator2.so > /dev/null 2>&1
+    test -e test-custom-mutator -a -e ./libexamplemutator.so && {
+      # Create input directory
+      mkdir -p in
+      echo "00000" > in/in
+
+      # Run afl-fuzz w/ the C mutator
+      $ECHO "$GREY[*] running afl-fuzz for the C mutator, this will take approx 5 seconds"
+      {
+        AFL_CUSTOM_MUTATOR_LIBRARY=./libexamplemutator.so AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V1 -m ${MEM_LIMIT} -i in -o out -- ./test-custom-mutator >>errors 2>&1
+      } >>errors 2>&1
+
+      # Check results
+      test -n "$( ls out/crashes/id:000000* 2>/dev/null )" && {  # TODO: update here
+        $ECHO "$GREEN[+] afl-fuzz is working correctly with the C mutator"
+      } || {
+        echo CUT------------------------------------------------------------------CUT
+        cat errors
+        echo CUT------------------------------------------------------------------CUT
+        $ECHO "$RED[!] afl-fuzz is not working correctly with the C mutator"
+        CODE=1
+      }
+
+      # Clean
+      rm -rf out errors
+
+      # Run afl-fuzz w/ multiple C mutators
+      $ECHO "$GREY[*] running afl-fuzz with multiple custom C mutators, this will take approx 5 seconds"
+      {
+        AFL_CUSTOM_MUTATOR_LIBRARY="./libexamplemutator.so;./libexamplemutator2.so" AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V1 -m ${MEM_LIMIT} -i in -o out -- ./test-multiple-mutators >>errors 2>&1
+      } >>errors 2>&1
+
+      test -n "$( ls out/crashes/id:000000* 2>/dev/null )" && {  # TODO: update here
+        $ECHO "$GREEN[+] afl-fuzz is working correctly with multiple C mutators"
+      } || {
+        echo CUT------------------------------------------------------------------CUT
+        cat errors
+        echo CUT------------------------------------------------------------------CUT
+        $ECHO "$RED[!] afl-fuzz is not working correctly with multiple C mutators"
+        CODE=1
+      }
+
+      # Clean
+      rm -rf out errors 
+
+      # Run afl-fuzz w/ the Python mutator
+      $ECHO "$GREY[*] running afl-fuzz for the Python mutator, this will take approx 5 seconds"
+      {
+        export PYTHONPATH=${CUSTOM_MUTATOR_PATH}
+        export AFL_PYTHON_MODULE=example
+        AFL_CUSTOM_MUTATOR_ONLY=1 ../afl-fuzz -V5 -m ${MEM_LIMIT} -i in -o out -- ./test-custom-mutator >>errors 2>&1
+        unset PYTHONPATH
+        unset AFL_PYTHON_MODULE
+      } >>errors 2>&1
+
+      # Check results
+      test -n "$( ls out/crashes/id:000000* 2>/dev/null )" && {  # TODO: update here
+        $ECHO "$GREEN[+] afl-fuzz is working correctly with the Python mutator"
+      } || {
+        echo CUT------------------------------------------------------------------CUT
+        cat errors
+        echo CUT------------------------------------------------------------------CUT
+        $ECHO "$RED[!] afl-fuzz is not working correctly with the Python mutator"
+        CODE=1
+      }
+
+      # Clean
+      rm -rf in out errors
+      rm -rf ${CUSTOM_MUTATOR_PATH}/__pycache__/
+      rm -f test-multiple-mutators test-custom-mutator libexamplemutator.so libexamplemutator2.so
+    } || {
+      ls .
+      ls ${CUSTOM_MUTATOR_PATH}
+      $ECHO "$RED[!] cannot compile the test program or the custom mutator"
+      CODE=1
+    }
+
+    #test "$CODE" = 1 && { $ECHO "$YELLOW[!] custom mutator tests currently will not fail travis" ; CODE=0 ; }
+
+    make -C ../examples/custom_mutators clean > /dev/null 2>&1
+    rm -f test-custom-mutator
+    rm -f test-custom-mutators
+  } || {
+    $ECHO "$YELLOW[-] no custom mutators in $CUSTOM_MUTATOR_PATH, cannot test"
+    INCOMPLETE=1
+  }
+  unset CUSTOM_MUTATOR_PATH
+} || {
+  $ECHO "$YELLOW[-] no python support in afl-fuzz, cannot test"
+  INCOMPLETE=1
+}
+
+source ./test-post.sh