Merge pull request AFLplusplus#352 from AFLplusplus/dev

Pull to master because of crash in string compare transform

Author: vanhauser-thc

GNUmakefile

@@ -307,7 +307,7 @@ test_python:
 else
 
 test_python:
-	@echo "[-] You seem to need to install the package python3-dev or python2-dev (and perhaps python[23]-apt), but it is optional so we continue"
+	@echo "[-] You seem to need to install the package python3-dev, python2-dev or python-dev (and perhaps python[23]-apt), but it is optional so we continue"
 
 endif
 
@@ -445,6 +445,7 @@ clean:
 	-$(MAKE) -C gcc_plugin clean
 	$(MAKE) -C libdislocator clean
 	$(MAKE) -C libtokencap clean
+	$(MAKE) -C examples/afl_network_proxy clean
 	$(MAKE) -C examples/socket_fuzzing clean
 	$(MAKE) -C examples/argv_fuzzing clean
 	$(MAKE) -C qemu_mode/unsigaction clean
@@ -468,6 +469,7 @@ distrib: all radamsa
 	-$(MAKE) -C gcc_plugin
 	$(MAKE) -C libdislocator
 	$(MAKE) -C libtokencap
+	$(MAKE) -C examples/afl_network_proxy
 	$(MAKE) -C examples/socket_fuzzing
 	$(MAKE) -C examples/argv_fuzzing
 	cd qemu_mode && sh ./build_qemu_support.sh
@@ -476,6 +478,7 @@ distrib: all radamsa
 binary-only: all radamsa
 	$(MAKE) -C libdislocator
 	$(MAKE) -C libtokencap
+	$(MAKE) -C examples/afl_network_proxy
 	$(MAKE) -C examples/socket_fuzzing
 	$(MAKE) -C examples/argv_fuzzing
 	cd qemu_mode && sh ./build_qemu_support.sh
@@ -486,6 +489,9 @@ source-only: all radamsa
 	-$(MAKE) -C gcc_plugin
 	$(MAKE) -C libdislocator
 	$(MAKE) -C libtokencap
+	#$(MAKE) -C examples/afl_network_proxy
+	#$(MAKE) -C examples/socket_fuzzing
+	#$(MAKE) -C examples/argv_fuzzing
 
 %.8:	%
 	@echo .TH $* 8 $(BUILD_DATE) "afl++" > $@
@@ -521,6 +527,7 @@ install: all $(MANPAGES)
 	if [ -f afl-fuzz-document ]; then set -e; install -m 755 afl-fuzz-document $${DESTDIR}$(BIN_PATH); fi
 	if [ -f socketfuzz32.so -o -f socketfuzz64.so ]; then $(MAKE) -C examples/socket_fuzzing install; fi
 	if [ -f argvfuzz32.so -o -f argvfuzz64.so ]; then $(MAKE) -C examples/argv_fuzzing install; fi
+	if [ -f examples/afl_network_proxy/afl-network-server ]; then $(MAKE) -C examples/afl_network_proxy install; fi
 
 	set -e; ln -sf afl-gcc $${DESTDIR}$(BIN_PATH)/afl-g++
 	set -e; if [ -f afl-clang-fast ] ; then ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang ; ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang++ ; else ln -sf afl-gcc $${DESTDIR}$(BIN_PATH)/afl-clang ; ln -sf afl-gcc $${DESTDIR}$(BIN_PATH)/afl-clang++; fi

TODO.md

@@ -1,9 +1,9 @@
 # TODO list for AFL++
 
-## Roadmap 2.65
+## Roadmap 2.65+
 
- - AFL_MAP_SIZE for afl-llvm-pass, qemu_mode and unicorn_mode
- - fix stability calculation bug
+ - InsTrim mode for LTO solution
+ - AFL_MAP_SIZE for qemu_mode and unicorn_mode
  - random crc32 HASH_CONST per run? because with 65536 paths we have collisions
  - namespace for targets? e.g. network
  - libradamsa as a custom module?
@@ -13,31 +13,22 @@
 afl-fuzz:
  - sync_fuzzers(): only masters sync from all, slaves only sync from master
    (@andrea: be careful, often people run all slaves)
- - ascii_only mode for mutation output
+ - ascii_only mode for mutation output - or use a custom mutator for this?
  - setting min_len/max_len/start_offset/end_offset limits for mutation output
 
 llvm_mode:
- - added context sensitive branch coverage
- - add CT cov and ngram cov to LTO and InsTrim
  - better whitelist solution for LTO
 
 gcc_plugin:
  - laf-intel
  - better instrumentation (seems to be better with gcc-9+)
 
 qemu_mode:
- - update to 4.x (probably this will be skipped :( )
+ - update to 5.x (if the performance bug if gone)
  - non colliding instrumentation
- - instrim for QEMU mode via static analysis (with r2pipe? or angr?)
-   Idea: The static analyzer outputs a map in which each edge that must be
-   skipped is marked with 1. QEMU loads it at startup in the parent process.
  - rename qemu specific envs to AFL_QEMU (AFL_ENTRYPOINT, AFL_CODE_START/END,
    AFL_COMPCOV_LEVEL?)
  - add AFL_QEMU_EXITPOINT (maybe multiple?), maybe pointless as we have
    persistent mode
  - add/implement AFL_QEMU_INST_LIBLIST and AFL_QEMU_NOINST_PROGRAM
  - add/implement AFL_QEMU_INST_REGIONS as a list of _START/_END addresses
-
-custom_mutators:
- - rip what Superion is doing into custom mutators for js, php, etc.
-

docs/Changelog.md

@@ -20,6 +20,12 @@ sending a mail to <[email protected]>.
        address for the shared memory map is used as this increases the
        fuzzing speed
      - fixes to LTO mode if instrumented edges > MAP_SIZE
+     - CTX and NGRAM can now be used together
+     - CTX and NGRAM are now also supported in CFG/INSTRIM mode
+     - AFL_LLVM_LAF_TRANSFORM_COMPARES could crash, fixed
+     - added AFL_LLVM_SKIP_NEVERZERO to skip the never zero coverage counter
+       implementation. For targets with few or no loops or heavily called
+       functions. Gives a small performance boost.
   - qemu_mode:
     - add information on PIE/PIC load addresses for 32 bit
     - better dependency checks
@@ -28,6 +34,13 @@ sending a mail to <[email protected]>.
   - unicorn_mode:
     - better submodule handling
   - afl-showmap: fix for -Q mode
+  - added examples/afl_network_proxy which allows to fuzz a target over the
+    network (not fuzzing tcp/ip services but running afl-fuzz on one system
+    and the target being on an embedded device)
+  - added examples/afl_untracer which does a binary-only fuzzing with the
+    modifications done in memory
+  - added examples/afl_proxy which can be easily used to fuzz and instrument
+    non-standard things
   - all:
     - forkserver communication now also used for error reporting
     - fix 32 bit build options

docs/env_variables.md

@@ -97,12 +97,15 @@ Then there are a few specific features that are only available in llvm_mode:
 
     - AFL_LLVM_INSTRUMENT - this configures the instrumentation mode. 
       Available options:
-        DEFAULT - classic AFL (map[cur_loc ^ prev_loc >> 1]++)
+        CLASSIC - classic AFL (map[cur_loc ^ prev_loc >> 1]++) (default)
         CFG - InsTrim instrumentation (see below)
         LTO - LTO instrumentation (see below)
         CTX - context sensitive instrumentation (see below)
         NGRAM-x - deeper previous location coverage (from NGRAM-2 up to NGRAM-16)
-      Only one can be used.
+      In CLASSIC (default) and CFG/INSTRIM you can also specify CTX and/or
+      NGRAM, seperate the options with a comma "," then, e.g.:
+        AFL_LLVM_INSTRUMENT=CFG,CTX,NGRAM-4
+      Not that this is a good idea to use both CTX and NGRAM :)
 
 ### LTO
 
@@ -204,6 +207,10 @@ Then there are a few specific features that are only available in llvm_mode:
       slowdown due a performance issue that is only fixed in llvm 9+.
       This feature increases path discovery by a little bit.
 
+    - Setting AFL_LLVM_SKIP_NEVERZERO=1 will not implement the skip zero
+      test. If the target performs only few loops then this will give a
+      small performance boost.
+
     See llvm_mode/README.neverzero.md
 
 ### CMPLOG

examples/afl_network_proxy/GNUmakefile

@@ -0,0 +1,41 @@
+PREFIX   ?= /usr/local
+BIN_PATH  = $(PREFIX)/bin
+DOC_PATH  = $(PREFIX)/share/doc/afl
+
+PROGRAMS = afl-network-client afl-network-server
+
+HASH=\#
+
+ifdef STATIC
+  CFLAGS += -static
+endif
+
+ifeq "$(shell echo '$(HASH)include <libdeflate.h>@int main() { struct libdeflate_compressor *d = libdeflate_alloc_compressor(1); return 0;}' | tr @ '\n' | $(CC) $(CFLAGS) -x c - -o .test2 -ldeflate 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1"
+ CFLAGS += -DUSE_DEFLATE=1
+ LDFLAGS += -ldeflate 
+ $(info libdeflate-dev was detected, using compression)
+else
+ $(warn did not find libdeflate-dev, cannot use compression)
+endif
+
+all:	$(PROGRAMS)
+
+help:
+	@echo make options:
+	@echo STATIC - build as static binaries
+	@echo COMPRESS_TESTCASES - compress test cases
+
+afl-network-client:	afl-network-client.c
+	$(CC) $(CFLAGS) -I../../include -o afl-network-client afl-network-client.c $(LDFLAGS)
+
+afl-network-server:	afl-network-server.c
+	$(CC) $(CFLAGS) -I../../include -o afl-network-server afl-network-server.c ../../src/afl-forkserver.c ../../src/afl-sharedmem.c ../../src/afl-common.c -DBIN_PATH=\"$(BIN_PATH)\" $(LDFLAGS)
+
+clean:
+	rm -f $(PROGRAMS) *~ core
+
+install: all
+	install -d -m 755 $${DESTDIR}$(BIN_PATH) $${DESTDIR}$(DOC_PATH)
+	install -m 755 $(PROGRAMS) $${DESTDIR}$(BIN_PATH)
+	install -T -m 644 README.md $${DESTDIR}$(DOC_PATH)/README.network_proxy.md
+

examples/afl_network_proxy/Makefile

@@ -0,0 +1,2 @@
+all:
+	@echo please use GNU make, thanks!

examples/afl_network_proxy/README.md

@@ -0,0 +1,61 @@
+# afl-network-proxy
+
+If you want to run afl-fuzz over the network than this is what you need :)
+Note that the impact on fuzzing speed will be huge, expect a loss of 90%.
+
+## When to use this
+
+1. when you have to fuzz a target that has to run on a system that cannot
+   contain the fuzzing output (e.g. /tmp too small and file system is read-only)
+2. when the target instantly reboots on crashes
+3. ... any other reason you would need this
+
+## how to get it running
+
+### Compiling
+
+Just type `make` and let the autodetection do everything for you.
+
+Note that you will get a 40-50% performance increase if you have libdeflate-dev
+installed. The GNUmakefile will autodetect it if present.
+
+If your target has large test cases (10+kb) that are ascii only or large chunks
+of zero blocks then set `CFLAGS=-DCOMPRESS_TESTCASES=1` to compress them.
+For most targets this hurts performance though so it is disabled by default.
+
+### on the target
+
+Run `afl-network-server` with your target with the -m and -t values you need.
+Important is the -i parameter which is the TCP port to listen on.
+e.g.:
+```
+$ afl-network-server -i 1111 -m 25M -t 1000 -- /bin/target -f @@
+```
+
+### on the (afl-fuzz) master
+
+Just run afl-fuzz with your normal options, however the target should be
+`afl-network-client` with the IP and PORT of the `afl-network-server` and
+increase the -t value:
+```
+$ afl-fuzz -i in -o out -t 2000+ -- afl-network-client TARGET-IP 1111
+```
+Note the '+' on the -t parameter value. The afl-network-server will take
+care of proper timeouts hence afl-fuzz should not. The '+' increases the
+timeout and the value itself should be 500-1000 higher than the one on 
+afl-network-server.
+
+### networking
+
+The TARGET can be an IPv4 or IPv6 address, or a host name that resolves to
+either. Note that also the outgoing interface can be specified with a '%' for
+`afl-network-client`, e.g. `fe80::1234%eth0`.
+
+Also make sure your default TCP window size is larger than your MAP_SIZE
+(130kb is a good value).
+On Linux that is the middle value of `/proc/sys/net/ipv4/tcp_rmem` 
+
+## how to compile and install
+
+`make && sudo make install`
+