cmplog routines instrumentation for qemu mode on x86

Author: andreafioraldi

qemu_mode/patches/afl-qemu-tcg-runtime-inl.h

@@ -158,3 +158,62 @@ void HELPER(afl_cmplog_64)(target_ulong cur_loc, target_ulong arg1,
 
 }
 
+#include <sys/mman.h>
+
+static int area_is_mapped(void* ptr, size_t len) {
+
+  char* p = ptr;
+  char* page = (char*)((uintptr_t)p & ~(sysconf(_SC_PAGE_SIZE) - 1));
+
+  int r = msync(page, (p - page) + len, MS_ASYNC);
+  if (r < 0) return errno != ENOMEM;
+  return 1;
+
+}
+
+void HELPER(afl_cmplog_rtn)(CPUX86State *env) {
+
+#if defined(TARGET_X86_64)
+
+  void* ptr1 = g2h(env->regs[R_EDI]);
+  void* ptr2 = g2h(env->regs[R_ESI]);
+
+#elif defined(TARGET_I386)
+
+  target_ulong* stack = g2h(env->regs[R_ESP]);
+  
+  if (!area_is_mapped(stack, sizeof(target_ulong)*2)) return;
+  
+  // when this hook is executed, the retaddr is not on stack yet
+  void* ptr1 = g2h(stack[0]);
+  void* ptr2 = g2h(stack[1]);
+
+#else
+
+  // dumb code to make it compile
+  void* ptr1 = NULL;
+  void* ptr2 = NULL;
+  return;
+
+#endif
+
+  if (!area_is_mapped(ptr1, 32) || !area_is_mapped(ptr2, 32)) return;
+
+  uintptr_t k = (uintptr_t)env->eip;
+  k = (k >> 4) ^ (k << 8);
+  k &= CMP_MAP_W - 1;
+
+  __afl_cmp_map->headers[k].type = CMP_TYPE_RTN;
+
+  u32 hits = __afl_cmp_map->headers[k].hits;
+  __afl_cmp_map->headers[k].hits = hits + 1;
+
+  __afl_cmp_map->headers[k].shape = 31;
+
+  hits &= CMP_MAP_RTN_H - 1;
+  __builtin_memcpy(((struct cmpfn_operands*)__afl_cmp_map->log[k])[hits].v0,
+                   ptr1, 32);
+  __builtin_memcpy(((struct cmpfn_operands*)__afl_cmp_map->log[k])[hits].v1,
+                   ptr2, 32);
+
+}

qemu_mode/patches/i386-translate.diff

@@ -1,5 +1,5 @@
 diff --git a/target/i386/translate.c b/target/i386/translate.c
-index 0dd5fbe4..a23da128 100644
+index 0dd5fbe4..0d405fb6 100644
 --- a/target/i386/translate.c
 +++ b/target/i386/translate.c
 @@ -32,6 +32,8 @@
@@ -40,3 +40,23 @@ index 0dd5fbe4..a23da128 100644
   next_byte:
      b = x86_ldub_code(env, s);
      /* Collect prefixes.  */
+@@ -5056,6 +5063,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
+                 tcg_gen_ext16u_tl(s->T0, s->T0);
+             }
+             next_eip = s->pc - s->cs_base;
++            if (__afl_cmp_map && next_eip >= afl_start_code &&
++                next_eip < afl_end_code)
++              gen_helper_afl_cmplog_rtn(cpu_env);
+             tcg_gen_movi_tl(s->T1, next_eip);
+             gen_push_v(s, s->T1);
+             gen_op_jmp_v(s->T0);
+@@ -6544,6 +6554,9 @@ static target_ulong disas_insn(DisasContext *s, CPUState *cpu)
+                 tval = (int16_t)insn_get(env, s, MO_16);
+             }
+             next_eip = s->pc - s->cs_base;
++            if (__afl_cmp_map && next_eip >= afl_start_code &&
++                next_eip < afl_end_code)
++              gen_helper_afl_cmplog_rtn(cpu_env);
+             tval += next_eip;
+             if (dflag == MO_16) {
+                 tval &= 0xffff;

qemu_mode/patches/tcg-runtime-head.diff

@@ -1,8 +1,8 @@
 diff --git a/accel/tcg/tcg-runtime.h b/accel/tcg/tcg-runtime.h
-index 1bd39d13..c58dee31 100644
+index 1bd39d13..81ef3973 100644
 --- a/accel/tcg/tcg-runtime.h
 +++ b/accel/tcg/tcg-runtime.h
-@@ -260,3 +260,12 @@ DEF_HELPER_FLAGS_4(gvec_leu8, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
+@@ -260,3 +260,13 @@ DEF_HELPER_FLAGS_4(gvec_leu8, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
  DEF_HELPER_FLAGS_4(gvec_leu16, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
  DEF_HELPER_FLAGS_4(gvec_leu32, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
  DEF_HELPER_FLAGS_4(gvec_leu64, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
@@ -15,3 +15,4 @@ index 1bd39d13..c58dee31 100644
 +DEF_HELPER_FLAGS_3(afl_cmplog_16, TCG_CALL_NO_RWG, void, tl, tl, tl)
 +DEF_HELPER_FLAGS_3(afl_cmplog_32, TCG_CALL_NO_RWG, void, tl, tl, tl)
 +DEF_HELPER_FLAGS_3(afl_cmplog_64, TCG_CALL_NO_RWG, void, tl, tl, tl)
++DEF_HELPER_FLAGS_1(afl_cmplog_rtn, TCG_CALL_NO_RWG, void, env)