in_tail: Add Unicode encoder support

From UTF-16LE, UTF-16BE and UTF-16LE with BOM, UTF-16BE with BOM to
UTF-8 are supported.
This could be useful for Windows' Unicode insisted logs.
They are usually using UTF-16LE with BOM.

Signed-off-by: Hiroshi Hatake <[email protected]>

Author: cosmo0920

plugins/in_tail/tail.c

@@ -804,6 +804,14 @@ static struct flb_config_map config_map[] = {
     },
 #endif
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+    {
+     FLB_CONFIG_MAP_STR, "unicode.encoding", NULL,
+     0, FLB_FALSE, 0,
+     "specify the preferred input encoding for converting to UTF-8. "
+     "Currently, UTF-16LE, UTF-16BE, auto are supported.",
+    },
+#endif
     /* EOF */
     {0}
 };

plugins/in_tail/tail_config.c

@@ -36,6 +36,10 @@
 #include "tail_multiline.h"
 #endif
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+#include <fluent-bit/simdutf/flb_simdutf_connector.h>
+#endif
+
 static int multiline_load_parsers(struct flb_tail_config *ctx)
 {
     struct mk_list *head;
@@ -95,6 +99,9 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
 #ifdef FLB_HAVE_SQLDB
     ctx->db_sync = 1;  /* sqlite sync 'normal' */
 #endif
+#ifdef FLB_HAVE_UNICODE_ENCODER
+    ctx->preferred_input_encoding = FLB_SIMDUTF_ENCODING_TYPE_UNSPECIFIED;
+#endif
 
     /* Load the config map */
     ret = flb_input_config_map_set(ins, (void *) ctx);
@@ -178,6 +185,24 @@ struct flb_tail_config *flb_tail_config_create(struct flb_input_instance *ins,
         return NULL;
     }
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+    tmp = flb_input_get_property("unicode.encoding", ins);
+    if (tmp) {
+        if (strcasecmp(tmp, "auto") == 0) {
+            ctx->preferred_input_encoding = FLB_SIMDUTF_ENCODING_TYPE_UNICODE_AUTO;
+        }
+        else if (strcasecmp(tmp, "utf-16le") == 0) {
+            ctx->preferred_input_encoding = FLB_SIMDUTF_ENCODING_TYPE_UTF16_LE;
+        }
+        else if (strcasecmp(tmp, "utf-16be") == 0) {
+            ctx->preferred_input_encoding = FLB_SIMDUTF_ENCODING_TYPE_UTF16_BE;
+        }
+        else {
+            flb_plg_error(ctx->ins, "invalid encoding 'unicode.encoding' value");
+        }
+    }
+#endif
+
 #ifdef FLB_HAVE_PARSER
     /* Config: multi-line support */
     if (ctx->multiline == FLB_TRUE) {

plugins/in_tail/tail_config.h

@@ -122,6 +122,10 @@ struct flb_tail_config {
     /* Parser / Format */
     struct flb_parser *parser;
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+    int preferred_input_encoding;
+#endif
+
     /* Multiline */
     int multiline;             /* multiline enabled ?  */
     int multiline_flush;       /* multiline flush/wait */

plugins/in_tail/tail_file.c

@@ -48,6 +48,10 @@
 #include "win32.h"
 #endif
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+#include <fluent-bit/simdutf/flb_simdutf_connector.h>
+#endif
+
 #include <cfl/cfl.h>
 
 static inline void consume_bytes(char *buf, int bytes, int length)
@@ -440,6 +444,10 @@ static int process_content(struct flb_tail_file *file, size_t *bytes)
     time_t now = time(NULL);
     struct flb_time out_time = {0};
     struct flb_tail_config *ctx;
+#ifdef FLB_HAVE_UNICODE_ENCODER
+    char *decoded = NULL;
+    size_t decoded_len;
+#endif
 
     ctx = (struct flb_tail_config *) file->config;
 
@@ -509,6 +517,24 @@ static int process_content(struct flb_tail_file *file, size_t *bytes)
         line_len = len - crlf;
         repl_line = NULL;
 
+#ifdef FLB_HAVE_UNICODE_ENCODER
+        if (ctx->preferred_input_encoding != FLB_SIMDUTF_ENCODING_TYPE_UNSPECIFIED) {
+            decoded = NULL;
+            ret = flb_simdutf_connector_convert_from_unicode(ctx->preferred_input_encoding,
+                                                             line, line_len, &decoded, &decoded_len);
+            if (ret == FLB_SIMDUTF_CONNECTOR_CONVERT_OK) {
+                line = decoded;
+                line_len  = decoded_len;
+            } else if (ret == FLB_SIMDUTF_CONNECTOR_CONVERT_NOP) {
+                flb_plg_debug(ctx->ins, "nothing to convert encoding '%.*s'", line_len, line);
+            }
+            else {
+                flb_plg_error(ctx->ins, "encoding failed '%.*s'", line_len, line);
+                goto go_next;
+            }
+        }
+#endif
+
         if (ctx->ml_ctx) {
             ret = flb_ml_append_text(ctx->ml_ctx,
                                      file->ml_stream_id,
@@ -601,6 +627,12 @@ static int process_content(struct flb_tail_file *file, size_t *bytes)
         lines++;
         file->parsed = 0;
         file->last_processed_bytes += processed_bytes;
+#ifdef FLB_HAVE_UNICODE_ENCODER
+        if (decoded) {
+            flb_free(decoded);
+            decoded = NULL;
+        }
+#endif
     }
     file->parsed = file->buf_len;