new json

Author: ruslandoga

lib/ch/row_binary.ex

@@ -516,6 +516,7 @@ defmodule Ch.RowBinary do
               :uuid,
               :date,
               :date32,
+              :json,
               :ipv4,
               :ipv6,
               :point,
@@ -697,6 +698,20 @@ defmodule Ch.RowBinary do
   defp utf8_size(codepoint) when codepoint <= 0xFFFF, do: 3
   defp utf8_size(codepoint) when codepoint <= 0x10FFFF, do: 4
 
+  @compile inline: [decode_json_decode_rows: 5]
+
+  for {pattern, size} <- varints do
+    defp decode_json_decode_rows(
+           <<unquote(pattern), s::size(unquote(size))-bytes, bin::bytes>>,
+           types_rest,
+           row,
+           rows,
+           types
+         ) do
+      decode_rows(types_rest, bin, [Jason.decode!(s) | row], rows, types)
+    end
+  end
+
   @compile inline: [decode_binary_decode_rows: 5]
 
   for {pattern, size} <- varints do
@@ -865,6 +880,9 @@ defmodule Ch.RowBinary do
         <<d::32-little-signed, bin::bytes>> = bin
         decode_rows(types_rest, bin, [Date.add(@epoch_date, d) | row], rows, types)
 
+      :json ->
+        decode_json_decode_rows(bin, types_rest, row, rows, types)
+
       {:datetime, timezone} ->
         <<s::32-little, bin::bytes>> = bin
 

lib/ch/types.ex

@@ -26,6 +26,7 @@ defmodule Ch.Types do
       # {"DateTime", :datetime, []},
       {"Date32", :date32, []},
       {"Date", :date, []},
+      {"JSON", :json, []},
       {"LowCardinality", :low_cardinality, [:type]},
       for size <- [32, 64, 128, 256] do
         {"Decimal#{size}", :"decimal#{size}", [:int]}

test/ch/connection_test.exs

@@ -568,20 +568,86 @@ defmodule Ch.ConnectionTest do
               }} = Ch.query(conn, "SELECT * FROM t_uuid ORDER BY y")
     end
 
+    # TODO non utf8
+    test "read json as string", %{conn: conn} do
+      assert Ch.query!(conn, ~s|select '{"a":42}'::JSON|, [],
+               settings: [
+                 enable_json_type: 1,
+                 output_format_binary_write_json_as_string: 1
+               ]
+             ).rows == [[%{"a" => "42"}]]
+    end
+
+    test "write->read json as string", %{conn: conn} do
+      Ch.query!(conn, "CREATE TABLE test_write_json(json JSON) ENGINE = Memory", [],
+        settings: [
+          enable_json_type: 1
+        ]
+      )
+
+      rowbinary =
+        Ch.RowBinary.encode_rows(
+          [
+            [Jason.encode_to_iodata!(%{"a" => 42})],
+            [Jason.encode_to_iodata!(%{"b" => 10})]
+          ],
+          _types = [:string]
+        )
+
+      Ch.query!(conn, ["insert into test_write_json(json) format RowBinary\n" | rowbinary], [],
+        settings: [
+          enable_json_type: 1,
+          input_format_binary_read_json_as_string: 1
+        ]
+      )
+
+      assert Ch.query!(conn, "select json from test_write_json", [],
+               settings: [
+                 enable_json_type: 1,
+                 output_format_binary_write_json_as_string: 1
+               ]
+             ).rows ==
+               [[%{"a" => "42"}], [%{"b" => "10"}]]
+    end
+
+    # https://clickhouse.com/docs/en/sql-reference/data-types/newjson
+    # https://clickhouse.com/docs/en/integrations/data-formats/json/overview
+    # https://clickhouse.com/blog/a-new-powerful-json-data-type-for-clickhouse
+    # https://clickhouse.com/blog/json-bench-clickhouse-vs-mongodb-elasticsearch-duckdb-postgresql
+    # https://github.com/ClickHouse/ClickHouse/pull/70288
+    # https://github.com/ClickHouse/ClickHouse/blob/master/src/Core/TypeId.h
     @tag :skip
     test "json", %{conn: conn} do
-      settings = [allow_experimental_object_type: 1]
+      settings = [enable_json_type: 1]
 
-      Ch.query!(conn, "CREATE TABLE json(o JSON) ENGINE = Memory", [], settings: settings)
+      assert Ch.query!(
+               conn,
+               ~s|select '{"a":42,"b":10}'::JSON|,
+               [],
+               settings: settings,
+               decode: false,
+               format: "RowBinary"
+             ).rows == [
+               <<2, 1, 97, 10, 42, 0, 0, 0, 0, 0, 0, 0, 1, 98, 10, 10, 0, 0, 0, 0, 0, 0, 0>>
+             ]
 
-      Ch.query!(conn, ~s|INSERT INTO json VALUES ('{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}')|)
+      # Ch.query!(conn, "CREATE TABLE test_json(json JSON) ENGINE = Memory", [], settings: settings)
 
-      assert Ch.query!(conn, "SELECT o.a, o.b.c, o.b.d[3] FROM json").rows == [[1, 2, 3]]
+      # Ch.query!(
+      #   conn,
+      #   ~s|INSERT INTO test_json VALUES ('{"a" : {"b" : 42}, "c" : [1, 2, 3]}'), ('{"f" : "Hello, World!"}'), ('{"a" : {"b" : 43, "e" : 10}, "c" : [4, 5, 6]}')|
+      # )
 
-      # named tuples are not supported yet
-      assert_raise ArgumentError, fn -> Ch.query!(conn, "SELECT o FROM json") end
+      # assert Ch.query!(conn, "SELECT json FROM test_json") == :asdf
+
+      # assert Ch.query!(conn, "SELECT json.a, json.b.c, json.b.d[3] FROM test_json").rows == [
+      #          [1, 2, 3]
+      #        ]
     end
 
+    # TODO variant (is there?)
+    # TODO dynamic
+
     # TODO enum16
 
     test "enum8", %{conn: conn} do