feat: implement new features introduced in Substrait 0.7.0 (#26)

* feat: implement new type parameter metatypes introduced by Substrait 0.7.0
* feat: implement type parameters for user-defined types in type system
* feat: implement new messages for compound extension types added in Substrait 0.7.0

BREAKING CHANGE: the Parameter enum has new values, and the Unsigned(u64) variant was replaced with Integer(i64).

Author: jvanstraten

proto/substrait/validator/validator.proto

@@ -4,6 +4,7 @@ syntax = "proto3";
 package substrait.validator;
 
 import "google/protobuf/any.proto";
+import "google/protobuf/empty.proto";
 
 option csharp_namespace = "Substrait.Validator.Protobuf";
 option java_multiple_files = true;
@@ -50,7 +51,7 @@ message Node {
 
     // This node is inserted as a placeholder when a required oneof field was
     // not populated in the input.
-    Empty proto_missing_oneof = 3;
+    google.protobuf.Empty proto_missing_oneof = 3;
 
     // Special case of proto_primitive for references to anchors defined
     // elsewhere in the plan.
@@ -63,11 +64,11 @@ message Node {
 
     // This node represents a YAML map/object. The keys are represented using
     // Field messages in data.
-    Empty yaml_map = 6;
+    google.protobuf.Empty yaml_map = 6;
 
     // This node represents a YAML array. The elements are represented using
     // ArrayElement messages in data.
-    Empty yaml_array = 7;
+    google.protobuf.Empty yaml_array = 7;
 
     // This node represents a YAML primitive.
     PrimitiveData yaml_primitive = 8;
@@ -299,20 +300,20 @@ message Comment {
       Span span = 1;
 
       // A newline, i.e. the next span should start on the next line.
-      Empty new_line = 2;
+      google.protobuf.Empty new_line = 2;
 
       // Opens a new unordered list. The next span is the start of the text for
       // the next item. list_next elements are used to advance to the next list
       // item; newlines can be used to add paragraphs without bullet points.
       // Each list_open should be matched with a list_close. Lists may be
       // nested.
-      Empty list_open = 3;
+      google.protobuf.Empty list_open = 3;
 
       // Advances to the next list item.
-      Empty list_next = 4;
+      google.protobuf.Empty list_next = 4;
 
       // Closes the current list.
-      Empty list_close = 5;
+      google.protobuf.Empty list_close = 5;
     }
   }
 
@@ -343,7 +344,7 @@ message DataType {
   // Type variation, if any.
   oneof variation {
     UserDefinedVariation user_defined_variation = 9;
-    Empty unresolved_variation = 15;
+    google.protobuf.Empty unresolved_variation = 15;
   }
 
   // Type parameters for non-simple types.
@@ -355,7 +356,7 @@ message DataType {
       Simple simple = 1;
       Compound compound = 2;
       UserDefinedType user_defined_type = 3;
-      Empty unresolved_type = 7;
+      google.protobuf.Empty unresolved_type = 7;
     }
   }
 
@@ -445,7 +446,7 @@ message DataType {
       oneof base_type {
         Class physical = 1;
         UserDefinedVariation logical = 2;
-        Empty unresolved = 7;
+        google.protobuf.Empty unresolved = 7;
       }
 
       // Function behavior for this type variation.
@@ -470,8 +471,26 @@ message DataType {
       // Named data type parameter, for example N:T in NSTRUCT<N:T>.
       Named named_type = 2;
 
-      // Unsigned integer parameter, for example the L in VARCHAR<L>.
-      uint64 unsigned = 3;
+      // Boolean paramater, used only for extension types.
+      bool boolean = 4;
+
+      // Integer parameter, for example the L in VARCHAR<L>.
+      int64 integer = 5;
+
+      // Enumeration parameter, used only for extension types.
+      string enumeration = 6;
+
+      // String parameter, used only for extension types.
+      string string = 7;
+
+      // Used to skip optional parameters.
+      google.protobuf.Empty null = 8;
+
+      // Used when the value of a parameter could not be resolved.
+      google.protobuf.Empty unresolved = 9;
+
+      // Replaced by `integer`.
+      uint64 unsigned = 3 [deprecated = true];
     }
   }
 
@@ -484,6 +503,3 @@ message DataType {
     DataType data_type = 2;
   }
 }
-
-// Used for oneof field variants that have no data associated with them.
-message Empty {}

rs/src/export/html/mod.rs

@@ -9,6 +9,7 @@ use crate::output::diagnostic;
 use crate::output::parse_result;
 use crate::output::path;
 use crate::output::tree;
+use crate::util;
 use std::sync::Arc;
 
 const HEADER1: &str = concat!(
@@ -463,15 +464,32 @@ fn format_data_type(prefix: &str, data_type: &Arc<data_type::DataType>) -> Vec<S
                 .parameter_name(index)
                 .unwrap_or_else(|| "?".to_string());
             match parameter {
+                data_type::Parameter::Unresolved => {
+                    html.push(format_data_type_card(&format!(".{name}: !")))
+                }
+                data_type::Parameter::Null => {
+                    html.push(format_data_type_card(&format!(".{name}: null")))
+                }
+                data_type::Parameter::Boolean(b) => {
+                    html.push(format_data_type_card(&format!(".{name}: {b}")))
+                }
+                data_type::Parameter::Integer(i) => {
+                    html.push(format_data_type_card(&format!(".{name}: {i}")))
+                }
+                data_type::Parameter::Enum(e) => html.push(format_data_type_card(&format!(
+                    ".{name}: {}",
+                    util::string::as_ident_or_string(e)
+                ))),
+                data_type::Parameter::String(s) => html.push(format_data_type_card(&format!(
+                    ".{name}: {}",
+                    util::string::as_quoted_string(s)
+                ))),
                 data_type::Parameter::Type(t) => {
                     html.extend(format_data_type(&format!(".{name}"), t))
                 }
                 data_type::Parameter::NamedType(n, t) => {
                     html.extend(format_data_type(&format!(".{n}"), t))
                 }
-                data_type::Parameter::Unsigned(i) => {
-                    html.push(format_data_type_card(&format!(".{name}: {i}")))
-                }
             }
         }
         html.push("</details>".to_string());

rs/src/export/proto.rs

@@ -133,18 +133,10 @@ impl From<&comment::Element> for validator::comment::Element {
                 comment::Element::Span(span) => {
                     validator::comment::element::Kind::Span(span.into())
                 }
-                comment::Element::NewLine => {
-                    validator::comment::element::Kind::NewLine(validator::Empty {})
-                }
-                comment::Element::ListOpen => {
-                    validator::comment::element::Kind::ListOpen(validator::Empty {})
-                }
-                comment::Element::ListNext => {
-                    validator::comment::element::Kind::ListNext(validator::Empty {})
-                }
-                comment::Element::ListClose => {
-                    validator::comment::element::Kind::ListClose(validator::Empty {})
-                }
+                comment::Element::NewLine => validator::comment::element::Kind::NewLine(()),
+                comment::Element::ListOpen => validator::comment::element::Kind::ListOpen(()),
+                comment::Element::ListNext => validator::comment::element::Kind::ListNext(()),
+                comment::Element::ListClose => validator::comment::element::Kind::ListClose(()),
             }),
         }
     }
@@ -182,9 +174,7 @@ impl From<&tree::NodeType> for validator::node::NodeType {
                     data: Some(data.into()),
                 })
             }
-            tree::NodeType::ProtoMissingOneOf => {
-                validator::node::NodeType::ProtoMissingOneof(validator::Empty::default())
-            }
+            tree::NodeType::ProtoMissingOneOf => validator::node::NodeType::ProtoMissingOneof(()),
             tree::NodeType::NodeReference(anchor, node) => {
                 validator::node::NodeType::NodeReference(validator::node::NodeReference {
                     value: *anchor,
@@ -196,12 +186,8 @@ impl From<&tree::NodeType> for validator::node::NodeType {
                     uri: info.uri.name().unwrap_or_default().to_string(),
                 })
             }
-            tree::NodeType::YamlMap => {
-                validator::node::NodeType::YamlMap(validator::Empty::default())
-            }
-            tree::NodeType::YamlArray => {
-                validator::node::NodeType::YamlArray(validator::Empty::default())
-            }
+            tree::NodeType::YamlMap => validator::node::NodeType::YamlMap(()),
+            tree::NodeType::YamlArray => validator::node::NodeType::YamlArray(()),
             tree::NodeType::YamlPrimitive(data) => {
                 validator::node::NodeType::YamlPrimitive(data.into())
             }
@@ -308,7 +294,7 @@ impl From<&data_type::Class> for validator::data_type::Class {
                     validator::data_type::class::Kind::UserDefinedType(user_defined.as_ref().into())
                 }
                 data_type::Class::Unresolved => {
-                    validator::data_type::class::Kind::UnresolvedType(validator::Empty {})
+                    validator::data_type::class::Kind::UnresolvedType(())
                 }
             }),
         }
@@ -393,7 +379,7 @@ impl From<&extension::Reference<extension::TypeVariation>> for validator::data_t
                 },
             )
         } else {
-            validator::data_type::Variation::UnresolvedVariation(validator::Empty {})
+            validator::data_type::Variation::UnresolvedVariation(())
         }
     }
 }
@@ -425,6 +411,22 @@ impl From<&data_type::Parameter> for validator::data_type::Parameter {
     fn from(node: &data_type::Parameter) -> Self {
         Self {
             kind: Some(match node {
+                data_type::Parameter::Unresolved => {
+                    validator::data_type::parameter::Kind::Unresolved(())
+                }
+                data_type::Parameter::Null => validator::data_type::parameter::Kind::Null(()),
+                data_type::Parameter::Boolean(x) => {
+                    validator::data_type::parameter::Kind::Boolean(*x)
+                }
+                data_type::Parameter::Integer(x) => {
+                    validator::data_type::parameter::Kind::Integer(*x)
+                }
+                data_type::Parameter::Enum(x) => {
+                    validator::data_type::parameter::Kind::Enumeration(x.clone())
+                }
+                data_type::Parameter::String(x) => {
+                    validator::data_type::parameter::Kind::String(x.clone())
+                }
                 data_type::Parameter::Type(data_type) => {
                     validator::data_type::parameter::Kind::DataType(data_type.as_ref().into())
                 }
@@ -434,9 +436,6 @@ impl From<&data_type::Parameter> for validator::data_type::Parameter {
                         data_type: Some(data_type.as_ref().into()),
                     })
                 }
-                data_type::Parameter::Unsigned(unsigned) => {
-                    validator::data_type::parameter::Kind::Unsigned(*unsigned)
-                }
             }),
         }
     }