VER: Release 0.32.0

Author: threecgreen

CHANGELOG.md

@@ -1,5 +1,29 @@
 # Changelog
 
+## 0.32.0 - 2025-04-14
+
+### Enhancements
+- Refactored the `rtype_dispatch` macro to be more flexible: it supports `ts_out`
+  dispatching, async functions and methods, and now always verifies the record length
+- Refactored the `schema_dispatch` macro to be more flexible: it supports `ts_out`
+  dispatching and async functions and methods
+
+### Breaking changes
+- Updated the `rtype_dispatch` and `schema_dispatch` macro invocations to look more like
+  function invocation
+
+### Deprecations
+- Deprecated macros `rtype_async_dispatch`, `rtype_ts_out_dispatch`,
+  `rtype_dispatch_with_ts_out`, `rtype_ts_out_async_dispatch`,
+  `rtype_ts_out_method_dispatch`, and `rtype_ts_out_async_method_dispatch` in favor of
+  an updated, more flexible `rtype_dispatch` macro
+
+### Bug fixes
+- Fixed `RType` variant names in Python to match `Schema`
+- Added missing Python type declarations for `RType` variants
+- Fixed issue with Python `_hidden_fields` definition that caused `KeyError: _reserved1_00`
+  with `CMBP1Msg` and other records with `ConsolidatedBidAskPair`
+
 ## 0.31.0 - 2025-04-01
 
 ### Enhancements

Cargo.lock

@@ -11,7 +11,7 @@ resolver = "2"
 [workspace.package]
 authors = ["Databento <[email protected]>"]
 edition = "2021"
-version = "0.31.0"
+version = "0.32.0"
 documentation = "https://databento.com/docs"
 repository = "https://github.com/databento/dbn"
 license = "Apache-2.0"

Cargo.toml

@@ -6,7 +6,7 @@ use std::{
 
 use dbn::{
     encode::{csv, json, DbnEncodable, EncodeRecordRef},
-    rtype, rtype_ts_out_dispatch, RecordHeader, RecordRef, Schema,
+    rtype, rtype_dispatch, RecordHeader, RecordRef, Schema,
 };
 
 use crate::cfile::CFileRef;
@@ -80,7 +80,7 @@ pub unsafe extern "C" fn s_serialize_record_header(
             .use_pretty_ts(options.pretty_ts)
             .build()
             .and_then(|mut encoder| {
-                rtype_ts_out_dispatch!(record, options.ts_out, serialize_csv_header, &mut encoder)
+                rtype_dispatch!(record, ts_out: options.ts_out, serialize_csv_header(&mut encoder))
             }),
     }
     // flatten
@@ -126,7 +126,7 @@ pub unsafe extern "C" fn f_serialize_record_header(
             .use_pretty_ts(options.pretty_ts)
             .build()
             .and_then(|mut encoder| {
-                rtype_ts_out_dispatch!(record, options.ts_out, serialize_csv_header, &mut encoder)
+                rtype_dispatch!(record, ts_out: options.ts_out, serialize_csv_header(&mut encoder))
             }),
     };
     res.map(|_| cfile.bytes_written() as i32)

c/src/text_serialization.rs

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databento-dbn"
-version = "0.31.0"
+version = "0.32.0"
 description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)"
 authors = ["Databento <[email protected]>"]
 license = "Apache-2.0"
@@ -17,7 +17,7 @@ build-backend = "maturin"
 
 [project]
 name = "databento-dbn"
-version = "0.31.0"
+version = "0.32.0"
 authors = [
     { name = "Databento", email = "[email protected]" }
 ]

python/pyproject.toml

@@ -1,4 +1,4 @@
-# ruff: noqa: UP007 PYI021 PYI053 PYI011
+# ruff: noqa: UP007 PYI021 PYI011
 from __future__ import annotations
 
 import datetime as dt
@@ -269,12 +269,12 @@ class RType(Enum):
     """
     A DBN record type.
 
-    MBP0
+    MBP_0
         Denotes a market-by-price record with a book depth of 0 (used for the `Trades` schema).
-    MBP1
+    MBP_1
         Denotes a market-by-price record with a book depth of 1 (also used for the
         `Tbbo` schema).
-    MBP10
+    MBP_10
         Denotes a market-by-price record with a book depth of 10.
     OHLCV_DEPRECATED
         Denotes an open, high, low, close, and volume record at an unspecified cadence.
@@ -306,7 +306,7 @@ class RType(Enum):
         Denotes a statistics record from the publisher (not calculated by Databento).
     MBO
         Denotes a market by order record.
-    CBBO
+    CMBP_1
         Denotes a consolidated best bid and offer record.
     CBBO_1S
         Denotes a consolidated best bid and offer record.
@@ -323,6 +323,30 @@ class RType(Enum):
 
     """  # noqa: D405, D411
 
+    MBP_0: int
+    MBP_1: int
+    MBP_10: int
+    OHLCV_DEPRECATED: int
+    OHLCV_1S: int
+    OHLCV_1M: int
+    OHLCV_1H: int
+    OHLCV_1D: int
+    OHLCV_EOD: int
+    STATUS: int
+    INSTRUMENT_DEF: int
+    IMBALANCE: int
+    ERROR: int
+    SYMBOL_MAPPING: int
+    SYSTEM: int
+    STATISTICS: int
+    MBO: int
+    CMBP_1: int
+    CBBO_1S: int
+    CBBO_1M: int
+    TCBBO: int
+    BBO_1S: int
+    BBO_1M: int
+
     @classmethod
     def from_int(cls, value: int) -> RType: ...
     @classmethod

python/python/databento_dbn/_lib.pyi

@@ -5,7 +5,7 @@ use pyo3::{prelude::*, IntoPyObjectExt};
 use dbn::{
     decode::dbn::{MetadataDecoder, RecordDecoder},
     python::to_py_err,
-    rtype_ts_out_dispatch, HasRType, VersionUpgradePolicy,
+    rtype_dispatch, HasRType, VersionUpgradePolicy,
 };
 
 #[pyclass(module = "databento_dbn", name = "DBNDecoder")]
@@ -96,7 +96,7 @@ impl DbnDecoder {
 
                 // Safety: It's safe to cast to `WithTsOut` because we're passing in the `ts_out`
                 // from the metadata header.
-                unsafe { rtype_ts_out_dispatch!(rec, self.ts_out, push_rec, py, &mut recs) }?;
+                rtype_dispatch!(rec, ts_out: self.ts_out, push_rec(py, &mut recs))?;
                 // keep track of position after last _successful_ decoding to
                 // ensure buffer is left in correct state in the case where one
                 // or more successful decodings is followed by a partial one, i.e.

python/src/dbn_decoder.rs

@@ -16,7 +16,7 @@ name = "dbn"
 path = "src/main.rs"
 
 [dependencies]
-dbn = { path = "../dbn", version = "=0.31.0", default-features = false }
+dbn = { path = "../dbn", version = "=0.32.0", default-features = false }
 
 anyhow = { workspace = true }
 clap = { version = "4.5", features = ["derive", "wrap_help"] }

rust/dbn-cli/Cargo.toml

@@ -52,6 +52,16 @@ By default, `dbn` will not overwrite an existing file.
 To replace the contents of an existing file and allow overwriting files, pass
 the `-f` or `--force` flag.
 
+### Merging DBN files
+You can use `dbn` to merge several DBN files into one, including files of different schemas.
+```sh
+# Merge files split by symbols
+dbn aapl.trades.dbn tsla.trades.dbn nvda.trades.dbn -o equities.trades.dbn
+# Or by date
+dbn equs-mini-20250331.dbn equs-mini-20250401.dbn equs-mini-20250402.dbn -o equs-mini-2025W14.dbn
+```
+The only limitation is they must be from the same dataset.
+
 ### Compressing the output
 In addition to reading Zstandard-compressed files, `dbn` can also write compressed JSON and CSV.