VER: Release 0.36.2

Author: threecgreen

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.36.2 - 2025-07-08
+
+### Enhancements
+- Upgraded `async-compression` version to 0.4.25
+- Upgraded `pyo3` version to 0.25.1
+
+### Bug fixes
+- Fixed change in behavior where Python `DBNDecoder.decode()` wouldn't always decode all
+  available data on the first call
+
 ## 0.36.1 - 2025-06-17
 
 ### Bug fixes

Cargo.lock

@@ -11,16 +11,16 @@ resolver = "2"
 [workspace.package]
 authors = ["Databento <[email protected]>"]
 edition = "2021"
-version = "0.36.1"
+version = "0.36.2"
 documentation = "https://databento.com/docs"
 repository = "https://github.com/databento/dbn"
 license = "Apache-2.0"
 
 [workspace.dependencies]
 anyhow = "1.0.98"
 csv = "1.3"
-pyo3 = "0.25.0"
-pyo3-build-config = "0.25.0"
+pyo3 = "0.25.1"
+pyo3-build-config = "0.25.1"
 rstest = "0.25.0"
 serde = { version = "1.0", features = ["derive"] }
 time = "0.3.41"

Cargo.toml

@@ -16,7 +16,7 @@ crate-type = ["staticlib"]
 [dependencies]
 anyhow = { workspace = true }
 dbn = { path = "../rust/dbn", features = [] }
-libc = "0.2.172"
+libc = "0.2.174"
 
 [build-dependencies]
 cbindgen = { version = "0.29.0", default-features = false }

c/Cargo.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databento-dbn"
-version = "0.36.1"
+version = "0.36.2"
 description = "Python bindings for encoding and decoding Databento Binary Encoding (DBN)"
 authors = ["Databento <[email protected]>"]
 license = "Apache-2.0"
@@ -17,7 +17,7 @@ build-backend = "maturin"
 
 [project]
 name = "databento-dbn"
-version = "0.36.1"
+version = "0.36.2"
 authors = [
     { name = "Databento", email = "[email protected]" }
 ]

python/pyproject.toml

@@ -7,6 +7,7 @@ use dbn::{
 };
 
 #[pyclass(module = "databento_dbn", name = "DBNDecoder")]
+#[derive(Debug)]
 pub struct DbnDecoder {
     fsm: DbnFsm,
 }
@@ -32,11 +33,20 @@ impl DbnDecoder {
             .map_err(to_py_err)?
             .upgrade_policy(upgrade_policy)
             .skip_metadata(!has_metadata)
+            .compat_size(if upgrade_policy == VersionUpgradePolicy::AsIs {
+                0
+            } else {
+                DbnFsm::DEFAULT_BUF_SIZE
+            })
             .build()
             .map_err(to_py_err)?;
         Ok(Self { fsm })
     }
 
+    fn __repr__(&self) -> String {
+        format!("{self:?}")
+    }
+
     fn write(&mut self, bytes: &[u8]) -> PyResult<()> {
         self.fsm.write_all(bytes);
         Ok(())
@@ -308,6 +318,42 @@ decoder.write(bytes(record))
 records = decoder.decode()
 assert len(records) == 1
 assert records[0] == record
+"#
+                ),
+                None,
+                None,
+            )
+        })
+        .unwrap();
+    }
+
+    #[test]
+    fn test_decode_all_data_in_compat_situation() {
+        setup();
+        Python::with_gil(|py| {
+            Python::run(
+                py,
+                c_str!(
+                    r#"from _lib import DBNDecoder, ErrorMsg, ErrorMsgV1, Metadata, Schema, SType
+
+decoder = DBNDecoder()
+metadata = Metadata(
+    version=1,
+    dataset="GLBX.MDP3",
+    schema=Schema.MBO,
+    start=0,
+    stype_in=SType.RAW_SYMBOL,
+    stype_out=SType.INSTRUMENT_ID,
+)
+decoder.write(bytes(metadata))
+for _ in range(3):
+    error = ErrorMsgV1(0, "test")
+    decoder.write(bytes(error))
+records = decoder.decode()
+assert len(records) == 4
+assert isinstance(records[0], Metadata)
+for i in range(1, 4):
+    assert isinstance(records[i], ErrorMsg)
 "#
                 ),
                 None,

python/src/dbn_decoder.rs

@@ -16,7 +16,7 @@ name = "dbn"
 path = "src/main.rs"
 
 [dependencies]
-dbn = { path = "../dbn", version = "=0.36.1", default-features = false }
+dbn = { path = "../dbn", version = "=0.36.2", default-features = false }
 
 anyhow = { workspace = true }
 clap = { version = "4.5", features = ["derive", "wrap_help"] }

rust/dbn-cli/Cargo.toml

@@ -25,9 +25,9 @@ serde = ["dep:serde", "time/parsing", "time/serde"]
 trivial_copy = []
 
 [dependencies]
-dbn-macros = { version = "=0.36.1", path = "../dbn-macros" }
+dbn-macros = { version = "=0.36.2", path = "../dbn-macros" }
 
-async-compression = { version = "0.4.23", features = ["tokio", "zstd"], optional = true }
+async-compression = { version = "0.4.25", features = ["tokio", "zstd"], optional = true }
 csv = { workspace = true }
 fallible-streaming-iterator = { version = "0.1.9", features = ["std"] }
 # Fast integer to string conversion

rust/dbn/Cargo.toml

@@ -21,7 +21,6 @@ use crate::{
 };
 
 /// State machine for decoding DBN with bring your own I/O.
-#[derive(Debug)]
 pub struct DbnFsm {
     input_dbn_version: Option<u8>,
     upgrade_policy: VersionUpgradePolicy,
@@ -31,6 +30,23 @@ pub struct DbnFsm {
     compat_buffer: oval::Buffer,
 }
 
+impl std::fmt::Debug for DbnFsm {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let dbg_available_data = self.buffer.available_data().min(MAX_RECORD_LEN);
+        f.debug_struct("DbnFsm")
+            .field("input_dbn_version", &self.input_dbn_version)
+            .field("upgrade_policy", &self.upgrade_policy)
+            .field("ts_out", &self.ts_out)
+            .field("state", &self.state)
+            .field(
+                "buffer_available_data",
+                &&self.buffer.data()[..dbg_available_data],
+            )
+            .field("compat_buffer_capacity", &self.compat_buffer.capacity())
+            .finish_non_exhaustive()
+    }
+}
+
 #[derive(Debug, Default)]
 enum State {
     #[default]
@@ -386,10 +402,7 @@ impl DbnFsm {
         let mut compat_bytes = 0;
         let mut remaining_compat = self.compat_buffer.space();
         let mut expand_compat = false;
-        while rec_ref_buf.has_capacity(record_count) {
-            if read_bytes >= self.buffer.available_data() {
-                break;
-            }
+        while rec_ref_buf.has_capacity(record_count) && read_bytes < self.buffer.available_data() {
             let remaining_data = &self.buffer.data()[read_bytes..];
 
             let length = remaining_data[0] as usize * RecordHeader::LENGTH_MULTIPLIER;
@@ -463,6 +476,9 @@ impl DbnFsm {
                 "invalid DBN metadata. Metadata length shorter than fixed length.",
             ));
         }
+        if self.upgrade_policy.is_upgrade_situation(version) && self.compat_buffer.capacity() == 0 {
+            self.double_compat_buffer();
+        }
         self.state = State::Metadata { length };
         self.buffer.consume_noshift(Self::METADATA_PRELUDE_LEN);
         self.buffer