Skip to content

Commit 9bc23e1

Browse files
committed
feat(inspect): implement SnapshotsTable scanning
- Add Scan() virtual method and Scan() convenience overload to MetadataTable - Add SnapshotSelection struct for time-travel snapshot resolution - Add supports_time_travel() concrete method driven by kind() - Implement SnapshotsTable::Scan() to materialize snapshot rows via ArrowRowBuilder
1 parent cd4ca42 commit 9bc23e1

9 files changed

Lines changed: 540 additions & 65 deletions

src/iceberg/inspect/metadata_table.cc

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,23 @@ MetadataTable::MetadataTable(std::shared_ptr<Table> source_table,
3535

3636
MetadataTable::~MetadataTable() = default;
3737

38+
bool MetadataTable::supports_time_travel() const noexcept {
39+
// Time travel is supported for tables that read from a single snapshot's
40+
// manifests. Tables that scan all snapshots or return in-memory history do
41+
// not.
42+
switch (kind()) {
43+
case Kind::kSnapshots:
44+
case Kind::kHistory:
45+
return false;
46+
}
47+
return false;
48+
}
49+
50+
Result<ArrowArray> MetadataTable::Scan(
51+
std::optional<SnapshotSelection> /*snapshot_selection*/) {
52+
return NotSupported("Scan is not supported for this metadata table type");
53+
}
54+
3855
Result<std::unique_ptr<MetadataTable>> MetadataTable::Make(std::shared_ptr<Table> table,
3956
Kind kind) {
4057
if (table == nullptr) [[unlikely]] {

src/iceberg/inspect/metadata_table.h

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,28 @@
2020
#pragma once
2121

2222
#include <memory>
23+
#include <optional>
24+
#include <string>
2325

26+
#include "iceberg/arrow_c_data.h"
2427
#include "iceberg/iceberg_export.h"
2528
#include "iceberg/result.h"
2629
#include "iceberg/table_identifier.h"
2730
#include "iceberg/type_fwd.h"
31+
#include "iceberg/util/timepoint.h"
2832

2933
namespace iceberg {
3034

35+
/// \brief Parameters for snapshot selection (time travel).
36+
struct SnapshotSelection {
37+
/// \brief The snapshot ID to read.
38+
std::optional<int64_t> snapshot_id;
39+
/// \brief Read the snapshot that was current at this timestamp.
40+
std::optional<TimePointMs> as_of_timestamp;
41+
/// \brief Read the snapshot referenced by this named ref (branch or tag).
42+
std::optional<std::string> ref_name;
43+
};
44+
3145
/// \brief Base class for Iceberg metadata tables.
3246
class ICEBERG_EXPORT MetadataTable {
3347
public:
@@ -43,6 +57,28 @@ class ICEBERG_EXPORT MetadataTable {
4357

4458
virtual Kind kind() const noexcept = 0;
4559

60+
/// \brief Whether this metadata table supports time-travel queries.
61+
///
62+
/// Time travel is supported for tables that read from a single snapshot's
63+
/// manifests (e.g., Entries, Files, Manifests, Partitions). Tables that
64+
/// scan all snapshots (All*) or return in-memory history (Snapshots,
65+
/// History, Refs) do not support time travel.
66+
bool supports_time_travel() const noexcept;
67+
68+
/// \brief Scan the metadata table using the current snapshot.
69+
///
70+
/// Convenience overload — delegates to Scan(std::nullopt).
71+
Result<ArrowArray> Scan() { return Scan(std::nullopt); }
72+
73+
/// \brief Scan the metadata table and return all rows as an Arrow struct array.
74+
///
75+
/// The returned ArrowArray is a struct array where each element is one row.
76+
/// The caller takes ownership and must call ArrowArrayRelease when done.
77+
///
78+
/// The default implementation returns NotSupported. Subclasses override this
79+
/// to materialize their data.
80+
virtual Result<ArrowArray> Scan(std::optional<SnapshotSelection> snapshot_selection);
81+
4682
const TableIdentifier& name() const { return identifier_; }
4783

4884
const std::shared_ptr<Schema>& schema() const { return schema_; }

src/iceberg/inspect/snapshots_table.cc

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,18 @@
1919

2020
#include "iceberg/inspect/snapshots_table.h"
2121

22+
#include <chrono>
2223
#include <memory>
2324
#include <utility>
2425
#include <vector>
2526

27+
#include "iceberg/arrow_row_builder_internal.h"
2628
#include "iceberg/schema.h"
2729
#include "iceberg/schema_field.h"
2830
#include "iceberg/table.h"
2931
#include "iceberg/table_identifier.h"
3032
#include "iceberg/type.h"
33+
#include "iceberg/util/macros.h"
3134

3235
namespace iceberg {
3336
namespace {
@@ -65,4 +68,46 @@ Result<std::unique_ptr<SnapshotsTable>> SnapshotsTable::Make(
6568
return std::unique_ptr<SnapshotsTable>(new SnapshotsTable(std::move(table)));
6669
}
6770

71+
Result<ArrowArray> SnapshotsTable::Scan(
72+
std::optional<SnapshotSelection> /*snapshot_selection*/) {
73+
ICEBERG_ASSIGN_OR_RAISE(auto builder, ArrowRowBuilder::Make(*schema()));
74+
75+
for (const auto& snapshot : source_table()->snapshots()) {
76+
// column 0: committed_at (timestamptz → int64 micros)
77+
ICEBERG_RETURN_UNEXPECTED(AppendInt(
78+
builder.column(0), std::chrono::duration_cast<std::chrono::microseconds>(
79+
snapshot->timestamp_ms.time_since_epoch())
80+
.count()));
81+
82+
// column 1: snapshot_id (long)
83+
ICEBERG_RETURN_UNEXPECTED(AppendInt(builder.column(1), snapshot->snapshot_id));
84+
85+
// column 2: parent_id (long, optional)
86+
if (snapshot->parent_snapshot_id.has_value()) {
87+
ICEBERG_RETURN_UNEXPECTED(
88+
AppendInt(builder.column(2), *snapshot->parent_snapshot_id));
89+
} else {
90+
ICEBERG_RETURN_UNEXPECTED(AppendNull(builder.column(2)));
91+
}
92+
93+
// column 3: operation (string, optional)
94+
auto op = snapshot->Operation();
95+
if (op.has_value()) {
96+
ICEBERG_RETURN_UNEXPECTED(AppendString(builder.column(3), *op));
97+
} else {
98+
ICEBERG_RETURN_UNEXPECTED(AppendNull(builder.column(3)));
99+
}
100+
101+
// column 4: manifest_list (string, optional)
102+
ICEBERG_RETURN_UNEXPECTED(AppendString(builder.column(4), snapshot->manifest_list));
103+
104+
// column 5: summary (map<string,string>)
105+
ICEBERG_RETURN_UNEXPECTED(AppendStringMap(builder.column(5), snapshot->summary));
106+
107+
ICEBERG_RETURN_UNEXPECTED(builder.FinishRow());
108+
}
109+
110+
return std::move(builder).Finish();
111+
}
112+
68113
} // namespace iceberg

src/iceberg/inspect/snapshots_table.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,13 @@ class ICEBERG_EXPORT SnapshotsTable : public MetadataTable {
3737

3838
Kind kind() const noexcept override { return Kind::kSnapshots; }
3939

40+
/// \brief Scan all snapshots as rows.
41+
///
42+
/// The snapshots table always returns every known snapshot, so the
43+
/// snapshot_selection parameter is ignored.
44+
Result<ArrowArray> Scan(
45+
std::optional<SnapshotSelection> /*snapshot_selection*/) override;
46+
4047
private:
4148
explicit SnapshotsTable(std::shared_ptr<Table> table);
4249
};

src/iceberg/test/CMakeLists.txt

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,12 @@ if(ICEBERG_BUILD_BUNDLE)
188188

189189
add_iceberg_test(catalog_test USE_BUNDLE SOURCES in_memory_catalog_test.cc)
190190

191-
add_iceberg_test(metadata_table_test USE_BUNDLE SOURCES metadata_table_test.cc)
191+
add_iceberg_test(metadata_table_test
192+
USE_BUNDLE
193+
SOURCES
194+
history_table_test.cc
195+
metadata_table_test.cc
196+
snapshots_table_test.cc)
192197

193198
add_iceberg_test(eval_expr_test
194199
USE_BUNDLE
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
/// \file history_table_test.cc
21+
/// Unit tests for HistoryTable.
22+
23+
#include <gmock/gmock.h>
24+
#include <gtest/gtest.h>
25+
26+
#include "iceberg/inspect/metadata_table.h"
27+
#include "iceberg/schema.h"
28+
#include "iceberg/schema_field.h"
29+
#include "iceberg/test/matchers.h"
30+
#include "iceberg/test/metadata_table_test_base.h"
31+
#include "iceberg/type.h"
32+
33+
namespace iceberg {
34+
namespace {
35+
36+
std::shared_ptr<Schema> MakeHistorySchema() {
37+
return std::make_shared<Schema>(std::vector<SchemaField>{
38+
SchemaField::MakeRequired(1, "made_current_at", timestamp_tz()),
39+
SchemaField::MakeRequired(2, "snapshot_id", int64()),
40+
SchemaField::MakeOptional(3, "parent_id", int64()),
41+
SchemaField::MakeRequired(4, "is_current_ancestor", boolean())});
42+
}
43+
44+
} // namespace
45+
46+
class HistoryTableTest : public MetadataTableTestBase {};
47+
48+
TEST_F(HistoryTableTest, SchemaMatchesIcebergSchema) {
49+
ICEBERG_UNWRAP_OR_FAIL(auto history_table,
50+
MetadataTable::Make(table_, MetadataTable::Kind::kHistory));
51+
EXPECT_TRUE(*history_table->schema() == *MakeHistorySchema());
52+
}
53+
54+
} // namespace iceberg

src/iceberg/test/metadata_table_test.cc

Lines changed: 16 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -33,88 +33,40 @@
3333
#include "iceberg/type.h"
3434

3535
namespace iceberg {
36-
namespace {
37-
38-
std::shared_ptr<Schema> MakeSnapshotsSchema() {
39-
return std::make_shared<Schema>(std::vector<SchemaField>{
40-
SchemaField::MakeRequired(1, "committed_at", timestamp_tz()),
41-
SchemaField::MakeRequired(2, "snapshot_id", int64()),
42-
SchemaField::MakeOptional(3, "parent_id", int64()),
43-
SchemaField::MakeOptional(4, "operation", string()),
44-
SchemaField::MakeOptional(5, "manifest_list", string()),
45-
SchemaField::MakeOptional(
46-
6, "summary",
47-
std::make_shared<MapType>(SchemaField::MakeRequired(7, "key", string()),
48-
SchemaField::MakeRequired(8, "value", string())))});
49-
}
50-
51-
std::shared_ptr<Schema> MakeHistorySchema() {
52-
return std::make_shared<Schema>(std::vector<SchemaField>{
53-
SchemaField::MakeRequired(1, "made_current_at", timestamp_tz()),
54-
SchemaField::MakeRequired(2, "snapshot_id", int64()),
55-
SchemaField::MakeOptional(3, "parent_id", int64()),
56-
SchemaField::MakeRequired(4, "is_current_ancestor", boolean())});
57-
}
58-
59-
} // namespace
6036

6137
class MetadataTableTest : public ::testing::Test {
6238
protected:
6339
void SetUp() override {
64-
io_ = std::make_shared<MockFileIO>();
65-
catalog_ = std::make_shared<MockCatalog>();
66-
6740
auto schema = std::make_shared<Schema>(
6841
std::vector<SchemaField>{SchemaField::MakeRequired(1, "id", int64()),
6942
SchemaField::MakeOptional(2, "name", string())},
7043
1);
71-
metadata_ = std::make_shared<TableMetadata>(
44+
auto metadata = std::make_shared<TableMetadata>(
7245
TableMetadata{.format_version = 2, .schemas = {schema}, .current_schema_id = 1});
7346

74-
TableIdentifier source_ident{.ns = Namespace{.levels = {"db"}},
75-
.name = "source_table"};
76-
auto source_table_result =
77-
Table::Make(source_ident, metadata_, "s3://bucket/meta.json", io_, catalog_);
78-
EXPECT_THAT(source_table_result, IsOk());
79-
source_table_ = *source_table_result;
80-
81-
auto snapshots_table_result =
82-
MetadataTable::Make(source_table_, MetadataTable::Kind::kSnapshots);
83-
EXPECT_THAT(snapshots_table_result, IsOk());
84-
snapshots_table_ = std::move(*snapshots_table_result);
47+
TableIdentifier ident{.ns = Namespace{.levels = {"db"}}, .name = "source_table"};
48+
ICEBERG_UNWRAP_OR_FAIL(table_, Table::Make(ident, metadata, "s3://bucket/meta.json",
49+
std::make_shared<MockFileIO>(),
50+
std::make_shared<MockCatalog>()));
8551
}
8652

87-
std::shared_ptr<MockFileIO> io_;
88-
std::shared_ptr<MockCatalog> catalog_;
89-
std::shared_ptr<TableMetadata> metadata_;
90-
std::shared_ptr<Table> source_table_;
91-
std::unique_ptr<MetadataTable> snapshots_table_;
53+
std::shared_ptr<Table> table_;
9254
};
9355

94-
TEST_F(MetadataTableTest, Constructor) {
95-
EXPECT_EQ(snapshots_table_->kind(), MetadataTable::Kind::kSnapshots);
96-
EXPECT_EQ(snapshots_table_->source_table(), source_table_);
97-
EXPECT_EQ(snapshots_table_->name().name, "source_table.snapshots");
98-
EXPECT_EQ(snapshots_table_->name().ns.levels, (std::vector<std::string>{"db"}));
99-
EXPECT_NE(snapshots_table_->schema(), nullptr);
100-
}
101-
102-
TEST_F(MetadataTableTest, SnapshotsSchemaMatchesIcebergSchema) {
103-
EXPECT_TRUE(*snapshots_table_->schema() == *MakeSnapshotsSchema());
104-
}
105-
106-
TEST_F(MetadataTableTest, HistorySchemaMatchesIcebergSchema) {
107-
auto history_table_result =
108-
MetadataTable::Make(source_table_, MetadataTable::Kind::kHistory);
109-
ASSERT_THAT(history_table_result, IsOk());
110-
111-
EXPECT_TRUE(*(*history_table_result)->schema() == *MakeHistorySchema());
112-
}
113-
11456
TEST_F(MetadataTableTest, FactoryRejectsNullSourceTable) {
11557
auto result = MetadataTable::Make(nullptr, MetadataTable::Kind::kSnapshots);
11658
EXPECT_THAT(result, IsError(ErrorKind::kInvalidArgument));
11759
EXPECT_THAT(result, HasErrorMessage("Table cannot be null"));
11860
}
11961

62+
TEST_F(MetadataTableTest, SupportsTimeTravel) {
63+
ICEBERG_UNWRAP_OR_FAIL(auto snapshots_table,
64+
MetadataTable::Make(table_, MetadataTable::Kind::kSnapshots));
65+
EXPECT_FALSE(snapshots_table->supports_time_travel());
66+
67+
ICEBERG_UNWRAP_OR_FAIL(auto history_table,
68+
MetadataTable::Make(table_, MetadataTable::Kind::kHistory));
69+
EXPECT_FALSE(history_table->supports_time_travel());
70+
}
71+
12072
} // namespace iceberg

0 commit comments

Comments
 (0)