diff --git a/.github/gh_matrix_builder.py b/.github/gh_matrix_builder.py index 7c414781b9b..c650e166046 100755 --- a/.github/gh_matrix_builder.py +++ b/.github/gh_matrix_builder.py @@ -51,6 +51,7 @@ "bgw_launcher", "telemetry", "memoize", + "net", } # Tests that we do not run as part of a Flake tests diff --git a/.github/workflows/linux-32bit-build-and-test.yaml b/.github/workflows/linux-32bit-build-and-test.yaml index 3294424a1a6..dc722dc1527 100644 --- a/.github/workflows/linux-32bit-build-and-test.yaml +++ b/.github/workflows/linux-32bit-build-and-test.yaml @@ -67,6 +67,7 @@ jobs: vector_agg_text vector_agg_uuid_segmentby vectorized_aggregation + net SKIPS: chunk_adaptive histogram_test-* EXTENSIONS: "postgres_fdw test_decoding pageinspect pgstattuple" strategy: diff --git a/.github/workflows/sanitizer-build-and-test.yaml b/.github/workflows/sanitizer-build-and-test.yaml index 6ec40167224..b65a0e4222e 100644 --- a/.github/workflows/sanitizer-build-and-test.yaml +++ b/.github/workflows/sanitizer-build-and-test.yaml @@ -48,7 +48,7 @@ env: print_stacktrace=1 halt_on_error=1 log_path=${{ github.workspace }}/sanitizer_logs/sanitizer log_exe_name=true print_suppressions=false exitcode=27 - IGNORES: "bgw_db_scheduler bgw_db_scheduler_fixed" + IGNORES: "bgw_db_scheduler bgw_db_scheduler_fixed net telemetry" EXTENSIONS: "postgres_fdw test_decoding pageinspect pgstattuple" jobs: diff --git a/.github/workflows/windows-build-and-test.yaml b/.github/workflows/windows-build-and-test.yaml index 25b1acb8214..8f0dad873af 100644 --- a/.github/workflows/windows-build-and-test.yaml +++ b/.github/workflows/windows-build-and-test.yaml @@ -81,6 +81,7 @@ jobs: chunk_adaptive metadata telemetry + net SKIPS: >- bgw_db_scheduler bgw_db_scheduler_fixed diff --git a/.unreleased/pr_8211 b/.unreleased/pr_8211 new file mode 100644 index 00000000000..0d4b5992695 --- /dev/null +++ b/.unreleased/pr_8211 @@ -0,0 +1 @@ +Fixes: #8211 Fix dump/restore when chunk skipping is enabled diff --git a/scripts/test_update_from_version.sh b/scripts/test_update_from_version.sh index 06e34cec3ac..6dc9b2974c0 100755 --- a/scripts/test_update_from_version.sh +++ b/scripts/test_update_from_version.sh @@ -12,7 +12,7 @@ FROM_VERSION=${FROM_VERSION:-$(grep '^previous_version ' version.config | awk '{ TO_VERSION=${TO_VERSION:-$(grep '^version ' version.config | awk '{ print $3 }')} TEST_REPAIR=${TEST_REPAIR:-false} -TEST_VERSION=${TEST_VERSION:-v8} +TEST_VERSION=${TEST_VERSION:-v9} OUTPUT_DIR=${OUTPUT_DIR:-update_test/${FROM_VERSION}_to_${TO_VERSION}} PGDATA="${OUTPUT_DIR}/data" @@ -135,5 +135,3 @@ if [ -f "${OUTPUT_DIR}/baseline_vs_updated.diff" ] || [ -f "${OUTPUT_DIR}/baseli echo "Update test for ${FROM_VERSION} -> ${TO_VERSION} failed" exit 1 fi - - diff --git a/scripts/test_updates.sh b/scripts/test_updates.sh index 1c5813c414f..cf03d08225c 100755 --- a/scripts/test_updates.sh +++ b/scripts/test_updates.sh @@ -87,14 +87,19 @@ if [ -n "${VERSIONS}" ]; then for version in ${VERSIONS}; do ts_minor_version=$(echo "${version}" | awk -F. '{print $2}') - TEST_VERSION=v8 - if [ "${ts_minor_version}" -ge 10 ]; then TEST_REPAIR=true else TEST_REPAIR=false fi + + if [ "${ts_minor_version}" -ge 16 ]; then + TEST_VERSION=v9 + else + TEST_VERSION=v8 + fi + export TEST_VERSION TEST_REPAIR FROM_VERSION=${version} "${SCRIPT_DIR}/test_update_from_version.sh" @@ -115,4 +120,3 @@ else fi exit $FAIL_COUNT - diff --git a/sql/pre_install/tables.sql b/sql/pre_install/tables.sql index 902702fcc8d..1c319a18782 100644 --- a/sql/pre_install/tables.sql +++ b/sql/pre_install/tables.sql @@ -230,7 +230,7 @@ SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_index', ' CREATE TABLE _timescaledb_catalog.chunk_column_stats ( id serial NOT NULL, hypertable_id integer NOT NULL, - chunk_id integer NOT NULL, + chunk_id integer NULL, column_name name NOT NULL, range_start bigint NOT NULL, range_end bigint NOT NULL, diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 6f78cef4516..c24aaedb9b8 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -39,4 +39,10 @@ AS $$ BEGIN -- empty body END; -$$ LANGUAGE PLPGSQL; \ No newline at end of file +$$ LANGUAGE PLPGSQL; + +-- Make chunk_id use NULL to mark special entries instead of 0 +-- (Invalid chunk) since that doesn't work with the FK constraint on +-- chunk_id. +ALTER TABLE _timescaledb_catalog.chunk_column_stats ALTER COLUMN chunk_id DROP NOT NULL; +UPDATE _timescaledb_catalog.chunk_column_stats SET chunk_id = NULL WHERE chunk_id = 0; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index 5b7c3cd3aec..75a1b9cc916 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -23,3 +23,28 @@ BEGIN -- empty body END; $$ LANGUAGE PLPGSQL; + +-- Add back the chunk_column_stats NOT NULL constraint. But first +-- delete all entries with with NULL since they will no longer be +-- allowed. Note that reverting chunk_id back to 0 because it will +-- violate the FK constraint. Even if we would revert, the downgrade +-- tests for "restore" would fail due to violating entries. Removing +-- the entries effectively means that collecting column stats for +-- those columns will be disabled. It can be enabled again after +-- downgrade. We emit a warning if anything was disabled. +DO $$ +DECLARE + num_null_chunk_ids int; +BEGIN + + SELECT count(*) INTO num_null_chunk_ids + FROM _timescaledb_catalog.chunk_column_stats WHERE chunk_id IS NULL; + + IF num_null_chunk_ids > 0 THEN + RAISE WARNING 'chunk skipping has been disabled for all hypertables' + USING HINT = 'Use enable_chunk_skipping() to re-enable chunk skipping'; + END IF; +END +$$; +DELETE FROM _timescaledb_catalog.chunk_column_stats WHERE chunk_id IS NULL; +ALTER TABLE _timescaledb_catalog.chunk_column_stats ALTER COLUMN chunk_id SET NOT NULL; diff --git a/src/ts_catalog/chunk_column_stats.c b/src/ts_catalog/chunk_column_stats.c index 103596c2782..08e769a8408 100644 --- a/src/ts_catalog/chunk_column_stats.c +++ b/src/ts_catalog/chunk_column_stats.c @@ -4,6 +4,14 @@ * LICENSE-APACHE for a copy of the license. */ #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include @@ -13,12 +21,14 @@ #include #include #include +#include #include -#include "compat/compat.h" +#include "chunk.h" #include "chunk_column_stats.h" -#include "dimension_vector.h" +#include "dimension_slice.h" #include "guc.h" +#include "ts_catalog/catalog.h" /* * Enable chunk column stats attributes @@ -131,6 +141,9 @@ chunk_column_stats_insert_relation(const Relation rel, Form_chunk_column_stats i Int64GetDatum(info->range_end); values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_valid)] = BoolGetDatum(info->valid); + if (info->chunk_id == INVALID_CHUNK_ID) + nulls[AttrNumberGetAttrOffset(Anum_chunk_column_stats_chunk_id)] = true; + ts_catalog_insert_values(rel, desc, values, nulls); ts_catalog_restore_user(&sec_ctx); @@ -354,13 +367,18 @@ ts_chunk_column_stats_add_internal(FunctionCallInfo fcinfo, Oid table_relid, Nam { ListCell *lc; List *chunk_id_list = ts_chunk_get_chunk_ids_by_hypertable_id(ht->fd.id); + Catalog *catalog = ts_catalog_get(); + Relation rel; + rel = table_open(catalog_get_table_id(catalog, CHUNK_COLUMN_STATS), RowExclusiveLock); foreach (lc, chunk_id_list) { /* other fields are set appropriately in fd above. Only change chunk_id */ fd.chunk_id = lfirst_int(lc); - chunk_column_stats_insert(&fd); + chunk_column_stats_insert_relation(rel, &fd); } + + table_close(rel, RowExclusiveLock); } do_return: @@ -600,22 +618,52 @@ create_col_stats_check_constraint(const Form_chunk_column_stats info, Oid main_t return constr; } +/* + * Fill in the form for chunk_column_stats. + * + * Note that it is necessary to deform the tuple since it is not possible to + * use GETSTRUCT when chunk_id can be NULL. + */ +static void +fill_form_from_slot(TupleTableSlot *slot, Form_chunk_column_stats form) +{ + bool should_free; + HeapTuple tuple = ExecFetchSlotHeapTuple(slot, false, &should_free); + Datum values[_Anum_chunk_column_stats_max]; + bool nulls[_Anum_chunk_column_stats_max]; + + heap_deform_tuple(tuple, slot->tts_tupleDescriptor, values, nulls); + + form->id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_id)]); + form->hypertable_id = + DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_hypertable_id)]); + + if (nulls[AttrNumberGetAttrOffset(Anum_chunk_column_stats_chunk_id)]) + form->chunk_id = INVALID_CHUNK_ID; + else + form->chunk_id = + DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_chunk_id)]); + + namestrcpy(&form->column_name, + NameStr(*DatumGetName( + values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_column_name)]))); + form->range_end = + DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_end)]); + form->range_start = + DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_range_start)]); + form->valid = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_chunk_column_stats_valid)]); + + if (should_free) + heap_freetuple(tuple); +} + static ScanTupleResult chunk_column_stats_tuple_found(TupleInfo *ti, void *data) { ChunkRangeSpace *rs = data; Form_chunk_column_stats d = &rs->range_cols[rs->num_range_cols++]; - Form_chunk_column_stats fd; - bool should_free; - HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); - Assert(rs->num_range_cols <= rs->capacity); - - fd = (Form_chunk_column_stats) GETSTRUCT(tuple); - memcpy(d, fd, sizeof(*fd)); - - if (should_free) - heap_freetuple(tuple); + fill_form_from_slot(ti->slot, d); return SCAN_CONTINUE; } @@ -640,11 +688,15 @@ ts_chunk_column_stats_range_space_scan(int32 hypertable_id, Oid ht_reloid, Memor BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); - ScanKeyInit( + + ScanKeyEntryInitialize( &scankey[1], + SK_ISNULL | SK_SEARCHNULL, Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, BTEqualStrategyNumber, - F_INT4EQ, + InvalidOid, + InvalidOid, + InvalidOid, Int32GetDatum(INVALID_CHUNK_ID)); chunk_column_stats_scan_internal(scankey, @@ -669,14 +721,7 @@ static ScanTupleResult form_range_tuple_found(TupleInfo *ti, void *data) { Form_chunk_column_stats rg = data; - bool should_free; - HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); - - memcpy(rg, GETSTRUCT(tuple), sizeof(FormData_chunk_column_stats)); - - if (should_free) - heap_freetuple(tuple); - + fill_form_from_slot(ti->slot, rg); return SCAN_DONE; } @@ -694,12 +739,29 @@ ts_chunk_column_stats_lookup(int32 hypertable_id, int32 chunk_id, const char *co BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(hypertable_id)); - ScanKeyInit( - &scankey[1], - Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(chunk_id)); + + if (chunk_id == INVALID_CHUNK_ID) + { + ScanKeyEntryInitialize( + &scankey[1], + SK_ISNULL | SK_SEARCHNULL, + Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, + BTEqualStrategyNumber, + InvalidOid, + InvalidOid, + InvalidOid, + Int32GetDatum(chunk_id)); + } + else + { + ScanKeyInit( + &scankey[1], + Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id, + BTEqualStrategyNumber, + F_INT4EQ, + Int32GetDatum(chunk_id)); + } + ScanKeyInit( &scankey[2], Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_column_name, @@ -760,7 +822,6 @@ ts_chunk_column_stats_calculate(const Hypertable *ht, const Chunk *chunk) char *col_name = NameStr(rs->range_cols[range_index].column_name); Oid col_type; - /* Get the attribute number in the HT for this column, and map to the chunk */ attno = get_attnum(ht->main_table_relid, col_name); attno = ts_map_attno(ht->main_table_relid, chunk->table_id, attno); col_type = get_atttype(chunk->table_id, attno); @@ -815,7 +876,6 @@ ts_chunk_column_stats_calculate(const Hypertable *ht, const Chunk *chunk) range->range_start = min; range->range_end = max; range->valid = true; - ts_chunk_column_stats_update_by_id(range->id, range); i++; } @@ -946,6 +1006,8 @@ ts_chunk_column_stats_delete_by_chunk_id(int32 chunk_id) ScanKeyData scankey[1]; int count = 0; + Assert(chunk_id != INVALID_CHUNK_ID); + /* Perform an index scan on chunk_id. */ ScanKeyInit( &scankey[0], @@ -977,6 +1039,8 @@ ts_chunk_column_stats_reset_by_chunk_id(int32 chunk_id) fd.range_end = PG_INT64_MAX; fd.valid = true; + Assert(chunk_id != INVALID_CHUNK_ID); + /* Perform an index scan on chunk_id. */ ScanKeyInit( &scankey[0], @@ -1107,19 +1171,22 @@ ts_chunk_column_stats_get_chunk_ids_by_scan(DimensionRestrictInfo *dri) ts_scan_iterator_start_or_restart_scan(&it); ts_scanner_foreach(&it) { - Form_chunk_column_stats fd; - bool should_free, matched = false; - HeapTuple tuple = ts_scanner_fetch_heap_tuple(it.tinfo, false, &should_free); - - fd = (Form_chunk_column_stats) GETSTRUCT(tuple); + FormData_chunk_column_stats fd; + bool matched = false; + bool chunk_id_isnull; + chunk_id_isnull = slot_attisnull( + it.tinfo->slot, + Anum_chunk_column_stats_ht_id_chunk_id_column_name_range_start_range_end_idx_chunk_id); /* * We have an entry with INVALID_CHUNK_ID which will match all cases due to * -INF/+INF range entries for it. Ignore that. */ - if (fd->chunk_id == INVALID_CHUNK_ID) + if (chunk_id_isnull) goto done; + fill_form_from_slot(it.tinfo->slot, &fd); + /* * If an entry is marked "invalid" then it means that the ranges cannot be relied * on. So, we assume the worse case and include this chunk for the scan. @@ -1130,7 +1197,7 @@ ts_chunk_column_stats_get_chunk_ids_by_scan(DimensionRestrictInfo *dri) * Also, if we have a valid chunnk with -inf/+inf entries then it matches all * queries */ - if (!fd->valid || (fd->range_start == PG_INT64_MIN && fd->range_end == PG_INT64_MAX)) + if (!fd.valid || (fd.range_start == PG_INT64_MIN && fd.range_end == PG_INT64_MAX)) { matched = true; goto done; @@ -1147,12 +1214,12 @@ ts_chunk_column_stats_get_chunk_ids_by_scan(DimensionRestrictInfo *dri) { case BTLessEqualStrategyNumber: /* e.g: id <= 90 */ { - matched = fd->range_start <= open->upper_bound; + matched = fd.range_start <= open->upper_bound; } break; case BTLessStrategyNumber: /* e.g: id < 90 */ { - matched = fd->range_start < open->upper_bound; + matched = fd.range_start < open->upper_bound; } break; default: @@ -1169,13 +1236,13 @@ ts_chunk_column_stats_get_chunk_ids_by_scan(DimensionRestrictInfo *dri) case BTGreaterEqualStrategyNumber: { /* range_end is exclusive */ - matched = (fd->range_end - 1) >= open->lower_bound; + matched = (fd.range_end - 1) >= open->lower_bound; } break; case BTGreaterStrategyNumber: { /* range_end is exclusive */ - matched = (fd->range_end - 1) > open->lower_bound; + matched = (fd.range_end - 1) > open->lower_bound; } break; default: @@ -1185,10 +1252,7 @@ ts_chunk_column_stats_get_chunk_ids_by_scan(DimensionRestrictInfo *dri) done: if (matched) - chunkids = lappend_int(chunkids, fd->chunk_id); - - if (should_free) - heap_freetuple(tuple); + chunkids = lappend_int(chunkids, fd.chunk_id); } ts_scan_iterator_close(&it); @@ -1223,7 +1287,7 @@ ts_chunk_column_stats_set_name(FormData_chunk_column_stats *in_fd, char *new_col ts_scanner_foreach(&it) { Datum values[Natts_chunk_column_stats] = { 0 }; - bool isnull[Natts_chunk_column_stats] = { 0 }; + bool isnull[Natts_chunk_column_stats] = { false }; bool doReplace[Natts_chunk_column_stats] = { 0 }; bool should_free; TupleInfo *ti = ts_scan_iterator_tuple_info(&it); @@ -1284,6 +1348,8 @@ ts_chunk_column_stats_set_invalid(int32 hypertable_id, int32 chunk_id) { ScanKeyData scankey[2]; + Assert(chunk_id != INVALID_CHUNK_ID); + /* Perform an index scan on hypertable_id, chunk_id. */ ScanKeyInit( &scankey[0], @@ -1320,13 +1386,13 @@ construct_check_constraint_range_tuple(TupleInfo *ti, void *data) { bool should_free; HeapTuple tuple = ts_scanner_fetch_heap_tuple(ti, false, &should_free); - Form_chunk_column_stats fd; + FormData_chunk_column_stats fd; Constraint *constr; CheckList *checklist = data; - fd = (Form_chunk_column_stats) GETSTRUCT(tuple); + fill_form_from_slot(ti->slot, &fd); - constr = create_col_stats_check_constraint(fd, + constr = create_col_stats_check_constraint(&fd, checklist->main_table_relid, checklist->chunk_relid, NULL); @@ -1361,6 +1427,8 @@ ts_chunk_column_stats_construct_check_constraints(Relation relation, Oid reloid, clist.chunk_relid = reloid; clist.main_table_relid = ts_hypertable_id_to_relid(fd.hypertable_id, false); + Assert(fd.id != INVALID_CHUNK_ID); + /* Perform an index scan on hypertable_id, chunk_id. */ ScanKeyInit( &scankey[0], diff --git a/test/sql/updates/cleanup.chunk_skipping.sql b/test/sql/updates/cleanup.chunk_skipping.sql new file mode 100644 index 00000000000..5e80c913792 --- /dev/null +++ b/test/sql/updates/cleanup.chunk_skipping.sql @@ -0,0 +1,5 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +DROP TABLE skip; diff --git a/test/sql/updates/cleanup.v7.sql b/test/sql/updates/cleanup.v7.sql index d14f54b3b07..9d6a6fa1b9b 100644 --- a/test/sql/updates/cleanup.v7.sql +++ b/test/sql/updates/cleanup.v7.sql @@ -8,4 +8,3 @@ \ir cleanup.continuous_aggs.v2.sql \ir cleanup.compression.sql \ir cleanup.policies.sql - diff --git a/test/sql/updates/cleanup.v8.sql b/test/sql/updates/cleanup.v8.sql new file mode 100644 index 00000000000..88201ccc1d9 --- /dev/null +++ b/test/sql/updates/cleanup.v8.sql @@ -0,0 +1,6 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +\ir cleanup.v7.sql + diff --git a/test/sql/updates/cleanup.v9.sql b/test/sql/updates/cleanup.v9.sql new file mode 100644 index 00000000000..3565b73bbcb --- /dev/null +++ b/test/sql/updates/cleanup.v9.sql @@ -0,0 +1,6 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +\ir cleanup.v8.sql +\ir cleanup.chunk_skipping.sql diff --git a/test/sql/updates/post.chunk_skipping.sql b/test/sql/updates/post.chunk_skipping.sql new file mode 100644 index 00000000000..af92e4449fe --- /dev/null +++ b/test/sql/updates/post.chunk_skipping.sql @@ -0,0 +1,11 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +-- Show chunk column stats after updating the extension. We need to +-- exclude the rows with chunk_id = 0 because we cannot keep those on +-- downgrade due to FK constraint. Showing them would mean a diff in +-- the output. We can still test that 0 chunk_ids are converted to +-- NULL values during upgrades, however. +SELECT * FROM _timescaledb_catalog.chunk_column_stats +WHERE chunk_id IS NULL OR chunk_id > 0 ORDER BY id; diff --git a/test/sql/updates/post.v9.sql b/test/sql/updates/post.v9.sql new file mode 100644 index 00000000000..c2fcc883d8f --- /dev/null +++ b/test/sql/updates/post.v9.sql @@ -0,0 +1,6 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +\ir post.v8.sql +\ir post.chunk_skipping.sql diff --git a/test/sql/updates/pre.cleanup.sql b/test/sql/updates/pre.cleanup.sql index cbf0d798a9d..bb70bb7aa41 100644 --- a/test/sql/updates/pre.cleanup.sql +++ b/test/sql/updates/pre.cleanup.sql @@ -30,6 +30,7 @@ DROP TABLE IF EXISTS repair_test_extra; DROP TABLE IF EXISTS repair_test_timestamp; DROP TABLE IF EXISTS repair_test_date; DROP TABLE IF EXISTS compress; +DROP TABLE IF EXISTS skip; DROP TABLE IF EXISTS devices; DROP TABLE IF EXISTS disthyper; DROP TABLE IF EXISTS policy_test_timestamptz; diff --git a/test/sql/updates/setup.chunk_skipping.sql b/test/sql/updates/setup.chunk_skipping.sql new file mode 100644 index 00000000000..36d28c97285 --- /dev/null +++ b/test/sql/updates/setup.chunk_skipping.sql @@ -0,0 +1,37 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + + +CREATE TABLE chunkskip ( + time TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + location INT, + temp FLOAT + ); + +SELECT setseed(0.1); +SELECT table_name FROM create_hypertable( 'chunkskip', 'time'); + +INSERT INTO chunkskip +SELECT g, g+interval '1 hour', ceil(random()*20), random()*30 +FROM generate_series('2018-12-01 00:00'::timestamp, '2018-12-31 00:00'::timestamp, '1 day') g; + +ALTER TABLE chunkskip SET (timescaledb.compress, timescaledb.compress_segmentby='location'); +SELECT count(compress_chunk(ch, true)) FROM show_chunks('chunkskip') ch; + +DO $$ +DECLARE + version text[] := (SELECT regexp_split_to_array(extversion,'(\.|-)') FROM pg_extension WHERE extname = 'timescaledb'); +BEGIN + + -- Enable chunk skipping. Doing this after compression to have a + -- "special" chunk_id entry of 0. We check that this is changed to + -- NULL during upgrade. + IF version[1]::int >= 2 AND version[2]::int >= 17 AND (version[2]::int > 17 OR version[3]::int > 0) THEN + SET timescaledb.enable_chunk_skipping = true; + END IF; +END +$$; + +SELECT enable_chunk_skipping('chunkskip', 'updated_at'); diff --git a/test/sql/updates/setup.v9.sql b/test/sql/updates/setup.v9.sql new file mode 100644 index 00000000000..bad5e0d571c --- /dev/null +++ b/test/sql/updates/setup.v9.sql @@ -0,0 +1,6 @@ +-- This file and its contents are licensed under the Apache License 2.0. +-- Please see the included NOTICE for copyright information and +-- LICENSE-APACHE for a copy of the license. + +\ir setup.v8.sql +\ir setup.chunk_skipping.sql diff --git a/tsl/test/expected/chunk_column_stats.out b/tsl/test/expected/chunk_column_stats.out index 48fe5f1694e..84907dd74df 100644 --- a/tsl/test/expected/chunk_column_stats.out +++ b/tsl/test/expected/chunk_column_stats.out @@ -81,7 +81,7 @@ SELECT * FROM enable_chunk_skipping('sample_table', 'sensor_id'); SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 1 | 1 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 1 | 1 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 2 | 1 | 1 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 3 | 1 | 2 | sensor_id | -9223372036854775808 | 9223372036854775807 | t (3 rows) @@ -564,7 +564,7 @@ INSERT INTO sample_table VALUES (:'start_date'::timestamptz, 1, 9, 78.999, 'new SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 1 | 1 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 1 | 1 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 3 | 1 | 2 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 2 | 1 | 1 | sensor_id | 1 | 9 | t 4 | 1 | 4 | sensor_id | -9223372036854775808 | 9223372036854775807 | t @@ -624,7 +624,7 @@ SELECT decompress_chunk(:'CH_NAME'); SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 5 | 1 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 5 | 1 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 6 | 1 | 2 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 8 | 1 | 4 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 7 | 1 | 1 | sensor_id | -9223372036854775808 | 9223372036854775807 | t @@ -641,7 +641,7 @@ WARNING: no index on "sensor_id" found for column range on chunk "_hyper_1_1_ch SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 5 | 1 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 5 | 1 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 6 | 1 | 2 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 8 | 1 | 4 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 7 | 1 | 1 | sensor_id | 1 | 9 | t @@ -657,7 +657,7 @@ SELECT decompress_chunk(:'CH_NAME'); SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 5 | 1 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 5 | 1 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 6 | 1 | 2 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 8 | 1 | 4 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 7 | 1 | 1 | sensor_id | -9223372036854775808 | 9223372036854775807 | t @@ -674,7 +674,7 @@ WARNING: no index on "sensor_id" found for column range on chunk "_hyper_1_1_ch SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 5 | 1 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 5 | 1 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 6 | 1 | 2 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 8 | 1 | 4 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 7 | 1 | 1 | sensor_id | 1 | 9 | t @@ -684,7 +684,7 @@ TRUNCATE :CH_NAME; SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 5 | 1 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 5 | 1 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 6 | 1 | 2 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 8 | 1 | 4 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 7 | 1 | 1 | sensor_id | -9223372036854775808 | 9223372036854775807 | t @@ -701,7 +701,7 @@ SELECT drop_chunks('sample_table', older_than => '2022-02-28'); SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 5 | 1 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 5 | 1 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 6 | 1 | 2 | sensor_id | -9223372036854775808 | 9223372036854775807 | t 8 | 1 | 4 | sensor_id | -9223372036854775808 | 9223372036854775807 | t (3 rows) @@ -750,7 +750,7 @@ SELECT * FROM enable_chunk_skipping('sample_table1', 'sensor_id'); SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 9 | 3 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 9 | 3 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t (1 row) \set start_date '2023-03-17 17:51:11.322998+05:30' @@ -759,7 +759,7 @@ INSERT INTO sample_table1 VALUES (:'start_date'::timestamptz, 12, 21, 33.123, 'n SELECT * from _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 9 | 3 | 0 | sensor_id | -9223372036854775808 | 9223372036854775807 | t + 9 | 3 | | sensor_id | -9223372036854775808 | 9223372036854775807 | t 10 | 3 | 7 | sensor_id | -9223372036854775808 | 9223372036854775807 | t (2 rows) @@ -816,7 +816,7 @@ SELECT compress_chunk(:'CH_NAME'); SELECT * FROM _timescaledb_catalog.chunk_column_stats; id | hypertable_id | chunk_id | column_name | range_start | range_end | valid ----+---------------+----------+-------------+----------------------+---------------------+------- - 11 | 4 | 0 | temperature | -9223372036854775808 | 9223372036854775807 | t + 11 | 4 | | temperature | -9223372036854775808 | 9223372036854775807 | t 12 | 4 | 8 | temperature | 366 | 502 | t (2 rows)