diff --git a/changelog/unreleased/pr-26581.toml b/changelog/unreleased/pr-26581.toml new file mode 100644 index 000000000000..ccf88cfa9774 --- /dev/null +++ b/changelog/unreleased/pr-26581.toml @@ -0,0 +1,5 @@ +type = "fixed" +message = "Fix content pack installation failing with \"Missing Stream for widget entity\" when a view references a stream that cannot be resolved. The unresolvable stream reference is now skipped with a warning instead of aborting the whole installation." + +issues = ["Graylog2/graylog-plugin-enterprise#12987"] +pulls = ["26581"] diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/QueryEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/QueryEntity.java index 97bb3cfdb8c3..fac1554a4ee8 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/QueryEntity.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/QueryEntity.java @@ -38,6 +38,8 @@ import org.graylog2.contentpacks.model.entities.references.ValueReference; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.plugin.streams.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -54,6 +56,7 @@ import static com.google.common.collect.ImmutableSortedSet.of; import static java.util.stream.Collectors.toSet; import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject; +import static org.graylog2.shared.utilities.StringUtils.f; @AutoValue @JsonAutoDetect @@ -61,6 +64,8 @@ @JsonDeserialize(builder = QueryEntity.Builder.class) public abstract class QueryEntity implements NativeEntityConverter { + private static final Logger LOG = LoggerFactory.getLogger(QueryEntity.class); + @JsonProperty public abstract String id(); @@ -150,15 +155,27 @@ private Filter shallowMappedFilter(Map nativeEntities) .map(filter -> { if (filter.type().matches(StreamFilter.NAME)) { final StreamFilter streamFilter = (StreamFilter) filter; - final Stream stream = (Stream) resolveStreamEntityObject(streamFilter.streamId(), nativeEntities); - if (Objects.isNull(stream)) { - throw new ContentPackException("Could not find matching stream id: " + - streamFilter.streamId()); + final Object object = resolveStreamEntityObject(streamFilter.streamId(), nativeEntities); + if (object == null) { + // Skip a dangling stream reference instead of aborting the whole content pack + // installation. This mirrors the export side, which also drops unresolvable references. + LOG.warn("Skipping unresolvable stream reference <{}> in query filter for query <{}> during content pack installation", + streamFilter.streamId(), id()); + return null; + } else if (object instanceof final Stream stream) { + return streamFilter.toBuilder().streamId(stream.getId()).build(); + } else { + // A non-null, non-Stream value indicates a corrupt entity map rather than a + // missing reference, so abort the installation like the other view resolvers. + throw new ContentPackException( + f("Invalid type for stream <%s> in query <%s>: %s", + streamFilter.streamId(), id(), object.getClass())); } - return streamFilter.toBuilder().streamId(stream.getId()).build(); } return filter; - }).collect(Collectors.toSet()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); return optFilter.toGenericBuilder().filters(newFilters).build(); }) .orElse(null); diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SearchTypeEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SearchTypeEntity.java index f83fe6238e66..ce2c085df668 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SearchTypeEntity.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/SearchTypeEntity.java @@ -29,6 +29,8 @@ import org.graylog2.contentpacks.exceptions.ContentPackException; import org.graylog2.contentpacks.model.entities.references.ValueReference; import org.graylog2.plugin.streams.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.util.Collections; @@ -40,6 +42,7 @@ import java.util.stream.Collectors; import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject; +import static org.graylog2.shared.utilities.StringUtils.f; @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -49,6 +52,8 @@ defaultImpl = SearchTypeEntity.Fallback.class) @JsonAutoDetect public interface SearchTypeEntity extends NativeEntityConverter { + Logger LOG = LoggerFactory.getLogger(SearchTypeEntity.class); + String TYPE_FIELD = "type"; String FIELD_SEARCH_FILTERS = "filters"; @@ -207,17 +212,29 @@ public SearchType toNativeEntity(Map parameters, Map mappedStreams(Map nativeEntities) { return streams().stream() - .map(id -> resolveStreamEntityObject(id, nativeEntities)) - .map(object -> { - if (object == null) { - throw new ContentPackException("Missing Stream for event definition"); - } else if (object instanceof Stream) { - Stream stream = (Stream) object; - return stream.getId(); - } else { - throw new ContentPackException( - "Invalid type for stream Stream for event definition: " + object.getClass()); - } - }).collect(Collectors.toSet()); + .map(streamId -> resolveStreamReference(streamId, nativeEntities)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toSet()); + } + + /** + * Resolves a single stream reference from the search type's {@code streams} set to the native stream id. + *

+ * A reference that cannot be resolved (e.g. a dangling reference left over in an exported content pack) is + * skipped with a warning rather than aborting the whole content pack installation. This mirrors the export + * side, which also silently drops stream references it cannot map. + */ + default Optional resolveStreamReference(String streamId, Map nativeEntities) { + final Object object = resolveStreamEntityObject(streamId, nativeEntities); + if (object == null) { + LOG.warn("Skipping unresolvable stream reference <{}> for search type <{}> during content pack installation", streamId, id()); + return Optional.empty(); + } else if (object instanceof Stream) { + return Optional.of(((Stream) object).getId()); + } else { + throw new ContentPackException( + f("Invalid type for stream <%s> in search type <%s>: %s", streamId, id(), object.getClass())); + } } } diff --git a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/WidgetEntity.java b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/WidgetEntity.java index 9320de67c7e8..c938dbc53511 100644 --- a/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/WidgetEntity.java +++ b/graylog2-server/src/main/java/org/graylog2/contentpacks/model/entities/WidgetEntity.java @@ -51,6 +51,8 @@ import org.graylog2.contentpacks.model.entities.references.ValueReference; import org.graylog2.plugin.indexer.searches.timeranges.TimeRange; import org.graylog2.plugin.streams.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import javax.annotation.Nullable; import java.util.Collections; @@ -64,10 +66,13 @@ import java.util.stream.Collectors; import static org.graylog2.contentpacks.facades.StreamReferenceFacade.resolveStreamEntityObject; +import static org.graylog2.shared.utilities.StringUtils.f; @AutoValue @JsonDeserialize(builder = WidgetEntity.Builder.class) public abstract class WidgetEntity implements NativeEntityConverter { + private static final Logger LOG = LoggerFactory.getLogger(WidgetEntity.class); + public static final String FIELD_ID = "id"; public static final String FIELD_TYPE = "type"; public static final String FIELD_FILTER = "filter"; @@ -181,17 +186,10 @@ public WidgetDTO toNativeEntity(Map parameters, Map filter.toNativeEntity(parameters, nativeEntities)).toList()) .id(this.id()) .streams(this.streams().stream() - .map(id -> resolveStreamEntityObject(id, nativeEntities)) - .map(object -> { - if (object == null) { - throw new ContentPackException("Missing Stream for widget entity"); - } else if (object instanceof final Stream stream) { - return stream.getId(); - } else { - throw new ContentPackException( - "Invalid type for stream Stream for event definition: " + object.getClass()); - } - }).collect(Collectors.toSet())) + .map(streamId -> resolveStreamReference(streamId, nativeEntities)) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toSet())) .streamCategories(this.streamCategories()) .type(this.type()); if (this.query().isPresent()) { @@ -203,6 +201,27 @@ public WidgetDTO toNativeEntity(Map parameters, Map + * A reference that cannot be resolved (e.g. a dangling reference left over in an exported content pack) is + * skipped with a warning rather than aborting the whole content pack installation. This mirrors the export + * side ({@link org.graylog.plugins.views.search.views.WidgetDTO#toContentPackEntity}), which also silently + * drops stream references it cannot map. + */ + private Optional resolveStreamReference(String streamId, Map nativeEntities) { + final Object object = resolveStreamEntityObject(streamId, nativeEntities); + if (object == null) { + LOG.warn("Skipping unresolvable stream reference <{}> for widget <{}> during content pack installation", streamId, id()); + return Optional.empty(); + } else if (object instanceof final Stream stream) { + return Optional.of(stream.getId()); + } else { + throw new ContentPackException( + f("Invalid type for stream <%s> in widget <%s>: %s", streamId, id(), object.getClass())); + } + } + @Override public void resolveForInstallation(EntityV1 entity, Map parameters, diff --git a/graylog2-server/src/test/java/org/graylog2/contentpacks/model/entities/QueryEntityTest.java b/graylog2-server/src/test/java/org/graylog2/contentpacks/model/entities/QueryEntityTest.java index 008255c0bb7f..925ea8b22b50 100644 --- a/graylog2-server/src/test/java/org/graylog2/contentpacks/model/entities/QueryEntityTest.java +++ b/graylog2-server/src/test/java/org/graylog2/contentpacks/model/entities/QueryEntityTest.java @@ -17,19 +17,28 @@ package org.graylog2.contentpacks.model.entities; import com.google.common.collect.ImmutableList; +import org.graylog.plugins.views.search.Filter; import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString; +import org.graylog.plugins.views.search.filter.OrFilter; +import org.graylog.plugins.views.search.filter.StreamFilter; import org.graylog.plugins.views.search.searchfilters.model.DBSearchFilter; import org.graylog.plugins.views.search.searchfilters.model.InlineQueryStringSearchFilter; import org.graylog.plugins.views.search.searchfilters.model.ReferencedQueryStringSearchFilter; import org.graylog.plugins.views.search.searchfilters.model.UsedSearchFilter; +import org.graylog2.contentpacks.exceptions.ContentPackException; import org.graylog2.contentpacks.model.ModelTypes; import org.graylog2.plugin.indexer.searches.timeranges.RelativeRange; +import org.graylog2.plugin.streams.Stream; import org.junit.jupiter.api.Test; import java.util.Collections; import java.util.Map; +import java.util.Set; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; class QueryEntityTest { @@ -76,6 +85,62 @@ public void testLoadsSearchFiltersCollectionFromContentPack() { .isEqualTo(expectedSearchFilters); } + @Test + public void dropsUnresolvableStreamFilterInsteadOfFailing() { + final QueryEntity query = QueryEntity.Builder + .createWithDefaults() + .id("nvmd") + .timerange(RelativeRange.allTime()) + .query(ElasticsearchQueryString.empty()) + .filter(OrFilter.or(StreamFilter.ofId("missing-stream-id"))) + .build(); + + assertThat(query.toNativeEntity(Collections.emptyMap(), Collections.emptyMap()).filter().filters()) + .noneMatch(f -> f instanceof StreamFilter); + } + + @Test + public void keepsResolvableStreamFilterAndDropsUnresolvable() { + final Stream stream = mock(Stream.class); + when(stream.getId()).thenReturn("native-stream-id"); + final Map nativeEntities = + Map.of(EntityDescriptor.create("cp-stream-id", ModelTypes.STREAM_REF_V1), stream); + + final QueryEntity query = QueryEntity.Builder + .createWithDefaults() + .id("nvmd") + .timerange(RelativeRange.allTime()) + .query(ElasticsearchQueryString.empty()) + .filter(OrFilter.or(StreamFilter.ofId("cp-stream-id"), StreamFilter.ofId("missing-stream-id"))) + .build(); + + final Set mappedFilters = + query.toNativeEntity(Collections.emptyMap(), nativeEntities).filter().filters(); + + assertThat(mappedFilters).hasSize(1); + assertThat(((StreamFilter) mappedFilters.iterator().next()).streamId()).isEqualTo("native-stream-id"); + } + + @Test + public void failsOnWrongTypeStreamReference() { + // A non-null value that is not a Stream indicates a corrupt entity map, which must still abort the install. + final Map nativeEntities = + Map.of(EntityDescriptor.create("cp-stream-id", ModelTypes.STREAM_REF_V1), "not-a-stream"); + + final QueryEntity query = QueryEntity.Builder + .createWithDefaults() + .id("nvmd") + .timerange(RelativeRange.allTime()) + .query(ElasticsearchQueryString.empty()) + .filter(OrFilter.or(StreamFilter.ofId("cp-stream-id"))) + .build(); + + assertThatThrownBy(() -> query.toNativeEntity(Collections.emptyMap(), nativeEntities)) + .isInstanceOf(ContentPackException.class) + .hasMessageContaining("cp-stream-id") + .hasMessageContaining("nvmd"); + } + private class TestDBSearchFilter implements DBSearchFilter { String id; diff --git a/graylog2-server/src/test/java/org/graylog2/contentpacks/model/entities/StreamReferenceResolutionTest.java b/graylog2-server/src/test/java/org/graylog2/contentpacks/model/entities/StreamReferenceResolutionTest.java new file mode 100644 index 000000000000..1df4fc84fea6 --- /dev/null +++ b/graylog2-server/src/test/java/org/graylog2/contentpacks/model/entities/StreamReferenceResolutionTest.java @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2020 Graylog, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package org.graylog2.contentpacks.model.entities; + +import com.google.common.collect.ImmutableSet; +import org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString; +import org.graylog.plugins.views.search.views.widgets.messagelist.MessageListConfigDTO; +import org.graylog2.contentpacks.exceptions.ContentPackException; +import org.graylog2.contentpacks.model.ModelTypes; +import org.graylog2.plugin.indexer.searches.timeranges.KeywordRange; +import org.graylog2.plugin.streams.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Shared behaviour for the view-side content pack resolvers that map a set of stream references to native stream ids + * during installation. {@link WidgetEntity} and {@link SearchTypeEntity} implement the same contract, so their cases + * are parameterized over a {@link StreamResolver} per subject. {@code QueryEntity} resolves stream references inside + * filters rather than a stream set and is covered separately in {@code QueryEntityTest}. + */ +class StreamReferenceResolutionTest { + + /** + * Maps a subject's content pack stream references to the resolved native stream ids, or throws a + * {@link ContentPackException} when the entity map holds a non-{@link Stream} value for a reference. + */ + @FunctionalInterface + interface StreamResolver { + Set resolve(String subjectId, Set streamRefs, Map nativeEntities); + } + + static List subjects() { + return List.of( + Arguments.of("widget", (StreamResolver) StreamReferenceResolutionTest::resolveViaWidget), + Arguments.of("searchType", (StreamResolver) StreamReferenceResolutionTest::resolveViaSearchType)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("subjects") + void dropsUnresolvableStreamReferenceInsteadOfFailing(String subject, StreamResolver resolver) { + assertThat(resolver.resolve("subject-id", Set.of("missing-stream-id"), Collections.emptyMap())) + .isEmpty(); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("subjects") + void keepsResolvableStreamReferenceAndDropsUnresolvable(String subject, StreamResolver resolver) { + assertThat(resolver.resolve("subject-id", Set.of("cp-stream-id", "missing-stream-id"), + nativeStream("cp-stream-id", "native-stream-id"))) + .containsExactly("native-stream-id"); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("subjects") + void failsOnWrongTypeStreamReference(String subject, StreamResolver resolver) { + // A non-null value that is not a Stream indicates a corrupt entity map, which must still abort the install. + final Map nativeEntities = + Map.of(EntityDescriptor.create("cp-stream-id", ModelTypes.STREAM_REF_V1), "not-a-stream"); + + assertThatThrownBy(() -> resolver.resolve("subject-id", Set.of("cp-stream-id"), nativeEntities)) + .isInstanceOf(ContentPackException.class) + .hasMessageContaining("cp-stream-id") + .hasMessageContaining("subject-id"); + } + + private static Map nativeStream(String contentPackId, String nativeId) { + final Stream stream = mock(Stream.class); + when(stream.getId()).thenReturn(nativeId); + return Map.of(EntityDescriptor.create(contentPackId, ModelTypes.STREAM_REF_V1), stream); + } + + private static Set resolveViaWidget(String subjectId, Set streamRefs, + Map nativeEntities) { + final WidgetEntity widget = WidgetEntity.builder() + .id(subjectId) + .type(MessageListConfigDTO.NAME) + .filters(Collections.emptyList()) + .timerange(KeywordRange.create("last 5 minutes", "Etc/UTC")) + .query(ElasticsearchQueryString.of("*")) + .streams(streamRefs) + .config(MessageListConfigDTO.Builder.builder() + .fields(ImmutableSet.of()) + .showMessageRow(false) + .build()) + .build(); + return widget.toNativeEntity(Collections.emptyMap(), nativeEntities).streams(); + } + + private static Set resolveViaSearchType(String subjectId, Set streamRefs, + Map nativeEntities) { + final MessageListEntity searchType = MessageListEntity.builder() + .id(subjectId) + .streams(ImmutableSet.copyOf(streamRefs)) + .build(); + return searchType.toNativeEntity(Collections.emptyMap(), nativeEntities).streams(); + } +}