Revert "Issues in Search Relevancy (#21841)" (#21853)

This reverts commit f388e57.

Author: harshach

ingestion/examples/sample_data/datasets/tables.json

@@ -32491,218 +32491,6 @@
       },
       "sourceUrl": "http://localhost:8080/test_long_description_table",
       "tags": []
-    },
-    {
-      "name": "ssot_utilization_detail",
-      "description": "A detail of each agent's time throughout the day. Individual events in each system are collapsed by aux_code, aux_channel, worktype_name, and reporting_date.",
-      "version": 0.5,
-      "updatedAt": 1638354087391,
-      "updatedBy": "anonymous",
-      "tableType": "Regular",
-      "columns": [
-        {
-          "name": "reporting_date",
-          "dataType": "DATE",
-          "dataTypeDisplay": "date",
-          "description": "The date the event began.",
-          "tags": [],
-          "ordinalPosition": 1
-        },
-        {
-          "name": "agent_id",
-          "dataType": "VARCHAR",
-          "dataLength": 100,
-          "dataTypeDisplay": "varchar",
-          "description": "The unique identifier for the agent from customer_success.remitly_employee.",
-          "tags": [],
-          "ordinalPosition": 2
-        },
-        {
-          "name": "agent_email",
-          "dataType": "VARCHAR",
-          "dataLength": 100,
-          "dataTypeDisplay": "varchar",
-          "description": "The agent email.",
-          "tags": [],
-          "ordinalPosition": 3
-        }
-      ],
-      "database": {
-        "id": "50da1ff8-4e1d-4967-8931-45edbf4dd908",
-        "type": "database",
-        "name": "sample_data.ecommerce_db",
-        "description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.",
-        "href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908"
-      },
-      "tags": [],
-      "usageSummary": {
-        "dailyStats": {
-          "count": 0,
-          "percentileRank": 0
-        },
-        "weeklyStats": {
-          "count": 0,
-          "percentileRank": 0
-        },
-        "monthlyStats": {
-          "count": 0,
-          "percentileRank": 0
-        },
-        "date": "2021-12-01"
-      },
-      "followers": []
-    },
-    {
-      "name": "itm_utilization_detail_login_events",
-      "description": "An intermediate table used to create ssot_utilization_detail. This table cleans up the \"CS Platform: login\" event dimensions to be congruent with downstream analytics.",
-      "version": 0.3,
-      "updatedAt": 1638354087391,
-      "updatedBy": "anonymous",
-      "tableType": "Regular",
-      "columns": [
-        {
-          "name": "reporting_date",
-          "dataType": "DATE",
-          "dataTypeDisplay": "date",
-          "description": "Event date",
-          "tags": [],
-          "ordinalPosition": 1
-        },
-        {
-          "name": "agent_id",
-          "dataType": "VARCHAR",
-          "dataLength": 100,
-          "dataTypeDisplay": "varchar",
-          "description": "Agent identifier",
-          "tags": [],
-          "ordinalPosition": 2
-        }
-      ],
-      "database": {
-        "id": "50da1ff8-4e1d-4967-8931-45edbf4dd908",
-        "type": "database",
-        "name": "sample_data.ecommerce_db",
-        "description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.",
-        "href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908"
-      },
-      "tags": [],
-      "usageSummary": {
-        "dailyStats": {
-          "count": 10,
-          "percentileRank": 30
-        },
-        "weeklyStats": {
-          "count": 50,
-          "percentileRank": 40
-        },
-        "monthlyStats": {
-          "count": 200,
-          "percentileRank": 50
-        },
-        "date": "2021-12-01"
-      },
-      "followers": []
-    },
-    {
-      "name": "customer_metrics_daily",
-      "description": "Daily customer metrics aggregated from various sources including ssot_utilization_detail and other operational tables.",
-      "version": 0.2,
-      "updatedAt": 1638354087391,
-      "updatedBy": "anonymous",
-      "tableType": "Regular",
-      "columns": [
-        {
-          "name": "metric_date",
-          "dataType": "DATE",
-          "dataTypeDisplay": "date",
-          "description": "Date of the metric",
-          "tags": [],
-          "ordinalPosition": 1
-        },
-        {
-          "name": "customer_count",
-          "dataType": "INT",
-          "dataTypeDisplay": "integer",
-          "description": "Total number of customers",
-          "tags": [],
-          "ordinalPosition": 2
-        }
-      ],
-      "database": {
-        "id": "50da1ff8-4e1d-4967-8931-45edbf4dd908",
-        "type": "database",
-        "name": "sample_data.ecommerce_db",
-        "description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.",
-        "href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908"
-      },
-      "tags": [],
-      "usageSummary": {
-        "dailyStats": {
-          "count": 100,
-          "percentileRank": 70
-        },
-        "weeklyStats": {
-          "count": 500,
-          "percentileRank": 80
-        },
-        "monthlyStats": {
-          "count": 2000,
-          "percentileRank": 90
-        },
-        "date": "2021-12-01"
-      },
-      "followers": []
-    },
-    {
-      "name": "agent_performance_summary",
-      "description": "Summary of agent performance metrics derived from multiple tables including ssot_utilization_detail for comprehensive reporting.",
-      "version": 0.4,
-      "updatedAt": 1638354087391,
-      "updatedBy": "anonymous",
-      "tableType": "Regular",
-      "columns": [
-        {
-          "name": "agent_id",
-          "dataType": "VARCHAR",
-          "dataLength": 100,
-          "dataTypeDisplay": "varchar",
-          "description": "Agent identifier",
-          "tags": [],
-          "ordinalPosition": 1
-        },
-        {
-          "name": "performance_score",
-          "dataType": "DECIMAL",
-          "dataTypeDisplay": "decimal",
-          "description": "Overall performance score",
-          "tags": [],
-          "ordinalPosition": 2
-        }
-      ],
-      "database": {
-        "id": "50da1ff8-4e1d-4967-8931-45edbf4dd908",
-        "type": "database",
-        "name": "sample_data.ecommerce_db",
-        "description": "This **mock** database contains tables related to shopify sales and orders with related dimension tables.",
-        "href": "http://localhost:8585/api/v1/databases/50da1ff8-4e1d-4967-8931-45edbf4dd908"
-      },
-      "tags": [],
-      "usageSummary": {
-        "dailyStats": {
-          "count": 50,
-          "percentileRank": 60
-        },
-        "weeklyStats": {
-          "count": 250,
-          "percentileRank": 65
-        },
-        "monthlyStats": {
-          "count": 1000,
-          "percentileRank": 70
-        },
-        "date": "2021-12-01"
-      },
-      "followers": []
     }
   ]
 }

openmetadata-service/pom.xml

@@ -948,12 +948,6 @@
       <artifactId>opensearch-rest-high-level-client</artifactId>
       <scope>test</scope>
     </dependency>
-      <dependency>
-          <groupId>org.open-metadata</groupId>
-          <artifactId>opensearch-deps</artifactId>
-          <version>1.8.0-SNAPSHOT</version>
-          <scope>compile</scope>
-      </dependency>
   </dependencies>
 
   <profiles>

openmetadata-service/src/main/java/org/openmetadata/service/search/SearchSourceBuilderFactory.java

@@ -72,20 +72,6 @@ public interface SearchSourceBuilderFactory<S, Q, H, F> {
    * @return a search source builder configured for the specific entity type
    */
   default S getSearchSourceBuilder(String index, String q, int from, int size) {
-    return getSearchSourceBuilder(index, q, from, size, false);
-  }
-
-  /**
-   * Get the appropriate search source builder based on the index name.
-   *
-   * @param index the index name
-   * @param q the search query
-   * @param from the starting offset
-   * @param size the number of results to return
-   * @param explain whether to include explanation of the search results
-   * @return a search source builder configured for the specific entity type
-   */
-  default S getSearchSourceBuilder(String index, String q, int from, int size, boolean explain) {
     String indexName = Entity.getSearchRepository().getIndexNameWithoutAlias(index);
 
     if (isTimeSeriesIndex(indexName)) {
@@ -101,7 +87,7 @@ default S getSearchSourceBuilder(String index, String q, int from, int size, boo
     }
 
     if (isDataAssetIndex(indexName)) {
-      return buildDataAssetSearchBuilder(indexName, q, from, size, explain);
+      return buildDataAssetSearchBuilder(indexName, q, from, size);
     }
 
     if (indexName.equals("all") || indexName.equals("dataAsset")) {
@@ -119,9 +105,6 @@ default S getSearchSourceBuilder(String index, String q, int from, int size, boo
 
   S buildDataAssetSearchBuilder(String indexName, String query, int from, int size);
 
-  S buildDataAssetSearchBuilder(
-      String indexName, String query, int from, int size, boolean explain);
-
   S buildCommonSearchBuilder(String query, int from, int size);
 
   S buildUserOrTeamSearchBuilder(String query, int from, int size);

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java

@@ -366,11 +366,7 @@ public Response doSearch(
         new ElasticSearchSourceBuilderFactory(searchSettings);
     SearchSourceBuilder searchSourceBuilder =
         searchBuilderFactory.getSearchSourceBuilder(
-            request.getIndex(),
-            request.getQuery(),
-            request.getFrom(),
-            request.getSize(),
-            request.getExplain());
+            request.getIndex(), request.getQuery(), request.getFrom(), request.getSize());
 
     buildSearchRBACQuery(subjectContext, searchSourceBuilder);
     // Add Filter

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchSourceBuilderFactory.java

@@ -163,12 +163,6 @@ public SearchSourceBuilder buildAggregateSearchBuilder(String query, int from, i
   @Override
   public SearchSourceBuilder buildDataAssetSearchBuilder(
       String indexName, String query, int from, int size) {
-    return buildDataAssetSearchBuilder(indexName, query, from, size, false);
-  }
-
-  @Override
-  public SearchSourceBuilder buildDataAssetSearchBuilder(
-      String indexName, String query, int from, int size, boolean explain) {
     AssetTypeConfiguration assetConfig = findAssetTypeConfig(indexName, searchSettings);
     Map<String, Float> fuzzyFields;
     Map<String, Float> nonFuzzyFields;
@@ -318,7 +312,7 @@ public SearchSourceBuilder buildDataAssetSearchBuilder(
     }
 
     addConfiguredAggregations(searchSourceBuilder, assetConfig);
-    searchSourceBuilder.explain(explain);
+    searchSourceBuilder.explain(true);
     return searchSourceBuilder;
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java

@@ -19,11 +19,11 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
-import java.util.stream.Collectors;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 import org.openmetadata.schema.EntityInterface;
@@ -128,12 +128,29 @@ default Map<String, Object> getCommonAttributesMap(EntityInterface entity, Strin
   }
 
   default Set<String> getFQNParts(String fqn) {
-    var parts = FullyQualifiedName.split(fqn);
-    var entityName = parts[parts.length - 1];
+    Set<String> fqnParts = new HashSet<>();
+    fqnParts.add(fqn);
+    String parent = FullyQualifiedName.getParentFQN(fqn);
+    while (parent != null) {
+      fqnParts.add(parent);
+      parent = FullyQualifiedName.getParentFQN(parent);
+    }
+    return fqnParts;
+  }
 
-    return FullyQualifiedName.getAllParts(fqn).stream()
-        .filter(part -> !part.equals(entityName))
-        .collect(Collectors.toSet());
+  // Add suggest inputs to fqnParts to support partial/wildcard search on names.
+  // In some case of basic Test suite name is not part of the fullyQualifiedName, so it must be
+  // added separately.
+  default Set<String> getFQNParts(String fqn, List<String> fqnSplits) {
+    Set<String> fqnParts = new HashSet<>();
+    fqnParts.add(fqn);
+    String parent = FullyQualifiedName.getParentFQN(fqn);
+    while (parent != null) {
+      fqnParts.add(parent);
+      parent = FullyQualifiedName.getParentFQN(parent);
+    }
+    fqnParts.addAll(fqnSplits);
+    return fqnParts;
   }
 
   default List<EntityReference> getEntitiesWithDisplayName(List<EntityReference> entities) {

openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java

@@ -137,7 +137,6 @@
 import os.org.opensearch.action.get.GetRequest;
 import os.org.opensearch.action.get.GetResponse;
 import os.org.opensearch.action.search.SearchResponse;
-import os.org.opensearch.action.search.SearchType;
 import os.org.opensearch.action.support.WriteRequest;
 import os.org.opensearch.action.support.master.AcknowledgedResponse;
 import os.org.opensearch.action.update.UpdateRequest;
@@ -385,11 +384,7 @@ public Response doSearch(
         new OpenSearchSourceBuilderFactory(searchSettings);
     SearchSourceBuilder searchSourceBuilder =
         searchBuilderFactory.getSearchSourceBuilder(
-            request.getIndex(),
-            request.getQuery(),
-            request.getFrom(),
-            request.getSize(),
-            request.getExplain());
+            request.getIndex(), request.getQuery(), request.getFrom(), request.getSize());
 
     buildSearchRBACQuery(subjectContext, searchSourceBuilder);
 
@@ -514,10 +509,6 @@ public Response searchWithNLQ(SearchRequest request, SubjectContext subjectConte
           os.org.opensearch.action.search.SearchRequest searchRequest =
               new os.org.opensearch.action.search.SearchRequest(request.getIndex());
           searchRequest.source(searchSourceBuilder);
-
-          // Use DFS Query Then Fetch for consistent scoring across shards
-          searchRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
-
           os.org.opensearch.action.search.SearchResponse response =
               client.search(searchRequest, os.org.opensearch.client.RequestOptions.DEFAULT);
           if (response.getHits() != null
@@ -550,10 +541,6 @@ private Response fallbackToBasicSearch(SearchRequest request, SubjectContext sub
       os.org.opensearch.action.search.SearchRequest osRequest =
           new os.org.opensearch.action.search.SearchRequest(request.getIndex());
       osRequest.source(searchSourceBuilder);
-
-      // Use DFS Query Then Fetch for consistent scoring across shards
-      osRequest.searchType(SearchType.DFS_QUERY_THEN_FETCH);
-
       getSearchBuilderFactory().addAggregationsToNLQQuery(searchSourceBuilder, request.getIndex());
       SearchResponse searchResponse = client.search(osRequest, OPENSEARCH_REQUEST_OPTIONS);
       return Response.status(Response.Status.OK).entity(searchResponse.toString()).build();