delete orphaned indexed equipments (#571)

Delete orphaned indexed equipments

Signed-off-by: achour94 <[email protected]>

Author: achour94

src/main/java/org/gridsuite/study/server/SupervisionController.java

@@ -11,8 +11,10 @@
 import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import io.swagger.v3.oas.annotations.responses.ApiResponses;
 import io.swagger.v3.oas.annotations.tags.Tag;
+import org.gridsuite.study.server.service.StudyService;
 import org.gridsuite.study.server.service.SupervisionService;
 
+import java.util.List;
 import java.util.UUID;
 
 import org.gridsuite.study.server.dto.ComputationType;
@@ -42,8 +44,11 @@ public class SupervisionController {
 
     private final SupervisionService supervisionService;
 
-    public SupervisionController(SupervisionService supervisionService) {
+    private final StudyService studyService;
+
+    public SupervisionController(SupervisionService supervisionService, StudyService studyService) {
         this.supervisionService = supervisionService;
+        this.studyService = studyService;
     }
 
     @DeleteMapping(value = "/computation/results")
@@ -96,6 +101,21 @@ public ResponseEntity<Long> deleteStudyIndexedEquipmentsAndTombstoned(@PathVaria
         return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(supervisionService.deleteStudyIndexedEquipmentsAndTombstoned(studyUuid));
     }
 
+    @GetMapping(value = "/orphan_indexed_network_uuids")
+    @Operation(summary = "Get all orphan indexed equipments network uuids")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "The list of orphan indexed equipments network uuids")})
+    public ResponseEntity<List<UUID>> getOrphanIndexedEquipments() {
+        return ResponseEntity.ok().contentType(MediaType.APPLICATION_JSON).body(studyService.getAllOrphanIndexedEquipmentsNetworkUuids());
+    }
+
+    @DeleteMapping(value = "/studies/{networkUuid}/indexed-equipments-by-network-uuid")
+    @Operation(summary = "delete indexed equipments and tombstoned equipments for the given networkUuid")
+    @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "all indexed equipments and tombstoned equipments for the given networkUuid have been deleted")})
+    public ResponseEntity<Long> deleteNetworkUuidIndexedEquipmentsAndTombstoned(@PathVariable("networkUuid") UUID networkUuid) {
+        studyService.deleteEquipmentIndexes(networkUuid);
+        return ResponseEntity.ok().build();
+    }
+
     @DeleteMapping(value = "/studies/{studyUuid}/nodes/builds")
     @Operation(summary = "Invalidate node builds for the given study")
     @ApiResponses(value = {@ApiResponse(responseCode = "200", description = "all built nodes for the given study have been invalidated")})

src/main/java/org/gridsuite/study/server/dto/BasicEquipmentInfos.java

@@ -8,6 +8,7 @@
 
 import io.swagger.v3.oas.annotations.media.Schema;
 import lombok.*;
+import lombok.experimental.FieldNameConstants;
 import lombok.experimental.SuperBuilder;
 import org.springframework.data.annotation.AccessType;
 import org.springframework.data.annotation.Id;
@@ -27,6 +28,7 @@
 @Setter
 @ToString
 @EqualsAndHashCode
+@FieldNameConstants
 @Schema(description = "Basic equipment infos")
 public class BasicEquipmentInfos {
     @Id

src/main/java/org/gridsuite/study/server/elasticsearch/EquipmentInfosService.java

@@ -7,18 +7,23 @@
 package org.gridsuite.study.server.elasticsearch;
 
 import co.elastic.clients.elasticsearch._types.FieldValue;
+import co.elastic.clients.elasticsearch._types.aggregations.*;
+import co.elastic.clients.elasticsearch._types.aggregations.Aggregation;
 import co.elastic.clients.elasticsearch._types.query_dsl.*;
 
 import com.powsybl.iidm.network.VariantManagerConstants;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.gridsuite.study.server.dto.BasicEquipmentInfos;
 import org.gridsuite.study.server.dto.EquipmentInfos;
 import org.gridsuite.study.server.dto.TombstonedEquipmentInfos;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.data.domain.PageRequest;
-import org.springframework.data.elasticsearch.client.elc.NativeQuery;
-import org.springframework.data.elasticsearch.client.elc.NativeQueryBuilder;
-import org.springframework.data.elasticsearch.client.elc.Queries;
+import org.springframework.data.elasticsearch.client.elc.*;
 import org.springframework.data.elasticsearch.core.ElasticsearchOperations;
 import org.springframework.data.elasticsearch.core.SearchHit;
+import org.springframework.data.elasticsearch.core.SearchHits;
 import org.springframework.lang.NonNull;
 import org.springframework.stereotype.Service;
 
@@ -42,6 +47,8 @@ public enum FieldSelector {
 
     private static final int PAGE_MAX_SIZE = 400;
 
+    private static final int COMPOSITE_AGGREGATION_BATCH_SIZE = 1000;
+
     public static final Map<String, Integer> EQUIPMENT_TYPE_SCORES = Map.ofEntries(
             entry("SUBSTATION", 15),
             entry("VOLTAGE_LEVEL", 14),
@@ -73,6 +80,8 @@ public enum FieldSelector {
 
     private final ElasticsearchOperations elasticsearchOperations;
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(EquipmentInfosService.class);
+
     public EquipmentInfosService(EquipmentInfosRepository equipmentInfosRepository, TombstonedEquipmentInfosRepository tombstonedEquipmentInfosRepository, ElasticsearchOperations elasticsearchOperations) {
         this.equipmentInfosRepository = equipmentInfosRepository;
         this.tombstonedEquipmentInfosRepository = tombstonedEquipmentInfosRepository;
@@ -106,6 +115,101 @@ public long getEquipmentInfosCount() {
         return equipmentInfosRepository.count();
     }
 
+    private CompositeAggregation buildCompositeAggregation(String field, Map<String, FieldValue> afterKey) {
+        List<Map<String, CompositeAggregationSource>> sources = List.of(
+                Map.of(field, CompositeAggregationSource.of(s -> s.terms(t -> t.field(field + ".keyword")))
+                )
+        );
+
+        CompositeAggregation.Builder compositeAggregationBuilder = new CompositeAggregation.Builder()
+                .size(COMPOSITE_AGGREGATION_BATCH_SIZE)
+                .sources(sources);
+
+        if (afterKey != null) {
+            compositeAggregationBuilder.after(afterKey);
+        }
+
+        return compositeAggregationBuilder.build();
+    }
+
+    /**
+     * Constructs a NativeQuery with a composite aggregation.
+     *
+     * @param compositeName The name of the composite aggregation.
+     * @param compositeAggregation The composite aggregation configuration.
+     * @return A NativeQuery object configured with the specified composite aggregation.
+     */
+    private NativeQuery buildCompositeAggregationQuery(String compositeName, CompositeAggregation compositeAggregation) {
+        Aggregation aggregation = Aggregation.of(a -> a.composite(compositeAggregation));
+
+        return new NativeQueryBuilder()
+                .withAggregation(compositeName, aggregation)
+                .build();
+    }
+
+    /**
+     * This method is used to extract the results of a composite aggregation from Elasticsearch search hits.
+     *
+     * @param searchHits The search hits returned from an Elasticsearch query.
+     * @param compositeName The name of the composite aggregation.
+     * @return A Pair consisting of two elements:
+     *         The left element of the Pair is a list of maps, where each map represents a bucket's key. Each bucket is a result of the composite aggregation.
+     *         The right element of the Pair is the afterKey map, which is used for pagination in Elasticsearch.
+     *         If there are no more pages, the afterKey will be null.
+     * @see <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-composite-aggregation.html">Elasticsearch Composite Aggregation Documentation</a>
+     */
+    private Pair<List<Map<String, FieldValue>>, Map<String, FieldValue>> extractCompositeAggregationResults(SearchHits<EquipmentInfos> searchHits, String compositeName) {
+        ElasticsearchAggregations aggregations = (ElasticsearchAggregations) searchHits.getAggregations();
+
+        List<Map<String, FieldValue>> results = new ArrayList<>();
+        if (aggregations != null) {
+            Map<String, ElasticsearchAggregation> aggregationList = aggregations.aggregationsAsMap();
+            if (!aggregationList.isEmpty()) {
+                Aggregate aggregate = aggregationList.get(compositeName).aggregation().getAggregate();
+                if (aggregate.isComposite() && aggregate.composite() != null) {
+                    for (CompositeBucket bucket : aggregate.composite().buckets().array()) {
+                        Map<String, FieldValue> key = bucket.key();
+                        results.add(key);
+                    }
+                    return Pair.of(results, aggregate.composite().afterKey());
+                }
+            }
+        }
+        return Pair.of(results, null);
+    }
+
+    public List<UUID> getEquipmentInfosDistinctNetworkUuids() {
+        List<UUID> networkUuids = new ArrayList<>();
+        Map<String, FieldValue> afterKey = null;
+        String compositeName = "composite_agg";
+        String networkUuidField = BasicEquipmentInfos.Fields.networkUuid;
+
+        do {
+            CompositeAggregation compositeAggregation = buildCompositeAggregation(networkUuidField, afterKey);
+            NativeQuery query = buildCompositeAggregationQuery(compositeName, compositeAggregation);
+
+            SearchHits<EquipmentInfos> searchHits = elasticsearchOperations.search(query, EquipmentInfos.class);
+            Pair<List<Map<String, FieldValue>>, Map<String, FieldValue>> searchResults = extractCompositeAggregationResults(searchHits, compositeName);
+
+            searchResults.getLeft().stream()
+                    .map(result -> result.get(networkUuidField))
+                    .filter(Objects::nonNull)
+                    .map(FieldValue::stringValue)
+                    .map(UUID::fromString)
+                    .forEach(networkUuids::add);
+
+            afterKey = searchResults.getRight();
+        } while (afterKey != null && !afterKey.isEmpty());
+
+        return networkUuids;
+    }
+
+    public List<UUID> getOrphanEquipmentInfosNetworkUuids(List<UUID> networkUuidsInDatabase) {
+        List<UUID> networkUuids = getEquipmentInfosDistinctNetworkUuids();
+        networkUuids.removeAll(networkUuidsInDatabase);
+        return networkUuids;
+    }
+
     public long getTombstonedEquipmentInfosCount() {
         return tombstonedEquipmentInfosRepository.count();
     }

src/main/java/org/gridsuite/study/server/service/StudyService.java

@@ -238,6 +238,16 @@ public List<CreatedStudyBasicInfos> getStudies() {
                 .collect(Collectors.toList());
     }
 
+    public List<UUID> getStudiesNetworkUuids() {
+        return studyRepository.findAll().stream()
+                .map(StudyEntity::getNetworkUuid)
+                .toList();
+    }
+
+    public List<UUID> getAllOrphanIndexedEquipmentsNetworkUuids() {
+        return equipmentInfosService.getOrphanEquipmentInfosNetworkUuids(getStudiesNetworkUuids());
+    }
+
     public String getStudyCaseName(UUID studyUuid) {
         Objects.requireNonNull(studyUuid);
         StudyEntity study = studyRepository.findById(studyUuid).orElseThrow(() -> new StudyException(STUDY_NOT_FOUND));

src/test/java/org/gridsuite/study/server/EquipmentInfosServiceTests.java

@@ -186,6 +186,38 @@ public void testDeleteAllEquipmentInfos() {
         assertEquals(0, equipmentInfosService.getEquipmentInfosCount());
     }
 
+    @Test
+    public void testGetOrphanEquipmentInfosNetworkUuids() {
+        // index some equipment infos as orphan
+        UUID orphanNetworkUuid = UUID.randomUUID();
+        EquipmentInfos orphanLoadInfos = EquipmentInfos.builder().networkUuid(orphanNetworkUuid).id("id").name("name").type("LOAD").voltageLevels(Set.of(VoltageLevelInfos.builder().id("vl").name("vl").build())).build();
+        UUID orphanNetworkUuid2 = UUID.randomUUID();
+        EquipmentInfos orphanVlInfos = EquipmentInfos.builder().networkUuid(orphanNetworkUuid2).id("id").name("name").type("VOLTAGE_LEVEL").voltageLevels(Set.of(VoltageLevelInfos.builder().id("vl").name("vl").build())).build();
+
+        // index an equipment infos for the existing network
+        EquipmentInfos loadInfos = EquipmentInfos.builder().networkUuid(NETWORK_UUID).id("id2").name("name2").type("LOAD").voltageLevels(Set.of(VoltageLevelInfos.builder().id("vl").name("vl").build())).build();
+
+        equipmentInfosService.addEquipmentInfos(loadInfos);
+        equipmentInfosService.addEquipmentInfos(orphanLoadInfos);
+        equipmentInfosService.addEquipmentInfos(orphanVlInfos);
+
+        // get the orphan network uuids
+        List<UUID> orphanNetworkUuids = equipmentInfosService.getOrphanEquipmentInfosNetworkUuids(List.of(NETWORK_UUID));
+
+        // check that the orphan network uuids are returned
+        assertEquals(2, orphanNetworkUuids.size());
+        assertTrue(orphanNetworkUuids.contains(orphanNetworkUuid));
+        assertTrue(orphanNetworkUuids.contains(orphanNetworkUuid2));
+
+        // delete the orphan equipment infos
+        equipmentInfosService.deleteAllByNetworkUuid(orphanNetworkUuid);
+        equipmentInfosService.deleteAllByNetworkUuid(orphanNetworkUuid2);
+
+        // check that the orphan network uuids are not returned anymore
+        orphanNetworkUuids = equipmentInfosService.getOrphanEquipmentInfosNetworkUuids(List.of(NETWORK_UUID));
+        assertEquals(0, orphanNetworkUuids.size());
+    }
+
     @Test
     public void testCloneVariant() {
         equipmentInfosService.addEquipmentInfos(EquipmentInfos.builder().networkUuid(NETWORK_UUID).id("id1").name("name1").type(IdentifiableType.LOAD.name()).variantId("variant1").voltageLevels(Set.of(VoltageLevelInfos.builder().id("vl1").name("vl1").build())).build());

src/test/java/org/gridsuite/study/server/StudyTest.java

@@ -182,6 +182,11 @@ public class StudyTest {
 
     private static final String DEFAULT_PROVIDER = "defaultProvider";
 
+    private static final List<UUID> ORPHAN_NETWORK_UUIDS = List.of(
+            UUID.fromString("88888888-7777-0000-abcd-000000000000"),
+            UUID.fromString("123e4567-e89b-12d3-a456-426614174000")
+    );
+
     @Value("${non-evacuated-energy.default-provider}")
     String defaultNonEvacuatedEnergyProvider;
 
@@ -298,6 +303,8 @@ private void initMockBeans(Network network) {
         when(networkStoreService.getNetwork(NETWORK_UUID)).thenReturn(network);
 
         doNothing().when(networkStoreService).deleteNetwork(NETWORK_UUID);
+
+        when(equipmentInfosService.getOrphanEquipmentInfosNetworkUuids(List.of(NETWORK_UUID))).thenReturn(ORPHAN_NETWORK_UUIDS);
     }
 
     private void initMockBeansNetworkNotExisting(Network notExistingNetwork) {
@@ -2614,6 +2621,18 @@ public void testSupervision() throws Exception {
         buildStatusMessage = output.receive(TIMEOUT, studyUpdateDestination);
         assertEquals(studyUuid, buildStatusMessage.getHeaders().get(NotificationService.HEADER_STUDY_UUID));
         assertEquals(NotificationService.NODE_BUILD_STATUS_UPDATED, buildStatusMessage.getHeaders().get(HEADER_UPDATE_TYPE));
+
+        // Test get orphan indexed equipments
+        mvcResult = mockMvc.perform(get("/v1/supervision/orphan_indexed_network_uuids"))
+                .andExpect(status().isOk())
+                .andReturn();
+
+        List<UUID> orphanIndexedEquipments = mapper.readValue(mvcResult.getResponse().getContentAsString(), new TypeReference<>() { });
+        assertEquals(ORPHAN_NETWORK_UUIDS, orphanIndexedEquipments);
+
+        // test delete orphan indexed equipments
+        mockMvc.perform(delete("/v1/supervision/studies/{networkUuid}/indexed-equipments-by-network-uuid", ORPHAN_NETWORK_UUIDS.get(0)))
+                .andExpect(status().isOk());
     }
 
     @After