Prod v6 fix (#908)

* enable communities removed and delete queries updated

* nginx security headers

* removed parameter to include chunk nodes as entity source

* Ready to reprocess

* communities fix

* dependency fix Reprocess fix

* icon size fix

* z index fix

* icon fix

* icon gap fix

* upload status

* Checkbox fix

---------

Co-authored-by: kaustubh-darekar <[email protected]>
Co-authored-by: kartikpersistent <[email protected]>
Co-authored-by: Prakriti Solankey <[email protected]>

Author: 4 people

backend/score.py

@@ -806,7 +806,7 @@ async def retry_processing(uri=Form(), userName=Form(), password=Form(), databas
         graph = create_graph_database_connection(uri, userName, password, database)
         await asyncio.to_thread(set_status_retry, graph,file_name,retry_condition)
         #set_status_retry(graph,file_name,retry_condition)
-        return create_api_response('Success',message=f"Status set to Reprocess for filename : {file_name}")
+        return create_api_response('Success',message=f"Status set to Ready to Reprocess for filename : {file_name}")
     except Exception as e:
         job_status = "Failed"
         message="Unable to set status to Retry"

backend/src/graphDB_dataAccess.py

@@ -187,14 +187,11 @@ def check_gds_version(self):
             result = self.graph.query(gds_procedure_count)
             total_gds_procedures = result[0]['totalGdsProcedures'] if result else 0
 
-            enable_communities = os.environ.get('ENABLE_COMMUNITIES','').upper() == "TRUE"
-            logging.info(f"Enable Communities {enable_communities}")
-
-            if enable_communities and total_gds_procedures > 0:
+            if total_gds_procedures > 0:
                 logging.info("GDS is available in the database.")
                 return True
             else:
-                logging.info("Communities are disabled or GDS is not available in the database.")
+                logging.info("GDS is not available in the database.")
                 return False
         except Exception as e:
             logging.error(f"An error occurred while checking GDS version: {e}")
@@ -277,27 +274,29 @@ def delete_file_from_graph(self, filenames, source_types, deleteEntities:str, me
             return count(*) as deletedChunks
             """
         query_to_delete_document_and_entities="""
-            match (d:Document) where d.fileName IN $filename_list and d.fileSource in $source_types_list
-            detach delete d
-            with collect(d) as documents
-            unwind documents as d
-            match (d)<-[:PART_OF]-(c:Chunk)
-            detach delete c
-            with *
-            match (c)-[:HAS_ENTITY]->(e)
-            where not exists { (e)<-[:HAS_ENTITY]-()-[:PART_OF]->(d2) where not d2 in documents }
-            detach delete e
-            """ 
+            MATCH (d:Document)
+            WHERE d.fileName IN $filename_list AND d.fileSource IN $source_types_list
+            WITH COLLECT(d) as documents
+            UNWIND documents AS d
+            MATCH (d)<-[:PART_OF]-(c:Chunk)
+            WITH d, c, documents
+            OPTIONAL MATCH (c)-[:HAS_ENTITY]->(e)
+            WHERE NOT EXISTS {
+                MATCH (e)<-[:HAS_ENTITY]-(c2)-[:PART_OF]->(d2:Document)
+                WHERE NOT d2 IN documents
+                }
+            DETACH DELETE c, e, d
+            """  
         query_to_delete_communities = """
             MATCH (c:`__Community__`) 
             WHERE NOT EXISTS { ()-[:IN_COMMUNITY]->(c) } AND c.level = 0 
             DETACH DELETE c 
 
             WITH *
             UNWIND range(1, $max_level) AS level
-            MATCH (c:`__Community__`) 
-            WHERE c.level = level AND NOT EXISTS { (c)<-[:PARENT_COMMUNITY]-(child) } 
-            DETACH DELETE c
+            MATCH (c1:`__Community__`) 
+            WHERE c1.level = level AND NOT EXISTS { (c1)<-[:PARENT_COMMUNITY]-(child) } 
+            DETACH DELETE c1
         """   
         param = {"filename_list" : filename_list, "source_types_list": source_types_list}
         community_param = {"max_level":MAX_COMMUNITY_LEVELS}

backend/src/main.py

@@ -705,7 +705,7 @@ def populate_graph_schema_from_text(text, model, is_schema_description_cheked):
 def set_status_retry(graph, file_name, retry_condition):
     graphDb_data_Access = graphDBdataAccess(graph)
     obj_source_node = sourceNode()
-    status = "Reprocess"
+    status = "Ready to Reprocess"
     obj_source_node.file_name = file_name
     obj_source_node.status = status
     obj_source_node.retry_condition = retry_condition

backend/src/shared/common_fn.py

@@ -94,7 +94,7 @@ def load_embedding_model(embedding_model_name: str):
     return embeddings, dimension
 
 def save_graphDocuments_in_neo4j(graph:Neo4jGraph, graph_document_list:List[GraphDocument]):
-  graph.add_graph_documents(graph_document_list, baseEntityLabel=True,include_source=True)
+  graph.add_graph_documents(graph_document_list, baseEntityLabel=True)
   # graph.add_graph_documents(graph_document_list)
 
 def handle_backticks_nodes_relationship_id_type(graph_document_list:List[GraphDocument]):

docker-compose.yml

@@ -63,11 +63,12 @@ services:
         - VITE_BATCH_SIZE=${VITE_BATCH_SIZE-2}
         - VITE_LLM_MODELS=${VITE_LLM_MODELS-}
         - VITE_LLM_MODELS_PROD=${VITE_LLM_MODELS_PROD-openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash}
+        - DEPLOYMENT_ENV=local
     volumes:
       - ./frontend:/app
       - /app/node_modules
-    # env_file:
-    #   - ./frontend/.env
+    env_file:
+      - ./frontend/.env
     container_name: frontend
     ports:
       - "8080:8080"

docs/backend/backend_docs.adoc

@@ -923,22 +923,22 @@ The API is used to drop and create the vector index when vector index dimesion a
 POST /retry_processing
 ----
 
-This API is used to reprocess cancelled, completed or failed file sources.
-Users have 3 options to reprocess files:
+This API is used to Ready to Reprocess cancelled, completed or failed file sources.
+Users have 3 options to Ready to Reprocess files:
 
 * Start from begnning - In this condition file will be processed from the begnning i.e. 1st chunk again.
 * Delete entities and start from begnning - If the file source is already processed and have any existing nodes and relations then those will be deleted and file will be reprocessed from the 1st chunk.
 * Start from last processed postion - Cancelled or failed files will be processed from the last successfully processed chunk position. This option is not available for completed files.
 
-Ones the status is set to 'Reprocess', user can again click on Generate graph to process the file for knowledge graph creation.
+Ones the status is set to 'Ready to Reprocess', user can again click on Generate graph to process the file for knowledge graph creation.
 
 **API Parameters :**
 
 * `uri`=Neo4j uri,
 * `userName`= Neo4j db username,
 * `password`= Neo4j db password,
 * `database`= Neo4j database name,
-* `file_name`= Name of the file which user want to reprocess.
+* `file_name`= Name of the file which user want to Ready to Reprocess.
 * `retry_condition` = One of the above 3 conditions which is selected for reprocessing.
 
 
@@ -947,7 +947,7 @@ Ones the status is set to 'Reprocess', user can again click on Generate graph to
 ....
 {
     "status": "Success",
-    "message": "Status set to Reprocess for filename : $filename"
+    "message": "Status set to Ready to Reprocess for filename : $filename"
 }
 ....
 

frontend/Dockerfile

@@ -28,13 +28,16 @@ RUN VITE_BACKEND_API_URL=$VITE_BACKEND_API_URL \
     VITE_LARGE_FILE_SIZE=${VITE_LARGE_FILE_SIZE} \
     VITE_CHAT_MODES=$VITE_CHAT_MODES \
     VITE_BATCH_SIZE=$VITE_BATCH_SIZE \
+    VITE_LLM_MODELS=$VITE_LLM_MODELS \
     VITE_LLM_MODELS_PROD=$VITE_LLM_MODELS_PROD \
     yarn run build
 
 # Step 2: Serve the application using Nginx
 FROM nginx:alpine
+ARG DEPLOYMENT_ENV="local"
+ENV DEPLOYMENT_ENV=$DEPLOYMENT_ENV
 COPY --from=build /app/dist /usr/share/nginx/html
-COPY nginx/nginx.conf /etc/nginx/conf.d/default.conf
+COPY /nginx/nginx.${DEPLOYMENT_ENV}.conf /etc/nginx/templates/nginx.conf.template
 
 EXPOSE 8080
-CMD ["nginx", "-g", "daemon off;"]
+CMD ["nginx", "-g", "daemon off;"]

frontend/nginx/nginx.conf renamed to frontend/nginx/nginx.local.conf

@@ -0,0 +1,22 @@
+server {
+  listen 8080;
+  add_header X-Frame-Options "DENY";
+  add_header X-Content-Type-Options "nosniff";
+  add_header Content-Security-Policy   "connect-src 'self' ${VITE_BACKEND_API_URL} ${VITE_SEGMENT_API_URL};
+                                        frame-src 'self' *.youtube.com *.wikipedia.org;
+                                        script-src 'self' 'unsafe-inline' https://accounts.google.com/gsi/client;
+                                        default-src 'self' *.${VITE_FRONTEND_HOSTNAME} data:;
+                                        style-src 'self' *.googleapis.com 'unsafe-inline';" always ;
+  gzip on;
+  location / {
+    root /usr/share/nginx/html;
+    index index.html index.htm;
+    try_files $uri $uri/ /index.html;
+  }
+
+    error_page 401 403 404 index.html;   
+
+    location /public {
+        root /usr/local/var/www;
+    }
+}

frontend/nginx/nginx.prod.conf

@@ -15,11 +15,12 @@
     "@mui/material": "^5.15.10",
     "@mui/styled-engine": "^5.15.9",
     "@neo4j-devtools/word-color": "^0.0.8",
-    "@neo4j-ndl/base": "^2.12.7",
-    "@neo4j-ndl/react": "^2.16.9",
-    "@neo4j-nvl/base": "^0.3.3",
-    "@neo4j-nvl/react": "^0.3.3",
+    "@neo4j-ndl/base": "^3.0.16",
+    "@neo4j-ndl/react": "^3.0.30",
+    "@neo4j-nvl/base": "^0.3.6",
+    "@neo4j-nvl/react": "^0.3.6",
     "@react-oauth/google": "^0.12.1",
+    "@tanstack/react-table": "^8.20.5",
     "@types/uuid": "^9.0.7",
     "axios": "^1.6.5",
     "clsx": "^2.1.1",