Use Weaviate built-in client batching

Author: kacperlukawski

engine/clients/weaviate/upload.py

@@ -24,26 +24,20 @@ def init_client(cls, host, distance, connection_params, upload_params):
     def upload_batch(
         cls, ids: List[int], vectors: List[list], metadata: List[Optional[dict]]
     ):
-        try:
+        # Weaviate introduced the batch_size, so it can handle built-in client's
+        # multi-threading. That should make the upload faster.
+        cls.client.batch.configure(
+            batch_size=100,
+            timeout_retries=3,
+        )
+
+        with cls.client.batch as batch:
             for id_, vector, data_object in zip(ids, vectors, metadata):
-                cls.client.batch.add_data_object(
+                batch.add_data_object(
                     data_object=data_object or {},
                     class_name=WEAVIATE_CLASS_NAME,
                     uuid=uuid.UUID(int=id_).hex,
                     vector=vector,
                 )
 
-            cls.client.batch.create_objects()
-        except Exception as e:
-            # It crashed unpredicsibly, so we probably need to do "try again thing"
-            # Issue to monitor: https://github.com/semi-technologies/weaviate/issues/2066
-            time.sleep(1)
-            for id_, vector, data_object in zip(ids, vectors, metadata):
-                cls.client.batch.add_data_object(
-                    data_object=data_object or {},
-                    class_name=WEAVIATE_CLASS_NAME,
-                    uuid=uuid.UUID(int=id_).hex,
-                    vector=vector,
-                )
-
-            cls.client.batch.create_objects()
+            batch.create_objects()