Skip to content

Commit 0027df0

Browse files
authored
Stop waiting if InitializationFailed state is reached (#495)
## Problem After creating an index, the SDK polls until the index is ready to be used. But if the index state transitions to `InitializationFailed`, the client will hang. Sometimes this causes our test suite to get stuck and run for hours until somebody manually intervenes or a CI timeout is reached. ## Solution Raise an exception and stop waiting if an index reaches the `InitializationFailed` state. ## Type of Change - [x] Bug fix (non-breaking change which fixes an issue)
2 parents 1ae09e7 + 695125e commit 0027df0

File tree

2 files changed

+37
-63
lines changed

2 files changed

+37
-63
lines changed

pinecone/db_control/resources/asyncio/index.py

Lines changed: 18 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -110,42 +110,28 @@ async def create_from_backup(
110110
return await self.__poll_describe_index_until_ready(name, timeout)
111111

112112
async def __poll_describe_index_until_ready(self, name: str, timeout: Optional[int] = None):
113-
description = None
114-
115-
async def is_ready() -> bool:
116-
nonlocal description
117-
description = await self.describe(name=name)
118-
return description.status.ready
119-
120113
total_wait_time = 0
121-
if timeout is None:
122-
# Wait indefinitely
123-
while not await is_ready():
124-
logger.debug(
125-
f"Waiting for index {name} to be ready. Total wait time {total_wait_time} seconds."
114+
while True:
115+
description = await self.describe(name=name)
116+
if description.status.state == "InitializationFailed":
117+
raise Exception(f"Index {name} failed to initialize.")
118+
if description.status.ready:
119+
return description
120+
121+
if timeout is not None and total_wait_time >= timeout:
122+
logger.error(
123+
f"Index {name} is not ready after {total_wait_time} seconds. Timeout reached."
126124
)
127-
total_wait_time += 5
128-
await asyncio.sleep(5)
125+
link = docslinks["API_DESCRIBE_INDEX"](API_VERSION)
126+
timeout_msg = f"Index {name} is not ready after {total_wait_time} seconds. Please call describe_index() to confirm index status. See docs at {link}"
127+
raise TimeoutError(timeout_msg)
129128

130-
else:
131-
# Wait for a maximum of timeout seconds
132-
while not await is_ready():
133-
if timeout < 0:
134-
logger.error(f"Index {name} is not ready. Timeout reached.")
135-
link = docslinks["API_DESCRIBE_INDEX"](API_VERSION)
136-
timeout_msg = (
137-
f"Please call describe_index() to confirm index status. See docs at {link}"
138-
)
139-
raise TimeoutError(timeout_msg)
140-
141-
logger.debug(
142-
f"Waiting for index {name} to be ready. Total wait time: {total_wait_time}"
143-
)
144-
total_wait_time += 5
145-
await asyncio.sleep(5)
146-
timeout -= 5
129+
logger.debug(
130+
f"Waiting for index {name} to be ready. Total wait time {total_wait_time} seconds."
131+
)
147132

148-
return description
133+
total_wait_time += 5
134+
await asyncio.sleep(5)
149135

150136
@require_kwargs
151137
async def delete(self, *, name: str, timeout: Optional[int] = None):

pinecone/db_control/resources/sync/index.py

Lines changed: 19 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -145,42 +145,30 @@ def create_from_backup(
145145
return self.__poll_describe_index_until_ready(name, timeout)
146146

147147
def __poll_describe_index_until_ready(self, name: str, timeout: Optional[int] = None):
148-
description = None
149-
150-
def is_ready() -> bool:
151-
nonlocal description
152-
description = self.describe(name=name)
153-
return description.status.ready
154-
155148
total_wait_time = 0
156-
if timeout is None:
157-
# Wait indefinitely
158-
while not is_ready():
159-
logger.debug(
160-
f"Waiting for index {name} to be ready. Total wait time {total_wait_time} seconds."
149+
while True:
150+
description = self.describe(name=name)
151+
if description.status.state == "InitializationFailed":
152+
raise Exception(
153+
f"Index {name} failed to initialize. The index status is {description.status.state}."
161154
)
162-
total_wait_time += 5
163-
time.sleep(5)
155+
if description.status.ready:
156+
return description
164157

165-
else:
166-
# Wait for a maximum of timeout seconds
167-
while not is_ready():
168-
if timeout < 0:
169-
logger.error(f"Index {name} is not ready. Timeout reached.")
170-
link = docslinks["API_DESCRIBE_INDEX"](API_VERSION)
171-
timeout_msg = (
172-
f"Please call describe_index() to confirm index status. See docs at {link}"
173-
)
174-
raise TimeoutError(timeout_msg)
175-
176-
logger.debug(
177-
f"Waiting for index {name} to be ready. Total wait time: {total_wait_time}"
158+
if timeout is not None and total_wait_time >= timeout:
159+
logger.error(
160+
f"Index {name} is not ready after {total_wait_time} seconds. Timeout reached."
178161
)
179-
total_wait_time += 5
180-
time.sleep(5)
181-
timeout -= 5
162+
link = docslinks["API_DESCRIBE_INDEX"](API_VERSION)
163+
timeout_msg = f"Index {name} is not ready after {total_wait_time} seconds. Please call describe_index() to confirm index status. See docs at {link}"
164+
raise TimeoutError(timeout_msg)
165+
166+
logger.debug(
167+
f"Waiting for index {name} to be ready. Total wait time {total_wait_time} seconds."
168+
)
182169

183-
return description
170+
total_wait_time += 5
171+
time.sleep(5)
184172

185173
@require_kwargs
186174
def delete(self, *, name: str, timeout: Optional[int] = None) -> None:

0 commit comments

Comments
 (0)