@@ -150,32 +150,165 @@ func hasLastOperationCondition(status OperationStatus, condType OperationConditi
150150 return false
151151}
152152
153- // isOperationStarting checks if an ml operation status is in starting condition
154- func isOperationStarting (status OperationStatus ) bool {
155- return hasLastOperationCondition (status , OperationStarting )
153+ // IsStarting checks if an ml operation status is in starting condition
154+ func (status * OperationStatus ) IsStarting ( ) bool {
155+ return hasLastOperationCondition (* status , OperationStarting )
156156}
157157
158- // isOperationRunning checks if an ml operation status is in running condition
159- func isOperationRunning (status OperationStatus ) bool {
160- return hasLastOperationCondition (status , OperationRunning )
158+ // IsRunning checks if an ml operation status is in running condition
159+ func (status * OperationStatus ) IsRunning ( ) bool {
160+ return hasLastOperationCondition (* status , OperationRunning )
161161}
162162
163- // isOperationWarning checks if an ml operation status is in warning condition
164- func isOperationWarning (status OperationStatus ) bool {
165- return hasLastOperationCondition (status , OperationWarning )
163+ // IsWarning checks if an ml operation status is in warning condition
164+ func (status * OperationStatus ) IsWarning ( ) bool {
165+ return hasLastOperationCondition (* status , OperationWarning )
166166}
167167
168- // isOperationSucceeded checks if an ml operation status is succeeded
169- func isOperationSucceeded (status OperationStatus ) bool {
170- return hasOperationCondition (status , OperationSucceeded )
168+ // IsSucceeded checks if an ml operation status is succeeded
169+ func (status * OperationStatus ) IsSucceeded ( ) bool {
170+ return hasOperationCondition (* status , OperationSucceeded )
171171}
172172
173- // isOperationFailed checks if an ml operation status is failed
174- func isOperationFailed (status OperationStatus ) bool {
175- return hasOperationCondition (status , OperationFailed )
173+ // IsFailed checks if an ml operation status is failed
174+ func (status * OperationStatus ) IsFailed ( ) bool {
175+ return hasOperationCondition (* status , OperationFailed )
176176}
177177
178- // isOperationStopped checks if an ml operation status is stopped
179- func isOperationStopped (status OperationStatus ) bool {
180- return hasOperationCondition (status , OperationStopped )
178+ // IsStopped checks if an ml operation status is stopped
179+ func (status * OperationStatus ) IsStopped () bool {
180+ return hasOperationCondition (* status , OperationStopped )
181+ }
182+
183+ // IsDone checks if it the Operation reached a final condition
184+ func (status * OperationStatus ) IsDone () bool {
185+ return status .IsSucceeded () || status .IsFailed () || status .IsStopped ()
186+ }
187+
188+ // IsOperationBeingDeleted checks if a Kubernetes resource is being deleted
189+ // This works with any type that implements metav1.Object (Job, Service, Operation, etc.)
190+ func IsOperationBeingDeleted [T metav1.Object ](obj T ) bool {
191+ return ! obj .GetDeletionTimestamp ().IsZero ()
192+ }
193+
194+ // removeCondition removes a condition of the specified type from the status
195+ func removeCondition (status * OperationStatus , conditionType OperationConditionType ) {
196+ var newConditions []OperationCondition
197+ for _ , c := range status .Conditions {
198+ if c .Type == conditionType {
199+ continue
200+ }
201+ newConditions = append (newConditions , c )
202+ }
203+ status .Conditions = newConditions
204+ }
205+
206+ // logCondition logs a condition to the status
207+ func logCondition (status * OperationStatus , condType OperationConditionType , conditionStatus corev1.ConditionStatus , reason , message string ) bool {
208+ currentCond := getLastEntityCondition (* status , condType )
209+ cond , isUpdated := getOrUpdateOperationCondition (currentCond , condType , conditionStatus , reason , message )
210+ if isUpdated && cond != nil {
211+ removeCondition (status , condType )
212+ status .Conditions = append (status .Conditions , * cond )
213+ return true
214+ }
215+ return false
216+ }
217+
218+ // LogStarting sets Operation to starting
219+ func (status * OperationStatus ) LogStarting () bool {
220+ return logCondition (status , OperationStarting , corev1 .ConditionTrue , "OperatorController" , "Operation is starting" )
221+ }
222+
223+ // LogRunning sets Operation to running
224+ func (status * OperationStatus ) LogRunning () bool {
225+ return logCondition (status , OperationRunning , corev1 .ConditionTrue , "OperatorController" , "Operation is running" )
226+ }
227+
228+ // LogWarning sets Operation to succeeded
229+ func (status * OperationStatus ) LogWarning (reason , message string ) bool {
230+ if reason == "" {
231+ reason = "OperatorController"
232+ }
233+ if message == "" {
234+ message = "Underlaying job has an issue"
235+ }
236+ return logCondition (status , OperationWarning , corev1 .ConditionTrue , reason , message )
237+ }
238+
239+ // LogSucceeded sets Operation to succeeded
240+ func (status * OperationStatus ) LogSucceeded () bool {
241+ return logCondition (status , OperationSucceeded , corev1 .ConditionTrue , "OperatorController" , "Operation has succeeded" )
242+ }
243+
244+ // LogFailed sets Operation to failed
245+ func (status * OperationStatus ) LogFailed (reason , message string ) bool {
246+ return logCondition (status , OperationFailed , corev1 .ConditionTrue , reason , message )
247+ }
248+
249+ // LogStopped sets Operation to stopped
250+ func (status * OperationStatus ) LogStopped (reason , message string ) bool {
251+ return logCondition (status , OperationStopped , corev1 .ConditionTrue , reason , message )
252+ }
253+
254+ // ShouldMarkJobAsDeleted checks if a job that doesn't exist should be marked as deleted
255+ // rather than being recreated. It returns true if the job was previously created and running
256+ // but is now missing (deleted externally).
257+ //
258+ // This function checks if the operation has progressed beyond initial creation by examining
259+ // the status conditions. If the last condition indicates the job was running or had warnings,
260+ // it means the job existed before and should not be recreated.
261+ //
262+ // Returns:
263+ // - shouldMarkDeleted: true if the job should be marked as deleted instead of recreated
264+ // - isDone: true if the operation is already in a terminal state
265+ func (status * OperationStatus ) ShouldMarkJobAsDeleted () (shouldMarkDeleted bool , isDone bool ) {
266+ // If operation is already done, don't recreate or mark as deleted
267+ if status .IsDone () {
268+ return false , true
269+ }
270+
271+ // Check if the job was previously created (status has progressed beyond creation)
272+ if len (status .Conditions ) == 0 {
273+ // No conditions mean the job was never created, so it's safe to create it
274+ return false , false
275+ }
276+
277+ lastCond := status .Conditions [len (status .Conditions )- 1 ]
278+
279+ // If the operation was previously running or had warnings, it means
280+ // the job existed before and was deleted externally
281+ if lastCond .Type == OperationRunning || lastCond .Type == OperationWarning {
282+ return true , false
283+ }
284+
285+ // For other states (Starting, etc.), allow job creation
286+ return false , false
287+ }
288+
289+ // MarkJobAsDeleted marks an operation as failed or stopped due to external job deletion.
290+ // This is a helper function that sets the appropriate status when a job is detected
291+ // as deleted externally.
292+ //
293+ // Parameters:
294+ // - jobType: a descriptive name for the job type (e.g., "Kubernetes Job", "TFJob")
295+ // - useFailed: if true, marks as Failed; if false, marks as Stopped
296+ //
297+ // Returns true if the status was updated
298+ func (status * OperationStatus ) MarkJobAsDeleted (jobType string , useFailed bool ) bool {
299+ now := metav1 .Now ()
300+ message := "The underlying " + jobType + " was deleted externally"
301+
302+ var updated bool
303+ if useFailed {
304+ updated = status .LogFailed ("JobDeleted" , message )
305+ } else {
306+ updated = status .LogStopped ("JobDeleted" , message )
307+ }
308+
309+ if updated {
310+ status .CompletionTime = & now
311+ }
312+
313+ return updated
181314}
0 commit comments