ingest failures will propagate to a lambda failure (#664)

* ingest error will return an error from the lambda

* changelog

Author: Phil Varner

CHANGELOG.md

@@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Changed
 
+- Ingest lambda will return a failure if there are errors during ingesting. Previously,
+  errors would only be logged and the lambda would always return a success response.
 - Landing page (/) and collections endpoint (/collections) now return a 500 if
   create_indices has not been run or cannot connect to database.
 

src/lambdas/ingest/index.js

@@ -61,8 +61,16 @@ export const handler = async (event, _context) => {
     : [event]
 
   try {
+    logger.debug('Attempting to ingest %d items', stacItems.length)
+
     const results = await ingestItems(stacItems)
-    logger.debug('Ingested %d items: %j', stacItems.length, stacItems)
+
+    const errorCount = results.filter((result) => result.error).length
+    if (errorCount) {
+      logger.debug('There were %d errors ingesting %d items', errorCount, stacItems.length)
+    } else {
+      logger.debug('Ingested %d items', results.length)
+    }
 
     const postIngestTopicArn = process.env['POST_INGEST_TOPIC_ARN']
 
@@ -72,6 +80,8 @@ export const handler = async (event, _context) => {
     } else {
       logger.debug('Skipping post-ingest notification since no topic is configured')
     }
+
+    if (errorCount) throw new Error('There was at least one error ingesting items.')
   } catch (error) {
     logger.error(error)
     throw (error)

src/lib/ingest.js

@@ -132,7 +132,7 @@ function logIngestItemsResults(results) {
       if (result.error instanceof InvalidIngestError) {
         // Attempting to ingest invalid stac objects is not a system error so we
         // log it as info and not error
-        logger.info('Invalid ingest item', result.error)
+        logger.warn('Invalid ingest item', result.error)
       } else {
         logger.error('Error while ingesting item', result.error)
       }

tests/helpers/ingest.js

@@ -74,7 +74,7 @@ export const ingestFixtureC = (ingestTopicArn, ingestQueueUrl) =>
     overrides
   })
 
-export async function testPostIngestSNS(t, record) {
+export async function testPostIngestSNS(t, record, shouldError = false) {
   // @ts-ignore
   process.env.POST_INGEST_TOPIC_ARN = t.context.postIngestTopicArn
 
@@ -83,7 +83,13 @@ export async function testPostIngestSNS(t, record) {
     Message: JSON.stringify(record)
   })
 
-  await sqsTriggerLambda(t.context.ingestQueueUrl, handler)
+  try {
+    await sqsTriggerLambda(t.context.ingestQueueUrl, handler)
+  } catch (e) {
+    if (!shouldError) {
+      t.fail('Ingest had error, but should not have.')
+    }
+  }
 
   const { Messages } = await sqs().receiveMessage({
     QueueUrl: t.context.postIngestQueueUrl,

tests/system/test-ingest.js

@@ -392,7 +392,7 @@ test('Ingest collection failure is published to post-ingest SNS topic', async (t
   const { message, attrs } = await testPostIngestSNS(t, {
     type: 'Collection',
     id: 'badCollection'
-  })
+  }, true)
 
   t.is(message.record.id, 'badCollection')
   t.is(attrs.collection.Value, 'badCollection')
@@ -482,7 +482,7 @@ test('Ingest item failure is published to post-ingest SNS topic', async (t) => {
     type: 'Feature',
     id: 'badItem',
     collection: collection.id
-  })
+  }, true)
 
   t.is(message.record.id, 'badItem')
   t.is(attrs.collection.Value, collection.id)
@@ -514,7 +514,7 @@ test('Ingested item is published to post-ingest SNS topic with updated links', a
   }
 })
 
-test('Ingested item facilure is published to post-ingest SNS topic without updated links', async (t) => {
+test('Ingested item failure is published to post-ingest SNS topic without updated links', async (t) => {
   const envBeforeTest = { ...process.env }
   try {
     const hostname = 'some-stac-server.com'
@@ -526,7 +526,7 @@ test('Ingested item facilure is published to post-ingest SNS topic without updat
       { id: randomId('item'), collection: 'INVALID COLLECTION' }
     )
 
-    const { message } = await testPostIngestSNS(t, item)
+    const { message } = await testPostIngestSNS(t, item, true)
 
     t.truthy(message.record.links)
     t.false(message.record.links.every((/** @type {Link} */ link) => (