libs/ingest-csv.js
function processFiles(bucket, key, transform, cb, currentFileNum=0, lastFileNum=0, arn=null, retries=0) {
  const maxRetries = 5

  var nextFileNum = (currentFileNum < lastFileNum) ? currentFileNum + 1 : null
  //invokeLambda(bucket, key, currentFileNum, lastFileNum, arn)

  processFile(
    bucket, `${key}${currentFileNum}.csv`, transform
  ).then((n_scenes) => {
    invokeLambda(bucket, key, nextFileNum, lastFileNum, arn, 0)
    cb()
  }).catch((e) => {
    // if CSV failed, try it again
    if (retries < maxRetries) {
      invokeLambda(bucket, key, currentFileNum, lastFileNum, arn, retries + 1)
    } else {
      // log and move onto the next one
      console.log(`error: maxRetries hit in file ${currentFileNum}`)
      invokeLambda(bucket, key, nextFileNum, lastFileNum, arn, 0)
    }
    cb()
  })
}
function processFiles(bucket, key, transform, cb, currentFileNum=0, lastFileNum=0, arn=null, retries=0) {
  const maxRetries = 5

  var nextFileNum = (currentFileNum < lastFileNum) ? currentFileNum + 1 : null
  //invokeLambda(bucket, key, currentFileNum, lastFileNum, arn)

  processFile(
    bucket, `${key}${currentFileNum}.csv`, transform
  ).then((n_scenes) => {
    invokeLambda(bucket, key, nextFileNum, lastFileNum, arn, 0)
    cb()
  }).catch((e) => {
    // if CSV failed, try it again
    if (retries < maxRetries) {
      invokeLambda(bucket, key, currentFileNum, lastFileNum, arn, retries + 1)
    } else {
      // log and move onto the next one
      console.log(`error: maxRetries hit in file ${currentFileNum}`)
      invokeLambda(bucket, key, nextFileNum, lastFileNum, arn, 0)
    }
    cb()
  })
}
function processFiles ( bucket , key , transform , cb , currentFileNum = 0 , lastFileNum = 0 , arn = null , retries = 0 ) { const maxRetries = 5 var nextFileNum = ( currentFileNum < lastFileNum ) ? currentFileNum + 1 : null //invokeLambda(bucket, key, currentFileNum, lastFileNum, arn) processFile ( bucket , ` ${ key } ${ currentFileNum } ` , transform ) . then ( ( n_scenes ) => { invokeLambda ( bucket , key , nextFileNum , lastFileNum , arn , 0 ) cb ( ) } ) . catch ( ( e ) => { // if CSV failed, try it again if ( retries < maxRetries ) { invokeLambda ( bucket , key , currentFileNum , lastFileNum , arn , retries + 1 ) } else { // log and move onto the next one console . log ( ` ${ currentFileNum } ` ) invokeLambda ( bucket , key , nextFileNum , lastFileNum , arn , 0 ) } cb ( ) } ) }
Process 1 or more CSV files by processing one at a time, then invoking the next
Process 1 or more CSV files by processing one at a time then invoking the next