Description: Api Gateway method, that calls lambda function, which gets lead from S3 export and sends it to LMS for reprocess. Search lead in exported S3 files

Parameters:
  ApiGateway:
    Type: String
    Description: LMS API Gateway
  ApiKey:
    Type: String
    Description: LMS API Key
  CompanyName:
    Type: String
    Description: Insert company name in lower case without spaces

Resources:
  S3Bucket:
    Type: "AWS::S3::Bucket"
    Properties:
      BucketName: !Sub "boberdooexport-${CompanyName}"

  IAMUser:
    DependsOn:
      - ReprocessLeadRequest
      - S3Bucket
    Type: "AWS::IAM::User"
    Properties:
      Path: /
      Policies:
        - PolicyName: !Sub "boberdooexport-${CompanyName}"
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - "kms:Encrypt"
                  - "kms:GenerateDataKey"
                Resource:
                  - !GetAtt KmsKey.Arn
              - Effect: Allow
                Action:
                  - "s3:PutObject"
                  - "s3:PutObjectAcl"
                Resource: !Sub "arn:aws:s3:::boberdooexport-${CompanyName}/*"

              - Effect: Allow
                Action:
                  - "execute-api:Invoke"
                Resource:
                  - 'Fn::Join':
                      - ''
                      - - 'arn:aws:execute-api:'
                        - !Ref AWS::Region
                        - ':'
                        - !Ref AWS::AccountId
                        - ':'
                        - !Ref 'LeadActionsApi'
                        - '/*/POST/reprocessLead'
      UserName: !Sub "boberdoo-${CompanyName}-exporting-user"

  IAMUserAccessKey:
    Type: AWS::IAM::AccessKey
    Properties:
      UserName:
        !Ref IAMUser

  IAMUserSearch:
    DependsOn:
      - SearchLeadRequest
    Type: "AWS::IAM::User"
    Properties:
      Path: /
      Policies:
        - PolicyName: !Sub "boberdoosearch-${CompanyName}"
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - "execute-api:Invoke"
                Resource:
                  - 'Fn::Join':
                      - ''
                      - - 'arn:aws:execute-api:'
                        - !Ref AWS::Region
                        - ':'
                        - !Ref AWS::AccountId
                        - ':'
                        - !Ref 'LeadActionsApi'
                        - '/*/POST/searchLead'
      UserName: !Sub "boberdoo-${CompanyName}-searching-user"
  
  IAMUserSearchAccessKey:
    Type: AWS::IAM::AccessKey
    Properties:
      UserName:
        !Ref IAMUserSearch
      
  KmsKey:
    Type: "AWS::KMS::Key"
    Properties:
      Description: Boberdoo exporting Key
      Enabled: True
      KeyPolicy:
        Version: "2012-10-17"
        Id: "key-default-1"
        Statement:
          - Sid: "Enable IAM User Permissions"
            Effect: "Allow"
            Principal: 
              AWS: !Sub "arn:aws:iam::${AWS::AccountId}:root"
            Action:
              - "kms:*"
            Resource: "*"
          - Sid: "Allow use of the key"
            Effect: "Allow"
            Principal: 
              AWS: !Sub "arn:aws:iam::${AWS::AccountId}:root"
            Action: 
              - "kms:Encrypt"
              - "kms:Decrypt"
              - "kms:ReEncrypt*"
              - "kms:GenerateDataKey*"
              - "kms:DescribeKey"
            Resource: "*"
          - Sid: "Allow attachment of persistent resources"
            Effect: "Allow"
            Principal: 
              AWS: !Sub "arn:aws:iam::${AWS::AccountId}:root"
            Action:
              - "kms:CreateGrant"
              - "kms:ListGrants"
              - "kms:RevokeGrant"
            Resource: "*"
            Condition:
              Bool:
                kms:GrantIsForAWSResource: "true"
          - Sid: "Allow administration of the key"
            Effect: "Allow"
            Principal: 
              AWS: !Sub "arn:aws:iam::${AWS::AccountId}:root"
            Action:
              - "kms:Create*"
              - "kms:Describe*"
              - "kms:Enable*"
              - "kms:List*"
              - "kms:Put*"
              - "kms:Update*"
              - "kms:Revoke*"
              - "kms:Disable*"
              - "kms:Get*"
              - "kms:Delete*"
              - "kms:TagResource"
              - "kms:UntagResource"
              - "kms:ScheduleKeyDeletion"
              - "kms:CancelKeyDeletion"
            Resource: "*"
  KmsKeyAlias:
    DependsOn:
      - KmsKey
    Type: "AWS::KMS::Alias"
    Properties:
      AliasName: alias/boberdooexport
      TargetKeyId: !Ref KmsKey

  LambdaFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: reprocessLeadS3
      Code:
        ZipFile: |
            import json
            import boto3
            import csv
            import os
            from botocore.vendored import requests
            from dateutil.parser import parse
            from botocore.exceptions import ClientError
  
            s3 = boto3.client('s3')
  
            def lambda_handler(event, context):
                try:
                    leadID            = event['body-json']['leadID']
                    leadTypeID        = event['body-json']['leadTypeID']
                    dateCreated       = event['body-json']['dateCreated']
                    bucket            = event['body-json']['bucket']
                    quotes            = event['body-json']['quotes']
                    delimiter         = event['body-json']['delimiter']
                    ignoreExclusivity = event['body-json']['ignoreExclusivity']
                    alowedTimesSold   = event['body-json']['alowedTimesSold']
                except KeyError as e :
                    return respond("Missing parameter %s" % e.args[0], True)
  
                apiKey     = os.environ['API_KEY']
                apiGateway = os.environ['API_GATEWAY']
  
                quoting       = '"' if quotes == 1 else ''
                date          = parse(dateCreated)
                paths         = []
                selectResults = {}
                select = "select * from S3Object s where s.Lead_Type_ID = '{}' AND s.Lead_ID = '{}'".format(leadTypeID, leadID)
                paths.append("%s/%s/%s/%s/%s-%s.csv" % (leadTypeID, date.year, date.month, date.day, leadID, leadTypeID))
                paths.append("%s/%s/%s/%s/%s_%s.csv" % (leadTypeID, date.year, date.month, date.day, date.strftime("%Y%m%d"), leadTypeID))
  
                for path in paths :
                    try :
                        response = s3.select_object_content(
                            Bucket=bucket,
                            Key=path,
                            ExpressionType = "SQL",
                            Expression = select,
                            InputSerialization = {
                                'CompressionType': 'NONE',
                                'CSV': {
                                    'FileHeaderInfo': 'USE',
                                    'RecordDelimiter': '\n',
                                    'FieldDelimiter': delimiter,
                                    'QuoteCharacter': quoting
                                }
                            },
                            OutputSerialization = {
                                'JSON': {}
                            }
                        )
  
                        for event in response['Payload']:
                            if 'Records' in event:
                                # selectResults[file] = list(filter(None, event['Records']['Payload'].decode('utf-8').split("\n")))
                                body = event['Records']['Payload'].decode('utf-8')
  
                    except ClientError as e :
                        # Can not read file
                        continue
                    except Exception as e :
                        return respond('Error getting object: ' + str(e), True)
  
                try:
                    # build post data from result body
                    postData = json.loads(body);
  
                    # Remove date created from post data
                    postData.pop('Date_Created', None)
                    postData.pop('Date_Created_Hour', None)
                    postData.pop('Day_Of_Week', None)
                    leadTypeIDPop = postData.pop('Lead_Type_ID', None)
  
                    # Add api key, api action and format to post data
                    postData.update({'Key' : apiKey, 'Format' : 'JSON', 'API_Action' : 'updateLeadAndReprocess', 'Lambda_Call' : 1, 'Ignore_Exclusivity' : ignoreExclusivity, 'Allowed_Times_Sold' : alowedTimesSold, 'TYPE' : leadTypeIDPop})
  
                    # Post lead for reprocess to LMS API
                    result = requests.post(apiGateway, data=postData)
  
                    return respond(result.text)
                except requests.exceptions.RequestException as e :
                    return respond('Failed to connect to LMS API url: {}'.format(apiGateway), True)
                except Exception as e:
                    return respond('Error: ' + str(e), True)
  
            def respond(message, error = False) :
                if error :
                    response = {
                        "status" : "error",
                         "data"   : message
                    }
                else :
                    response = {
                        "status" : "success",
                        "data"   : message
                    }
  
                return response
      Environment:
        Variables:
          API_GATEWAY: !Ref ApiGateway
          API_KEY: !Ref ApiKey
      Description: Parse Lead From Amazon S3 csv export and send to LMS for reprocess
      Handler: index.lambda_handler
      Role: !GetAtt 'LambdaExecutionRole.Arn'
      Runtime: python3.6
      Timeout: '20'

  LambdaExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      Policies:
        - PolicyName: lambda-inline
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                  - kms:Decrypt
                Resource: '*'
              - Effect: Allow
                Action:
                  - s3:GetObject
                Resource: !Sub "arn:aws:s3:::boberdooexport-${CompanyName}/*"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Path: /
      
  SearchFunction:
    Type: AWS::Lambda::Function
    Properties:
      FunctionName: searchLeadS3
      Code:
        ZipFile: |
          import boto3
          import json
          import datetime
          from dateutil.parser import parse
          from botocore.exceptions import ClientError

          def lambda_handler(event, context):
              try:
                  leadTypeID        = event['body-json']['leadTypeID']
                  bucket            = event['body-json']['bucket']
                  quotes            = event['body-json']['quotes']
                  delimiter         = event['body-json']['delimiter']
                  dateFrom          = event['body-json']['dateFrom']
                  dateTo            = event['body-json']['dateTo']
                  searchField       = event['body-json']['searchField']
                  searchValue       = event['body-json']['searchValue']
                  searchSubStrings  = event['body-json']['searchSubStrings']
              except KeyError as e :
                  return respond("Missing parameter %s" % e.args[0], True)

              if searchSubStrings :
                  select = "select * from S3Object s where s.{} LIKE '%{}%'".format(searchField, searchValue)
              else :
                  select = "select * from S3Object s where s.{} = '{}'".format(searchField, searchValue)

              files         = getSearchFiles(leadTypeID, dateFrom, dateTo)
              selectResults = {}
              s3            = boto3.client('s3')

              for file in files :
                  try :
                      response = s3.select_object_content(
                          Bucket=bucket,
                          Key=file,
                          ExpressionType = "SQL",
                          Expression = select,
                          InputSerialization = {
                              'CompressionType': 'NONE',
                              'CSV': {
                                  'FileHeaderInfo': 'USE',
                                  'RecordDelimiter': '\n',
                                  'FieldDelimiter': delimiter,
                                  'QuoteCharacter': quotes
                              }
                          },
                          OutputSerialization = {
                              'JSON': {}
                          }
                      )

                      for event in response['Payload']:
                          if 'Records' in event:
                              selectResults[file] = list(filter(None, event['Records']['Payload'].decode('utf-8').split("\n")))

                  except ClientError as e :
                      # Can not read file
                      continue
                  except Exception as e :
                      return respond('Failed to run query: ' + str(e), True)

              return respond(json.dumps(selectResults))
    
          def respond(message, error = False) :
              if error :
                  response = {
                      "status" : "error",
                      "data"   : message
                  }
              else :
                  response = {
                      "status" : "success",
                      "data"   : message
                  }

              return response
        
          def getSearchFiles(leadTypeID, dateFrom, dateTo) :
              result         = []
              start          = datetime.datetime.strptime(dateFrom, "%Y-%m-%d")
              end            = datetime.datetime.strptime(dateTo, "%Y-%m-%d")
              delta          = end - start
              date_generated = [start + datetime.timedelta(days=x) for x in range(0, (end-start).days+1)]

              for date in date_generated:
                  result.append("%s/%s/%s/%s/%s_%s.csv" % (leadTypeID, date.year, date.month, date.day, date.strftime("%Y%m%d"), leadTypeID))

              return result
      Description: Search Lead in Amazon S3 csv export
      Handler: index.lambda_handler
      Role: !GetAtt 'SearchExecutionRole.Arn'
      Runtime: python3.6
      Timeout: '40'

  SearchExecutionRole:
    Type: AWS::IAM::Role
    Properties:
      Policies:
        - PolicyName: lambda-inline
          PolicyDocument:
            Version: '2012-10-17'
            Statement:
              - Effect: Allow
                Action:
                  - logs:CreateLogGroup
                  - logs:CreateLogStream
                  - logs:PutLogEvents
                  - kms:Decrypt
                Resource: '*'
              - Effect: Allow
                Action:
                  - s3:GetObject
                Resource: !Sub "arn:aws:s3:::boberdooexport-${CompanyName}/*"
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - lambda.amazonaws.com
            Action:
              - sts:AssumeRole
      Path: /

  LeadActionsApi:
    Type: 'AWS::ApiGateway::RestApi'
    Properties:
      Name: Lead Actions API
      Description: API used to handle actions over exported Leads
      FailOnWarnings: true
  LambdaPermission2:
    Type: 'AWS::Lambda::Permission'
    Properties:
      Action: 'lambda:invokeFunction'
      FunctionName:
        'Fn::GetAtt':
          - LambdaFunction
          - Arn
      Principal: apigateway.amazonaws.com
      SourceArn:
        'Fn::Join':
          - ''
          - - 'arn:aws:execute-api:'
            - Ref: 'AWS::Region'
            - ':'
            - Ref: 'AWS::AccountId'
            - ':'
            - Ref: LeadActionsApi
            - /*
  SearchPermission2:
    Type: 'AWS::Lambda::Permission'
    Properties:
      Action: 'lambda:invokeFunction'
      FunctionName:
        'Fn::GetAtt':
          - SearchFunction
          - Arn
      Principal: apigateway.amazonaws.com
      SourceArn:
        'Fn::Join':
          - ''
          - - 'arn:aws:execute-api:'
            - Ref: 'AWS::Region'
            - ':'
            - Ref: 'AWS::AccountId'
            - ':'
            - Ref: LeadActionsApi
            - /*
  ApiGatewayCloudWatchLogsRole:
    Type: 'AWS::IAM::Role'
    Properties:
      AssumeRolePolicyDocument:
        Version: 2012-10-17
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - apigateway.amazonaws.com
            Action:
              - 'sts:AssumeRole'
      Policies:
        - PolicyName: ApiGatewayLogsPolicy
          PolicyDocument:
            Version: 2012-10-17
            Statement:
              - Effect: Allow
                Action:
                  - 'logs:CreateLogGroup'
                  - 'logs:CreateLogStream'
                  - 'logs:DescribeLogGroups'
                  - 'logs:DescribeLogStreams'
                  - 'logs:PutLogEvents'
                  - 'logs:GetLogEvents'
                  - 'logs:FilterLogEvents'
                Resource: '*'
  ApiGatewayAccount:
    Type: 'AWS::ApiGateway::Account'
    Properties:
      CloudWatchRoleArn:
        'Fn::GetAtt':
          - ApiGatewayCloudWatchLogsRole
          - Arn
  LeadActionsApiStage:
    DependsOn:
      - ApiGatewayAccount
    Type: 'AWS::ApiGateway::Stage'
    Properties:
      DeploymentId:
        Ref: ApiDeployment
      MethodSettings:
          - HttpMethod: '*'
            ResourcePath: /*
      RestApiId:
        Ref: LeadActionsApi
      StageName: prod
  ApiDeployment:
    Type: 'AWS::ApiGateway::Deployment'
    DependsOn:
      - ReprocessLeadRequest
      - SearchLeadRequest
    Properties:
      RestApiId:
        Ref: LeadActionsApi
  ReprocessLeadResource:
    Type: 'AWS::ApiGateway::Resource'
    Properties:
      RestApiId:
        Ref: LeadActionsApi
      ParentId:
        'Fn::GetAtt':
          - LeadActionsApi
          - RootResourceId
      PathPart: reprocessLead
  SearchLeadResource:
    Type: 'AWS::ApiGateway::Resource'
    Properties:
      RestApiId:
        Ref: LeadActionsApi
      ParentId:
        'Fn::GetAtt':
          - LeadActionsApi
          - RootResourceId
      PathPart: searchLead
  ReprocessLeadRequest:
    DependsOn: LambdaPermission2
    Type: 'AWS::ApiGateway::Method'
    Properties:
      AuthorizationType: AWS_IAM
      HttpMethod: POST
      Integration:
        Type: AWS
        IntegrationHttpMethod: POST
        PassthroughBehavior: WHEN_NO_TEMPLATES
        RequestTemplates:
          "application/json": |
            #set($allParams = $input.params())
            {
            "body-json" : $input.json('$'),
            "params" : {
            #foreach($type in $allParams.keySet())
                #set($params = $allParams.get($type))
            "$type" : {
                #foreach($paramName in $params.keySet())
                "$paramName" : "$util.escapeJavaScript($params.get($paramName))"
                    #if($foreach.hasNext),#end
                #end
            }
                #if($foreach.hasNext),#end
            #end
            },
            "stage-variables" : {
            #foreach($key in $stageVariables.keySet())
            "$key" : "$util.escapeJavaScript($stageVariables.get($key))"
                #if($foreach.hasNext),#end
            #end
            },
            "context" : {
                "account-id" : "$context.identity.accountId",
                "api-id" : "$context.apiId",
                "api-key" : "$context.identity.apiKey",
                "authorizer-principal-id" : "$context.authorizer.principalId",
                "caller" : "$context.identity.caller",
                "cognito-authentication-provider" : "$context.identity.cognitoAuthenticationProvider",
                "cognito-authentication-type" : "$context.identity.cognitoAuthenticationType",
                "cognito-identity-id" : "$context.identity.cognitoIdentityId",
                "cognito-identity-pool-id" : "$context.identity.cognitoIdentityPoolId",
                "http-method" : "$context.httpMethod",
                "stage" : "$context.stage",
                "source-ip" : "$context.identity.sourceIp",
                "user" : "$context.identity.user",
                "user-agent" : "$context.identity.userAgent",
                "user-arn" : "$context.identity.userArn",
                "request-id" : "$context.requestId",
                "resource-id" : "$context.resourceId",
                "resource-path" : "$context.resourcePath"
                }
            }

        Uri:
          'Fn::Join':
            - ''
            - - 'arn:aws:apigateway:'
              - Ref: 'AWS::Region'
              - ':lambda:path/2015-03-31/functions/'
              - 'Fn::GetAtt':
                  - LambdaFunction
                  - Arn
              - /invocations
        IntegrationResponses:
          - StatusCode: 200
      ResourceId:
        Ref: ReprocessLeadResource
      RestApiId:
        Ref: LeadActionsApi
      MethodResponses:
        - StatusCode: 200
  SearchLeadRequest:
    DependsOn: SearchPermission2
    Type: 'AWS::ApiGateway::Method'
    Properties:
      AuthorizationType: AWS_IAM
      HttpMethod: POST
      Integration:
        Type: AWS
        IntegrationHttpMethod: POST
        PassthroughBehavior: WHEN_NO_TEMPLATES
        RequestTemplates:
          "application/json": |
            #set($allParams = $input.params())
            {
            "body-json" : $input.json('$'),
            "params" : {
            #foreach($type in $allParams.keySet())
                #set($params = $allParams.get($type))
            "$type" : {
                #foreach($paramName in $params.keySet())
                "$paramName" : "$util.escapeJavaScript($params.get($paramName))"
                    #if($foreach.hasNext),#end
                #end
            }
                #if($foreach.hasNext),#end
            #end
            },
            "stage-variables" : {
            #foreach($key in $stageVariables.keySet())
            "$key" : "$util.escapeJavaScript($stageVariables.get($key))"
                #if($foreach.hasNext),#end
            #end
            },
            "context" : {
                "account-id" : "$context.identity.accountId",
                "api-id" : "$context.apiId",
                "api-key" : "$context.identity.apiKey",
                "authorizer-principal-id" : "$context.authorizer.principalId",
                "caller" : "$context.identity.caller",
                "cognito-authentication-provider" : "$context.identity.cognitoAuthenticationProvider",
                "cognito-authentication-type" : "$context.identity.cognitoAuthenticationType",
                "cognito-identity-id" : "$context.identity.cognitoIdentityId",
                "cognito-identity-pool-id" : "$context.identity.cognitoIdentityPoolId",
                "http-method" : "$context.httpMethod",
                "stage" : "$context.stage",
                "source-ip" : "$context.identity.sourceIp",
                "user" : "$context.identity.user",
                "user-agent" : "$context.identity.userAgent",
                "user-arn" : "$context.identity.userArn",
                "request-id" : "$context.requestId",
                "resource-id" : "$context.resourceId",
                "resource-path" : "$context.resourcePath"
                }
            }

        Uri:
          'Fn::Join':
            - ''
            - - 'arn:aws:apigateway:'
              - Ref: 'AWS::Region'
              - ':lambda:path/2015-03-31/functions/'
              - 'Fn::GetAtt':
                  - SearchFunction
                  - Arn
              - /invocations
        IntegrationResponses:
          - StatusCode: 200
      ResourceId:
        Ref: SearchLeadResource
      RestApiId:
        Ref: LeadActionsApi
      MethodResponses:
        - StatusCode: 200

Outputs:
  KMSKeyId:
    Description: KMS Key ID
    Value: !Ref KmsKey
  S3BucketName:
    Description: S3 Bucket Name
    Value: !Sub "boberdooexport-${CompanyName}"
  Region:
    Description: AWS Region
    Value: !Ref AWS::Region
  AccessKey:
    Description: IAM Access Key
    Value: !Ref IAMUserAccessKey
  SecretKey:
    Description: IAM Secret Key
    Value: !GetAtt IAMUserAccessKey.SecretAccessKey
  AccessKeySearch:
    Description: IAM Search Lead Access Key
    Value: !Ref IAMUserSearchAccessKey
  SecretKeySearch:
    Description: IAM Search Lead Secret Key
    Value: !GetAtt IAMUserSearchAccessKey.SecretAccessKey
  APIGateWayURL:
    Description: Root URL of the API gateway
    Value:
      'Fn::Join':
        - ''
        - - 'https://'
          - Ref: LeadActionsApi
          - .execute-api.
          - Ref: 'AWS::Region'
          - .amazonaws.com
          - '/prod/'
