AWSTemplateFormatVersion: 2010-09-09
Description: Creates solution to visualize Service Catalog AppRegistry application costs.

Parameters:
# ======================================================================================================================
# AWS CloudFormation Stacks Cost Resources
# ======================================================================================================================

  CollectCostsEveryTimePeriod:
    Description: |
      Schedule how often costs should be collected and Amazon QuickSight dashboard be updated. For example:
      1) Daily: every day. E.g. (Tue, 10 Aug 2021 06:00:00 GMT)
      2) Weekly: once a week, every Sunday. E.g. (Sun, 15 Aug 2021 06:00:00 GMT)
      3) Monthly: once a month, every first day of the month. E.g. (Wed, 01 Sep 2021 06:00:00 GMT)
    Type: String
    Default: "Daily"
    AllowedValues:
      - 'Daily'
      - 'Weekly'
      - 'Monthly'
# ======================================================================================================================
# AppRegistry Attribute Cost Resources
# ======================================================================================================================
  AttributeGroupCostsEnabled:
    Description: |
      You have costs in your AppRegistry attribute groups that you want to be added and reflected
      in the Amazon QuickSight dashboard.
    Type: String
    Default: "false"
    AllowedValues:
      - 'false'
      - 'true'
  AttributeGroupCostsJsonPath:
    Description: |
      Conditional, required if AttributeGroupCostsEnabled set to true. In your AppRegistry attribute groups,
      json path to object where costs are located. For instance: $.costs would be mapped to attribute group
      {"costs": [{"name": "database", "cost": "123"}]}.
    Type: String
    Default: "$.costs"
  AttributeGroupCostsKeyName:
    Description: |
      In your attribute group, key name for the cost's description. For instance: "name" is the key name in
      {"costs": [{"name": "database", "cost": "123"}]}.
    Type: String
    Default: "name"
  AttributeGroupCostsValueName:
    Description: |
      In your attribute group, value name for the costs amount. For instance: "cost" is the value name in
      {"costs": [{"name": "database", "cost": "123"}]}.
    Type: String
    Default: "cost"

# ======================================================================================================================
# Finalize Resources
# ======================================================================================================================
  S3BucketName:
    Description: |
      Amazon S3 bucket name where to store costs data files for Amazon QuickSight to access and upload it.
    Type: String

# ======================================================================================================================
# QuickSight Resources
# ======================================================================================================================
  DeployQuickSight:
    Description: |
      Whether to deploy Amazon QuickSight resources in the CloudFormation stack deployment region.
      AppRegistry application is a regional service. You can deploy QuickSight in one region but deploy
      this solution in every region you have AppRegistry application. All data will be stored in one central
      S3 bucket and one QuickSight deployment is used to visualize the data.
    Type: String
    Default: "true"
    AllowedValues:
      - 'false'
      - 'true'

  QuickSightSubscription:
    Description: |
      Amazon QuickSight subscription edition in your AWS Account.
    Type: String
    Default: "Standard"
    AllowedValues:
      - 'Standard'
      - 'Enterprise'

  QuickSightUsername:
    Type: String
    Description: |
      Username of QuickSight author/admin from default namespace (as displayed in QuickSight admin panel).
      Dashboard created by this template will be shared with this user.
      To find user name of your QuickSight users,
      see https://docs.aws.amazon.com/quicksight/latest/user/managing-users.html.

Mappings:
# ======================================================================================================================
# AWS CloudFormation Stacks Cost Resources
# ======================================================================================================================
  CollectCostsEveryTimePeriod:
    Daily:
      cron: 'cron(0 6 * * ? *)'
    Weekly:
      cron: 'cron(0 6 ? * SUN *)'
    Monthly:
      cron: 'cron(0 6 1 * ? *)'

# ======================================================================================================================
# QuickSight Resources
# ======================================================================================================================
Conditions:
  DeployQuickSightResourcesCondition: !Equals
    - !Ref DeployQuickSight
    - 'true'
  QuickSightEnterpriseLicenseCondition: !And
    - !Equals ["Enterprise", !Ref QuickSightSubscription]
    - !Condition DeployQuickSightResourcesCondition

Resources:
# ======================================================================================================================
# EventBridge Resources
# ======================================================================================================================
  EventBridgeRule:
    # Creates EventBridge rule to watch for CloudFormation API events
    Type: AWS::Events::Rule
    Properties:
      Description: Rule will send CloudFormation events to Step Function
      EventBusName: default
      ScheduleExpression: !FindInMap [CollectCostsEveryTimePeriod, !Ref CollectCostsEveryTimePeriod, cron]
      Name: AppRegistryVisualizedEventRule

      State: ENABLED
      Targets:
        - Arn: !GetAtt CollectorLambdaFunc.Arn
          Id: AppRegStateMachine

  CollectorLambdaFuncLambdaFuncPermission:
    Type: 'AWS::Lambda::Permission'
    Properties:
      Action: "lambda:InvokeFunction"
      FunctionName: !GetAtt CollectorLambdaFunc.Arn
      Principal: "events.amazonaws.com"
      SourceArn: !GetAtt EventBridgeRule.Arn

# ======================================================================================================================
# StepFunctions Resources
# ======================================================================================================================
  StepFunctionPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      Roles:
        - !Ref StepFunctionRole
      PolicyName: !Sub 'AppRegistryVisualizedStepFunctionIamPolicy-${AWS::Region}'
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Action:
              - lambda:InvokeFunction
            Resource:
              - !GetAtt AwsResourcesCostsLambdaFunc.Arn
              - !GetAtt AttributesCostsLambdaFunc.Arn
              - !GetAtt FinalizeLambdaFunc.Arn
  StepFunctionRole:
    Type: 'AWS::IAM::Role'
    Properties:
      RoleName: !Sub 'AppRegistryVisualizedStepFunctionIamRole-${AWS::Region}'
      AssumeRolePolicyDocument: |-
        {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Principal": {
                "Service": "states.amazonaws.com"
              },
              "Effect": "Allow"
            }
          ]
        }

  AppRegStateMachine:
    Type: AWS::StepFunctions::StateMachine
    Properties:
      StateMachineName: AppRegistryApplicationsVisualized
      RoleArn: !GetAtt StepFunctionRole.Arn
      StateMachineType: STANDARD
      DefinitionString: !Sub |
        {
          "Comment": "This is your state machine",
          "StartAt": "Extract",
          "States": {
            "Extract": {
              "Type": "Map",
              "Iterator": {
                "StartAt": "aws-stack-costs",
                "States": {
                  "aws-stack-costs": {
                    "Type": "Task",
                    "Resource": "arn:aws:states:::lambda:invoke",
                    "Parameters": {
                      "Payload.$": "$",
                      "FunctionName": "${AwsResourcesCostsLambdaFunc.Arn}"
                    },
                    "Retry": [
                      {
                        "ErrorEquals": [
                          "Lambda.ServiceException",
                          "Lambda.AWSLambdaException",
                          "Lambda.SdkClientException"
                        ],
                        "IntervalSeconds": 2,
                        "MaxAttempts": 6,
                        "BackoffRate": 2
                      }
                    ],
                    "Next": "ar-atrr-costs",
                    "OutputPath": "$.Payload"
                  },
                  "ar-atrr-costs": {
                    "Type": "Task",
                    "Resource": "arn:aws:states:::lambda:invoke",
                    "Parameters": {
                      "Payload.$": "$",
                      "FunctionName": "${AttributesCostsLambdaFunc.Arn}"
                    },
                    "Retry": [
                      {
                        "ErrorEquals": [
                          "Lambda.ServiceException",
                          "Lambda.AWSLambdaException",
                          "Lambda.SdkClientException"
                        ],
                        "IntervalSeconds": 2,
                        "MaxAttempts": 6,
                        "BackoffRate": 2
                      }
                    ],
                    "End": true,
                    "OutputPath": "$.Payload"
                  }
                }
              },
              "ItemsPath": "$.apps_data",
              "Next": "finalize",
              "ResultPath": "$.apps_data_output"
            },
            "finalize": {
              "Type": "Task",
              "Resource": "arn:aws:states:::lambda:invoke",
              "OutputPath": "$.Payload",
              "Parameters": {
                "Payload.$": "$",
                "FunctionName": "${FinalizeLambdaFunc.Arn}"
              },
              "Retry": [
                {
                  "ErrorEquals": [
                    "Lambda.ServiceException",
                    "Lambda.AWSLambdaException",
                    "Lambda.SdkClientException"
                  ],
                  "IntervalSeconds": 2,
                  "MaxAttempts": 6,
                  "BackoffRate": 2
                }
              ],
              "End": true
            }
          }
        }

# ======================================================================================================================
# Collector Resources
# ======================================================================================================================
  CollectorIamPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      Roles:
        - !Ref CollectorIamRole
      PolicyName: !Sub 'AppRegistryVisualizedCollectorIamPolicy-${AWS::Region}'
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Action:
              - servicecatalog:GetAssociatedResource
              - servicecatalog:ListApplications
              - servicecatalog:ListAssociatedResources
              - servicecatalog:ListAttributeGroups
              - servicecatalog:GetAttributeGroup
              - servicecatalog:GetApplication
              - servicecatalog:ListAssociatedAttributeGroups
            Resource:
              - '*'
          - Effect: Allow
            Action:
              - 'states:StartExecution'
            Resource: !Ref AppRegStateMachine
          - Effect: Allow
            Action:
              - 'logs:CreateLogGroup'
            Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
          - Effect: Allow
            Action:
              - 'logs:CreateLogStream'
              - 'logs:PutLogEvents'
            Resource:
              - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/AppRegistryVisualizedCollectorLambdaFunc:*'
  CollectorIamRole:
    Type: 'AWS::IAM::Role'
    Properties:
      RoleName: !Sub 'AppRegistryVisualizedCollectorIamRole-${AWS::Region}'
      AssumeRolePolicyDocument: |-
        {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              },
              "Effect": "Allow",
              "Sid": ""
            }
          ]
        }
  CollectorLambdaFunc:
    Type: 'AWS::Lambda::Function'
    DependsOn:
      - CollectorIamPolicy
    Properties:
      FunctionName: AppRegistryVisualizedCollectorLambdaFunc
      Runtime: "python3.7"
      Handler: "index.lambda_handler"
      Timeout: 180
      Environment:
        Variables:
          STATE_MACHINE_ARN: !Ref AppRegStateMachine
      Role: !GetAtt CollectorIamRole.Arn
      Code:
        ZipFile: |

          import boto3
          import json
          import os
          import uuid


          state_machine_arn = os.environ['STATE_MACHINE_ARN']


          ar = boto3.client('servicecatalog-appregistry')
          sf = boto3.client('stepfunctions')


          class ArWrapper:

              @staticmethod
              def list_of(func, resp_key, obj_key=None, kwargs={}):
                  apps = []

                  while True:
                      response = func(**kwargs)
                      [apps.append(s[obj_key] if obj_key else s) for s in response[resp_key]]

                      next_token = response.get('NextToken')

                      # break if no or no more token
                      if not next_token:
                          break

                      # assign next token to kwargs
                      kwargs['NextToken'] = next_token

                  return apps


          def start_execution(data):
              name = str(uuid.uuid4())
              response = sf.start_execution(
                  stateMachineArn=state_machine_arn,
                  name=name,
                  input=json.dumps(data)
              )

              print(f'Started new Step Function execution. Execution ARN: {response["executionArn"]}.')


          def lambda_handler(event, context):
              apps = ArWrapper.list_of(ar.list_applications, 'applications', 'name')
              apps_data = []

              for a in apps:
                  apps_data.append({
                      'app_name': a,
                      'resources': ArWrapper.list_of(ar.list_associated_resources, 'resources', 'arn', {'application': a}),
                      'attrs': ArWrapper.list_of(ar.list_associated_attribute_groups, 'attributeGroups', kwargs={'application': a})
                  })

              start_execution({'apps_data': apps_data})
              print(json.dumps({'apps_data': apps_data}))

# ======================================================================================================================
# AWS CloudFormation Stacks Cost Resources
# ======================================================================================================================
  AwsResourcesCostsIamPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      Roles:
        - !Ref AwsResourcesCostsIamRole
      PolicyName: !Sub 'AppRegistryVisualizedAwsResourcesCostsIamPolicy-${AWS::Region}'
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Action: "ce:GetCostAndUsage"
            Resource: "*"
          - Effect: Allow
            Action:
              - 'logs:CreateLogGroup'
            Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
          - Effect: Allow
            Action:
              - 'logs:CreateLogStream'
              - 'logs:PutLogEvents'
            Resource:
              - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/AppRegistryVisualizedAwsResourcesCostsLambdaFunc:*'
  AwsResourcesCostsIamRole:
    Type: 'AWS::IAM::Role'
    Properties:
      RoleName: !Sub 'AppRegistryVisualizedAwsResourcesCostsIamRole-${AWS::Region}'
      AssumeRolePolicyDocument: |-
        {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              },
              "Effect": "Allow",
              "Sid": ""
            }
          ]
        }
  AwsResourcesCostsLambdaFunc:
    Type: 'AWS::Lambda::Function'
    DependsOn:
      - AwsResourcesCostsIamPolicy
    Properties:
      FunctionName: AppRegistryVisualizedAwsResourcesCostsLambdaFunc
      Runtime: "python3.7"
      Handler: "index.lambda_handler"
      Timeout: 180
      Environment:
        Variables:
          TIME_PERIOD: !Ref CollectCostsEveryTimePeriod
          COST_EXPLORER_METRICS: BlendedCost
          COST_EXPLORER_GRANULARITY: DAILY
      Role: !GetAtt AwsResourcesCostsIamRole.Arn
      Code:
        ZipFile: |

          import boto3
          import os
          import json
          import calendar
          import datetime


          t_period = os.environ.get('TIME_PERIOD', 'daily').lower()
          metrics = os.environ.get('COST_EXPLORER_METRICS', 'BlendedCost')
          granularity = os.environ.get('COST_EXPLORER_GRANULARITY', 'DAILY')

          ce = boto3.client('ce')


          def get_days_in_month(year, month):
              return calendar.monthrange(year, month)[1]


          def get_date(delta=0):
              date = datetime.datetime.now()
              date += datetime.timedelta(days=delta)
              return date.date().strftime('%Y-%m-%d')


          def get_previous_month_of_a_year():
              date = datetime.datetime.today().strftime('%Y-%m')
              year, month = date.split('-')
              return int(year), int(month) - 1


          def time_period(period='daily', days_back=1):
              tp = {}

              if period == 'daily':
                  tp = {'Start': get_date(-1), 'End': get_date()}
              elif period == 'weekly':
                  tp = {'Start': get_date(-7), 'End': get_date()}
              elif period == 'monthly':
                  year, month = get_previous_month_of_a_year()
                  days = get_days_in_month(year, month)
                  tp = {'Start': get_date(-days), 'End': get_date()}
              else:
                  tp = {'Start': get_date(-days_back), 'End': get_date()}

              return tp


          def get_cost_and_usage(time_period, cf_stack_arn):
              filters = {
                  "Tags": {
                      "Key": "aws:cloudformation:stack-id",
                      "Values": [cf_stack_arn]
                  }
              }

              response = ce.get_cost_and_usage(
                  TimePeriod=time_period,
                  Granularity=granularity,
                  Filter=filters,
                  Metrics=metrics.split(","),
                  GroupBy=[
                      {
                          'Type': 'COST_CATEGORY',
                          'Key': 'SERVICE'
                      },
                  ],
              )

              return response


          def get_cf_stack_name(cf_stack_arn):
              return cf_stack_arn.split('stack/')[1].split('/')[0]


          def get_cf_stack_cost(response, metrics):
              status_code = response['ResponseMetadata']['HTTPStatusCode']
              if status_code != 200:
                  print(json.dumps(response['ResponseMetadata']))
                  msg = f'Request failed. Func: <get_cost_and_usage>. Status code: {status_code}.'
                  raise RuntimeError(msg)

              total = float(0)
              for time in response['ResultsByTime']:
                  for group in time['Groups']:
                      total += float(group['Metrics'][metrics]['Amount'])

              return total


          def lambda_handler(event, context):
              print(event)

              resource_costs = []
              period = time_period(t_period)

              for cf_stack_arn in event['resources']:
                  stack_name = get_cf_stack_name(cf_stack_arn)
                  response = get_cost_and_usage(period, cf_stack_arn)

                  cost = get_cf_stack_cost(response, metrics)
                  resource_costs.append({'name': stack_name, 'cost': cost})

              event['resource_costs'] = resource_costs
              event['date'] = get_date()

              print(json.dumps(event))

              return event
# ======================================================================================================================
# AppRegistry Attribute Cost Resources
# ======================================================================================================================
  AttributesCostsIamPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      Roles:
        - !Ref AttributesCostsIamRole
      PolicyName: !Sub 'AppRegistryVisualizedAttributesCostsIamPolicy-${AWS::Region}'
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Action:
              - servicecatalog:GetAttributeGroup
              - servicecatalog:ListAttributeGroups
            Resource: "*"
          - Effect: Allow
            Action:
            - 'logs:CreateLogGroup'
            Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
          - Effect: Allow
            Action:
              - 'logs:CreateLogStream'
              - 'logs:PutLogEvents'
            Resource:
              - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/AppRegistryVisualizedAttributesCostsLambdaFunc:*'
  AttributesCostsIamRole:
    Type: 'AWS::IAM::Role'
    Properties:
      RoleName: !Sub 'AppRegistryVisualizedAttributesCostsIamRole-${AWS::Region}'
      AssumeRolePolicyDocument: |-
        {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              },
              "Effect": "Allow",
              "Sid": ""
            }
          ]
        }
  AttributesCostsLambdaFunc:
    Type: 'AWS::Lambda::Function'
    DependsOn:
      - AttributesCostsIamPolicy
    Properties:
      FunctionName: AppRegistryVisualizedAttributesCostsLambdaFunc
      Runtime: "python3.7"
      Handler: "index.lambda_handler"
      Timeout: 180
      Environment:
        Variables:
          ATTRIBUTE_GROUP_COSTS_ENABLED: !Ref AttributeGroupCostsEnabled
          ATTRIBUTE_GROUP_COSTS_JSON_PATH: !Ref AttributeGroupCostsJsonPath
          ATTRIBUTE_GROUP_COSTS_KEY_NAME: !Ref AttributeGroupCostsKeyName
          ATTRIBUTE_GROUP_COSTS_VALUE_NAME: !Ref AttributeGroupCostsValueName
      Role: !GetAtt AttributesCostsIamRole.Arn
      Code:
        ZipFile: |

          import boto3
          import os
          import json


          attr_group_costs_enabled = os.environ.get('ATTRIBUTE_GROUP_COSTS_ENABLED', 'False') in ['True', 'true']
          json_path = os.environ.get('ATTRIBUTE_GROUP_COSTS_JSON_PATH', '$')
          key_name = os.environ.get('ATTRIBUTE_GROUP_COSTS_KEY_NAME', 'name')
          value_name = os.environ.get('ATTRIBUTE_GROUP_COSTS_VALUE_NAME', 'cost')

          ar = boto3.client('servicecatalog-appregistry')


          def get_attribute_group(attribute_group):
              return json.loads(ar.get_attribute_group(attributeGroup=attribute_group)['attributes'])


          def get_costs_object(json_path, attributes):
              path = json_path.split('.')[1:]
              obj = attributes

              for p in path:
                  obj = obj[p]
              return obj


          def lambda_handler(event, context):
              print(event)
              print(f'ATTRIBUTE_GROUP_COSTS_JSON_PATH: <{json_path}>.')
              event['attr_costs'] = []

              if not attr_group_costs_enabled:
                  print(f'ATTRIBUTE_GROUP_COSTS_ENABLED: {attr_group_costs_enabled}.')
                  return event

              if len(json_path) <= 2:
                  print(f'ATTRIBUTE_GROUP_COSTS_JSON_PATH not specified. Path found: {json_path}.')
                  return event

              attr_costs = []

              for attr in event['attrs']:
                  attributes = get_attribute_group(attr)

                  try:
                      obj = get_costs_object(json_path, attributes)
                  except Exception as e:  # path or object in appRegistry doesn't exist as specified in env var.
                      print(e)
                      return event

                  for cost in obj:
                      print(cost)

                      if key_name in cost:
                          cost_key_name = cost[key_name]
                      else:
                          print(f'ATTRIBUTE_GROUP_COSTS_KEY_NAME <{key_name}> not found in <{cost}> under JSON path <{json_path}>.')
                          cost_key_name = 'Unknown'

                      if value_name in cost:
                          cost_value_name = cost[value_name]
                      else:
                          print(f'ATTRIBUTE_GROUP_COSTS_KEY_NAME <{key_name}> not found in <{cost}> under JSON path <{json_path}>.')
                          cost_value_name = 0
                      attr_costs.append({'name': cost_key_name, 'cost': float(cost_value_name)})

              event['attr_costs'] = attr_costs
              print(json.dumps(event))

              return event
# ======================================================================================================================
# Finalize Resources
# ======================================================================================================================
  FinalizeIamPolicy:
    Type: 'AWS::IAM::Policy'
    Properties:
      Roles:
        - !Ref FinalizeIamRole
      PolicyName: !Sub 'AppRegistryVisualizedFinalizeIamPolicy-${AWS::Region}'
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Action: "s3:PutObject"
            Resource: !Sub "arn:aws:s3:::${S3BucketName}/*"
          - Effect: Allow
            Action: "quicksight:CreateIngestion"
            Resource: !Sub "arn:aws:quicksight:*:${AWS::AccountId}:dataset/AppRegistryVisualizedS3/ingestion/*"
          - Effect: Allow
            Action:
              - 'logs:CreateLogGroup'
            Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
          - Effect: Allow
            Action:
              - 'logs:CreateLogStream'
              - 'logs:PutLogEvents'
            Resource:
              - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/AppRegistryVisualizedFinalizeLambdaFunc:*'
  FinalizeIamRole:
    Type: 'AWS::IAM::Role'
    Properties:
      RoleName: !Sub 'AppRegistryVisualizedFinalizeIamRole-${AWS::Region}'
      AssumeRolePolicyDocument: |-
        {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              },
              "Effect": "Allow",
              "Sid": ""
            }
          ]
        }
  FinalizeLambdaFunc:
    Type: 'AWS::Lambda::Function'
    DependsOn:
      - FinalizeIamPolicy
    Properties:
      FunctionName: AppRegistryVisualizedFinalizeLambdaFunc
      Runtime: "python3.7"
      Handler: "index.lambda_handler"
      Timeout: 180
      Environment:
        Variables:
          S3_BUCKET: !Ref S3BucketName
          DATA_SET_ID: "AppRegistryVisualizedS3"
          AWS_ACCOUNT_ID: !Ref AWS::AccountId
          THIS_AWS_REGION: !Ref AWS::Region
          QUICKSIGHT_REGION: !Ref DeployQuickSight
      Role: !GetAtt FinalizeIamRole.Arn
      Code:
        ZipFile: |

          import csv
          import json
          import os
          import logging
          import uuid
          import boto3
          from botocore.exceptions import ClientError


          s3_bucket = os.environ['S3_BUCKET']
          s3_prefix = os.environ.get('S3_BUCKET_PREFIX', 'costs_data')
          local_file_path_prefix = os.environ.get('LOCAL_FILE_PATH_PREFIX', '/tmp')
          data_set_id = os.environ['DATA_SET_ID']
          aws_account_id = os.environ['AWS_ACCOUNT_ID']
          aws_region = os.environ['THIS_AWS_REGION']
          qs_region = os.environ['QUICKSIGHT_REGION'] in ['True', 'true']


          s3 = boto3.client('s3')
          qs = boto3.client('quicksight')


          def write_to_csv(file_name, data):
              with open(file_name, "wt") as fp:
                  writer = csv.writer(fp, delimiter=",")
                  writer.writerows(data)


          def upload_file(file_name, bucket, object_name=None):
              """Upload a file to an S3 bucket

              :param file_name: File to upload
              :param bucket: Bucket to upload to
              :param object_name: S3 object name. If not specified then file_name is used
              :return: True if file was uploaded, else False
              """

              # If S3 object_name was not specified, use file_name
              if object_name is None:
                  object_name = file_name

              try:
                  s3.upload_file(file_name, bucket, object_name)
              except ClientError as e:
                  logging.error(e)
                  return False
              return True


          def write_file_to_s3(data, date):
              file_name = f'{aws_account_id}/{aws_region}/{date}.csv'
              s3_path = f'{s3_prefix}/{file_name}'
              local_path = f'{local_file_path_prefix}/{date}.csv'

              write_to_csv(local_path, data)
              upload_file(local_path, s3_bucket, s3_path)

              return s3_path


          def create_quicksight_ingestion(data_set_id, aws_account_id, aws_region, qs_region):
              if not qs_region:
                  print(f'QuickSight ingestion request skipped, {aws_region} is not a QuickSight region.')
                  return

              ingestion_id = str(uuid.uuid4())
              response = qs.create_ingestion(DataSetId=data_set_id, IngestionId=ingestion_id, AwsAccountId=aws_account_id)
              print('QuickSight create_ingestion response:')
              print(json.dumps(response))


          def lambda_handler(event, context):
              print(event)

              schema = ['ApplicationName', 'ResourceName', 'Cost', 'Type', 'Date']
              data = [schema]

              apps_data_output = event['apps_data_output']
              if not apps_data_output:
                  print('No AppRegistry application found.')
                  return

              date = apps_data_output[0]['date']

              for app in apps_data_output:
                  for rc in app['resource_costs']:
                      data.append([app['app_name'], rc['name'], rc['cost'], 'CloudFormationStack', app['date']])

                  for rc in app['attr_costs']:
                      data.append([app['app_name'], rc['name'], rc['cost'], 'Other', app['date']])

              print(f'Writing data to S3 bucket: {s3_bucket}, with S3_BUCKET_PREFIX: {s3_prefix}.')
              print(json.dumps(data))

              write_file_to_s3(data, date)
              create_quicksight_ingestion(data_set_id, aws_account_id, aws_region, qs_region)

              return 'ok'
# ======================================================================================================================
# QuickSight Resources
# ======================================================================================================================
  QuickSightInitIamPolicy:
    Condition: DeployQuickSightResourcesCondition
    Type: 'AWS::IAM::Policy'
    Properties:
      Roles:
        - !Ref QuickSightInitIamRole
      PolicyName: !Sub 'AppRegistryVisualizedQuickSightInitIamPolicy-${AWS::Region}'
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Action: "s3:PutObject"
            Resource: !Sub "arn:aws:s3:::${S3BucketName}/*"
          - Effect: Allow
            Action: "quicksight:ListUsers"
            Resource:
              - !Sub 'arn:aws:quicksight:${AWS::Region}:${AWS::AccountId}:user/*'
          - Effect: Allow
            Action:
              - 'logs:CreateLogGroup'
            Resource: !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:*'
          - Effect: Allow
            Action:
              - 'logs:CreateLogStream'
              - 'logs:PutLogEvents'
            Resource:
              - !Sub 'arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/AppRegistryVisualizedQuickSightInitLambdaFunc:*'

  QuickSightInitIamRole:
    Condition: DeployQuickSightResourcesCondition
    Type: 'AWS::IAM::Role'
    Properties:
      RoleName: !Sub 'AppRegistryVisualizedQuickSightInitIamRole-${AWS::Region}'
      AssumeRolePolicyDocument: |-
        {
          "Version": "2012-10-17",
          "Statement": [
            {
              "Action": "sts:AssumeRole",
              "Principal": {
                "Service": "lambda.amazonaws.com"
              },
              "Effect": "Allow",
              "Sid": ""
            }
          ]
        }

  QuickSightInitLambdaFunc:
    Condition: DeployQuickSightResourcesCondition
    Type: 'AWS::Lambda::Function'
    DependsOn:
      - QuickSightInitIamPolicy
    Properties:
      FunctionName: AppRegistryVisualizedQuickSightInitLambdaFunc
      Runtime: "python3.7"
      Handler: "index.lambda_handler"
      Timeout: 180
      Environment:
        Variables:
          S3_BUCKET: !Ref S3BucketName
          AWS_ACCOUNT_ID: !Ref AWS::AccountId
          NAMESPACE: 'default'
      Role: !GetAtt QuickSightInitIamRole.Arn
      Code:
        ZipFile: |

          import json
          import os
          import logging
          import boto3
          import datetime
          import re
          import cfnresponse

          from botocore.exceptions import ClientError


          s3_bucket = os.environ['S3_BUCKET']
          s3_prefix = os.environ.get('S3_BUCKET_PREFIX', 'costs_data')
          local_file_path_prefix = os.environ.get('LOCAL_FILE_PATH_PREFIX', '/tmp')

          aws_account_id = os.environ['AWS_ACCOUNT_ID']
          namespace = os.environ.get('NAMESPACE', 'default')


          s3 = boto3.client('s3')
          qs = boto3.client('quicksight')


          config = {
              "fileLocations": [
                  {
                      "URIPrefixes": [
                          f's3://{s3_bucket}/{s3_prefix}/'
                      ]
                  }
              ],
              "globalUploadSettings": {
                  "format": "CSV",
                  "delimiter": ",",
                  "textqualifier": "'",
                  "containsHeader": "true"
              }
          }


          def write_to_json(file_name, data):
              print(f'Writing file to path: {local_file_path_prefix}.')
              with open(file_name, 'w') as f:
                  json.dump(data, f)


          def upload_file(file_name, bucket, object_name=None):
              """Upload a file to an S3 bucket

              :param file_name: File to upload
              :param bucket: Bucket to upload to
              :param object_name: S3 object name. If not specified then file_name is used
              :return: True if file was uploaded, else False
              """

              # If S3 object_name was not specified, use file_name
              if object_name is None:
                  object_name = file_name

              # Upload the file
              s3_client = boto3.client('s3')
              try:
                  response = s3_client.upload_file(file_name, bucket, object_name)
              except ClientError as e:
                  logging.error(e)
                  return False
              return True


          def write_file_to_s3(data, file_name, s3_path):
              local_path = f'{local_file_path_prefix}/{file_name}'

              write_to_json(local_path, data)
              upload_file(local_path, s3_bucket, s3_path)

              return s3_path


          def get_date(delta=0):
              date = datetime.datetime.now()
              date += datetime.timedelta(days=delta)
              return date.date().strftime('%Y-%m-%d')


          def write_first_data_for_data_set():
              data = [['ApplicationName', 'ResourceName', 'Cost', 'Type', 'Date']]

              file_name = 'init_data_set.csv'
              write_file_to_s3(data, file_name, f'{s3_prefix}/{file_name}')


          def get_quicksight_user_identity_region():
              quicksight_users_region = 'unknown-region'

              try:
                  print(qs.list_users(AwsAccountId=aws_account_id, Namespace=namespace))
                  quicksight_users_region = boto3.session.Session().region_name
              except Exception as e:
                  print(e)
                  regex = r"(us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d"
                  regions = [x.group() for x in re.finditer(regex, str(e))]

                  if len(regions) > 1:
                      quicksight_users_region = regions[2]

              print(f'QuickSight user identity region: {quicksight_users_region}')

              return quicksight_users_region


          def lambda_handler(event, context):
              try:
                  print(f'Writing Amazon QuickSight manifest to S3 bucket: {s3_bucket}.')
                  print(json.dumps(config))

                  file_name = 'quickSightManifest.json'
                  write_file_to_s3(config, file_name, file_name)
                  write_first_data_for_data_set()
                  quicksight_user_identity_region = get_quicksight_user_identity_region()

                  cfnresponse.send(event, context, cfnresponse.SUCCESS, {
                      'QuickSightUserIdentityRegion': quicksight_user_identity_region
                  }, "AppRegistryCaller")
              except Exception as e:
                  print(e)
                  cfnresponse.send(event, context, cfnresponse.FAILED, {'Data': str(e)}, "AppRegistryCaller")

  CustomQuickSightInitLambdaFunc:
    Condition: DeployQuickSightResourcesCondition
    Type: Custom::QuickSightInitLambdaFunc
    Properties:
      ServiceToken: !GetAtt QuickSightInitLambdaFunc.Arn

  QuickSightDataSource:
    Condition: DeployQuickSightResourcesCondition
    Type: AWS::QuickSight::DataSource
    Properties:
      AwsAccountId: !Ref AWS::AccountId
      DataSourceId: AppRegistryVisualizedS3
      Name: AppRegistryVisualizedS3
      Type: S3
      DataSourceParameters:
        S3Parameters:
          ManifestFileLocation:
            Bucket: !Ref S3BucketName
            Key: 'quickSightManifest.json'
      Permissions:
        - Principal: !Join
            - ''
            - - 'arn:aws:quicksight:'
              - !GetAtt CustomQuickSightInitLambdaFunc.QuickSightUserIdentityRegion
              - ':'
              - !Ref AWS::AccountId
              - ':user/default/'
              - !Ref QuickSightUsername
          Actions:
            - quicksight:UpdateDataSourcePermissions
            - quicksight:DescribeDataSource
            - quicksight:DescribeDataSourcePermissions
            - quicksight:PassDataSource
            - quicksight:UpdateDataSource
            - quicksight:DeleteDataSource

  QuickSightDataSet:
    Condition: DeployQuickSightResourcesCondition
    Type: AWS::QuickSight::DataSet
    Properties:
      ImportMode: SPICE
      DataSetId: AppRegistryVisualizedS3
      Name: AppRegistryVisualizedS3
      AwsAccountId: !Ref AWS::AccountId
      PhysicalTableMap:
        PhysicalTable1:
          S3Source:
            DataSourceArn: !GetAtt QuickSightDataSource.Arn
            InputColumns:
              - Name: ApplicationName
                Type: STRING
              - Name: ResourceName
                Type: STRING
              - Name: Cost
                Type: STRING
              - Name: Type
                Type: STRING
              - Name: Date
                Type: STRING
      LogicalTableMap:
        LogicalTable1:
          Alias: AppRegistryVisualizedS3
          DataTransforms:
            - CastColumnTypeOperation:
                ColumnName: Date
                NewColumnType: DATETIME
            - CastColumnTypeOperation:
                ColumnName: Cost
                NewColumnType: DECIMAL
          Source:
            PhysicalTableId: PhysicalTable1
      Permissions:
        - Principal: !Join
            - ''
            - - 'arn:aws:quicksight:'
              - !GetAtt CustomQuickSightInitLambdaFunc.QuickSightUserIdentityRegion
              - ':'
              - !Ref AWS::AccountId
              - ':user/default/'
              - !Ref QuickSightUsername
          Actions:
            - quicksight:UpdateDataSetPermissions
            - quicksight:DescribeDataSet
            - quicksight:DescribeDataSetPermissions
            - quicksight:PassDataSet
            - quicksight:DescribeIngestion
            - quicksight:ListIngestions
            - quicksight:UpdateDataSet
            - quicksight:DeleteDataSet
            - quicksight:CreateIngestion
            - quicksight:CancelIngestion

  QuickSightTemplate:
    Condition: QuickSightEnterpriseLicenseCondition
    Type: AWS::QuickSight::Template
    Properties:
      TemplateId: AppRegistryVisualizedTemplate
      Name: AppRegistryVisualizedTemplate
      AwsAccountId: !Ref AWS::AccountId
      SourceEntity:
        SourceTemplate:
          Arn: arn:aws:quicksight:us-east-1:544737550582:template/AppRegistryVisualizedBlogTemplate
      Permissions:
        - Principal: !Join
            - ''
            - - 'arn:aws:quicksight:'
              - !GetAtt CustomQuickSightInitLambdaFunc.QuickSightUserIdentityRegion
              - ':'
              - !Ref AWS::AccountId
              - ':user/default/'
              - !Ref QuickSightUsername
          Actions:
            - quicksight:DescribeTemplate
      VersionDescription: Initial version - Copied over from AWS account.

  QuickSightAnalysis:
    Condition: QuickSightEnterpriseLicenseCondition
    Type: AWS::QuickSight::Analysis
    Properties:
      AnalysisId: AppRegistryVisualizedAnalysis
      Name: AppRegistryVisualizedAnalysis
      AwsAccountId: !Ref AWS::AccountId
      SourceEntity:
        SourceTemplate:
          Arn: !GetAtt QuickSightTemplate.Arn
          DataSetReferences:
            - DataSetPlaceholder: AppRegistryVisualizedDataSetTemplate
              DataSetArn: !GetAtt QuickSightDataSet.Arn
      Permissions:
        - Principal: !Join
            - ''
            - - 'arn:aws:quicksight:'
              - !GetAtt CustomQuickSightInitLambdaFunc.QuickSightUserIdentityRegion
              - ':'
              - !Ref AWS::AccountId
              - ':user/default/'
              - !Ref QuickSightUsername
          Actions:
            - quicksight:RestoreAnalysis
            - quicksight:UpdateAnalysisPermissions
            - quicksight:DeleteAnalysis
            - quicksight:DescribeAnalysisPermissions
            - quicksight:QueryAnalysis
            - quicksight:DescribeAnalysis
            - quicksight:UpdateAnalysis

  QuickSightDashboard:
    Condition: QuickSightEnterpriseLicenseCondition
    Type: AWS::QuickSight::Dashboard
    Properties:
      DashboardId: AppRegistryVisualizedAnalysis
      Name: AppRegistryVisualizedAnalysis
      AwsAccountId: !Ref AWS::AccountId
      SourceEntity:
        SourceTemplate:
          Arn: !GetAtt QuickSightTemplate.Arn
          DataSetReferences:
            - DataSetPlaceholder: AppRegistryVisualizedDataSetTemplate
              DataSetArn: !GetAtt QuickSightDataSet.Arn
      Permissions:
        - Principal: !Join
            - ''
            - - 'arn:aws:quicksight:'
              - !GetAtt CustomQuickSightInitLambdaFunc.QuickSightUserIdentityRegion
              - ':'
              - !Ref AWS::AccountId
              - ':user/default/'
              - !Ref QuickSightUsername
          Actions:
            - quicksight:DescribeDashboard
            - quicksight:ListDashboardVersions
            - quicksight:UpdateDashboardPermissions
            - quicksight:QueryDashboard
            - quicksight:UpdateDashboard
            - quicksight:DeleteDashboard
            - quicksight:DescribeDashboardPermissions
            - quicksight:UpdateDashboardPublishedVersion
      DashboardPublishOptions:
        AdHocFilteringOption:
          AvailabilityStatus: DISABLED
