Final Project: Build It, Break It, Fix It

Sign in and save the template:

cd C:\aws-lab
aws login --profile saat-admin
$env:AWS_PROFILE = 'saat-admin'
$v = Get-Content C:\aws-lab\saat-vpc.json | ConvertFrom-Json
$tpl = @'
AWSTemplateFormatVersion: '2010-09-09'
Description: SaatVedha final project - SaatShop online
Parameters:
  VpcId:
    Type: AWS::EC2::VPC::Id
  PublicSubnets:
    Type: List<AWS::EC2::Subnet::Id>
  PrivateSubnets:
    Type: List<AWS::EC2::Subnet::Id>
  AlbSg:
    Type: AWS::EC2::SecurityGroup::Id
  AppSg:
    Type: AWS::EC2::SecurityGroup::Id
  AlertTopic:
    Type: String
Resources:
  AppRole:
    Type: AWS::IAM::Role
    Properties:
      AssumeRolePolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service: ec2.amazonaws.com
            Action: sts:AssumeRole
      ManagedPolicyArns:
        - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore
  AppProfile:
    Type: AWS::IAM::InstanceProfile
    Properties:
      Roles:
        - !Ref AppRole
  AdminSg:
    Type: AWS::EC2::SecurityGroup
    Properties:
      GroupDescription: SaatShop admin access (no rules on purpose)
      VpcId: !Ref VpcId
  AppLT:
    Type: AWS::EC2::LaunchTemplate
    Properties:
      LaunchTemplateData:
        ImageId: '{{resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64}}'
        InstanceType: t3.micro
        IamInstanceProfile:
          Arn: !GetAtt AppProfile.Arn
        SecurityGroupIds:
          - !Ref AppSg
          - !Ref AdminSg
        MetadataOptions:
          HttpTokens: required
        TagSpecifications:
          - ResourceType: instance
            Tags:
              - Key: Name
                Value: saat-final-app
              - Key: course
                Value: saatvedha-aws
        UserData:
          Fn::Base64: |
            #!/bin/bash
            cat > /opt/app.py <<'EOF2'
            import http.server, socket
            class H(http.server.BaseHTTPRequestHandler):
                def do_GET(self):
                    if self.path not in ('/', '/health'):
                        self.send_error(404)
                        return
                    body = ('SaatShop online from ' + socket.gethostname() + '\n').encode()
                    self.send_response(200)
                    self.send_header('Content-Length', str(len(body)))
                    self.end_headers()
                    self.wfile.write(body)
                def log_message(self, *args):
                    pass
            http.server.ThreadingHTTPServer(('', 8080), H).serve_forever()
            EOF2
            systemd-run --unit=app8080 python3 /opt/app.py
  AppTG:
    Type: AWS::ElasticLoadBalancingV2::TargetGroup
    Properties:
      VpcId: !Ref VpcId
      Protocol: HTTP
      Port: 8080
      TargetType: instance
      HealthCheckPath: /health
      HealthCheckIntervalSeconds: 10
      HealthyThresholdCount: 2
      UnhealthyThresholdCount: 2
  Alb:
    Type: AWS::ElasticLoadBalancingV2::LoadBalancer
    Properties:
      Type: application
      Subnets: !Ref PublicSubnets
      SecurityGroups:
        - !Ref AlbSg
  Listener:
    Type: AWS::ElasticLoadBalancingV2::Listener
    Properties:
      LoadBalancerArn: !Ref Alb
      Port: 80
      Protocol: HTTP
      DefaultActions:
        - Type: forward
          TargetGroupArn: !Ref AppTG
  AppAsg:
    Type: AWS::AutoScaling::AutoScalingGroup
    Properties:
      MinSize: '2'
      MaxSize: '4'
      DesiredCapacity: '2'
      VPCZoneIdentifier: !Ref PrivateSubnets
      LaunchTemplate:
        LaunchTemplateId: !Ref AppLT
        Version: !GetAtt AppLT.LatestVersionNumber
      TargetGroupARNs:
        - !Ref AppTG
      HealthCheckType: ELB
      HealthCheckGracePeriod: 120
  AssetBucket:
    Type: AWS::S3::Bucket
  UnhealthyAlarm:
    Type: AWS::CloudWatch::Alarm
    Properties:
      AlarmDescription: SaatShop has unhealthy app servers
      Namespace: AWS/ApplicationELB
      MetricName: UnHealthyHostCount
      Dimensions:
        - Name: TargetGroup
          Value: !GetAtt AppTG.TargetGroupFullName
        - Name: LoadBalancer
          Value: !GetAtt Alb.LoadBalancerFullName
      Statistic: Maximum
      Period: 60
      EvaluationPeriods: 2
      Threshold: 0
      ComparisonOperator: GreaterThanThreshold
      TreatMissingData: notBreaching
      AlarmActions:
        - !Ref AlertTopic
Outputs:
  AlbDns:
    Value: !GetAtt Alb.DNSName
  AlbFullName:
    Value: !GetAtt Alb.LoadBalancerFullName
  AsgName:
    Value: !Ref AppAsg
  TgArn:
    Value: !Ref AppTG
  LtId:
    Value: !Ref AppLT
  AlarmName:
    Value: !Ref UnhealthyAlarm
  Bucket:
    Value: !Ref AssetBucket
  AdminSgId:
    Value: !Ref AdminSg
'@
$tpl | Set-Content final.yaml
aws cloudformation validate-template --template-body file://final.yaml --query "Capabilities" --output text

You should see this:

CAPABILITY_IAM

1Deploy SaatShop online

Paste this whole block (about 6 minutes):

$topic = aws sns list-topics --query "Topics[?ends_with(TopicArn, ':saat-alerts')].TopicArn | [0]" --output text
@(
  @{ ParameterKey = 'VpcId'; ParameterValue = $v.VpcId },
  @{ ParameterKey = 'PublicSubnets'; ParameterValue = "$($v.PublicA),$($v.PublicB)" },
  @{ ParameterKey = 'PrivateSubnets'; ParameterValue = "$($v.PrivateA),$($v.PrivateB)" },
  @{ ParameterKey = 'AlbSg'; ParameterValue = $v.WebSg },
  @{ ParameterKey = 'AppSg'; ParameterValue = $v.AppSg },
  @{ ParameterKey = 'AlertTopic'; ParameterValue = $topic }) | ConvertTo-Json | Set-Content final-params.json
aws cloudformation create-stack --stack-name saat-final --template-body file://final.yaml --parameters file://final-params.json --capabilities CAPABILITY_IAM --tags Key=course,Value=saatvedha-aws Key=lesson,Value=69 | Out-Null
aws cloudformation wait stack-create-complete --stack-name saat-final
aws cloudformation describe-stacks --stack-name saat-final --query "Stacks[0].StackStatus" --output text

You should see this:

CREATE_COMPLETE

2Check its health

Paste this whole block. It loads the stack outputs and two helpers, then waits until both servers are healthy:

$o = @{}
aws cloudformation describe-stacks --stack-name saat-final --query "Stacks[0].Outputs" --output json | ConvertFrom-Json | ForEach-Object { $o[$_.OutputKey] = $_.OutputValue }
function Get-Code([string]$Path) {
  $sw = [Diagnostics.Stopwatch]::StartNew()
  try { $r = Invoke-WebRequest "http://$($o.AlbDns)$Path" -UseBasicParsing -TimeoutSec 30; $code = $r.StatusCode; $body = $r.Content.Trim() } catch { $code = [int]$_.Exception.Response.StatusCode; $body = '' }
  '{0}  {1}  {2:N1} s  {3}' -f $Path, $code, $sw.Elapsed.TotalSeconds, $body
}
function Get-ShopHealth {
  "Site:   $(Get-Code /)"
  "Alarm:  $(aws cloudwatch describe-alarms --alarm-names $o.AlarmName --query 'MetricAlarms[0].StateValue' --output text)"
  "ASG:    $(aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $o.AsgName --query 'AutoScalingGroups[0].[MinSize,DesiredCapacity,length(Instances)]' --output text)"
  aws elbv2 describe-target-health --target-group-arn $o.TgArn --query "TargetHealthDescriptions[].[Target.Id,TargetHealth.State,TargetHealth.Reason]" --output text
}
do { Start-Sleep 20; $h = aws elbv2 describe-target-health --target-group-arn $o.TgArn --query "length(TargetHealthDescriptions[?TargetHealth.State=='healthy'])" --output text } until ($h -eq '2')
Get-ShopHealth

You should see something like this:

Site:   /  200  0.1 s  SaatShop online from ip-10-20-11-201.ap-south-1.compute.internal
Alarm:  OK
ASG:    2       2       2
i-0a1b2c3d4e5f60718     healthy None
i-0b2c3d4e5f6a70819     healthy None

This is “normal”. Take a screenshot; after every fix, Get-ShopHealth should look like this again.

3Save the scenario script

Paste this whole block. Break-SaatShop.ps1 injects one failure each time you run it:

$b = @'
param([Parameter(Mandatory)][ValidateRange(1, 10)][int]$Scenario)
$o = @{}
aws cloudformation describe-stacks --stack-name saat-final --query "Stacks[0].Outputs" --output json | ConvertFrom-Json | ForEach-Object { $o[$_.OutputKey] = $_.OutputValue }
$v = Get-Content C:\aws-lab\saat-vpc.json | ConvertFrom-Json
$ids = @((aws autoscaling describe-auto-scaling-groups --auto-scaling-group-names $o.AsgName --query "AutoScalingGroups[0].Instances[].InstanceId" --output text) -split '\s+' | Where-Object { $_ })
switch ($Scenario) {
  1 { aws autoscaling update-auto-scaling-group --auto-scaling-group-name $o.AsgName --min-size 0 --desired-capacity 0 }
  2 { aws ec2 revoke-security-group-ingress --group-id $v.AppSg --ip-permissions "IpProtocol=tcp,FromPort=8080,ToPort=8080,UserIdGroupPairs=[{GroupId=$($v.WebSg)}]" | Out-Null }
  3 { aws elbv2 modify-target-group --target-group-arn $o.TgArn --health-check-path /status | Out-Null }
  4 {
    $assoc = aws ec2 describe-network-acls --filters "Name=association.subnet-id,Values=$($v.PrivateA)" --query "NetworkAcls[0].Associations[?SubnetId=='$($v.PrivateA)'].NetworkAclAssociationId | [0]" --output text
    $orig = aws ec2 describe-network-acls --filters "Name=association.subnet-id,Values=$($v.PrivateA)" --query "NetworkAcls[0].NetworkAclId" --output text
    $nacl = aws ec2 create-network-acl --vpc-id $v.VpcId --tag-specifications 'ResourceType=network-acl,Tags=[{Key=Name,Value=saat-final-nacl},{Key=course,Value=saatvedha-aws}]' --query NetworkAcl.NetworkAclId --output text
    aws ec2 create-network-acl-entry --network-acl-id $nacl --ingress --rule-number 100 --protocol tcp --port-range From=8080,To=8080 --cidr-block 10.20.0.0/16 --rule-action allow
    $new = aws ec2 replace-network-acl-association --association-id $assoc --network-acl-id $nacl --query NewAssociationId --output text
    @{ OriginalAcl = $orig; NewAssociation = $new; BadAcl = $nacl } | ConvertTo-Json | Set-Content C:\aws-lab\final-nacl.json
  }
  5 { aws ec2 stop-instances --instance-ids $ids[0] | Out-Null }
  6 {
    'Diwali sale: 20% off' | Set-Content promo.txt
    aws s3 cp promo.txt "s3://$($o.Bucket)/promo.txt" --only-show-errors
    aws s3 presign "s3://$($o.Bucket)/promo.txt" --expires-in 5 | Set-Content C:\aws-lab\final-link.txt
    Remove-Item promo.txt
  }
  7 {
    $topic = aws cloudwatch describe-alarms --alarm-names $o.AlarmName --query "MetricAlarms[0].AlarmActions[0]" --output text
    aws cloudwatch put-metric-alarm --alarm-name $o.AlarmName --namespace AWS/ApplicationELB --metric-name UnHealthyHostCount --dimensions "Name=LoadBalancer,Value=$($o.AlbFullName)" --statistic Maximum --period 60 --evaluation-periods 2 --threshold 0 --comparison-operator GreaterThanThreshold --alarm-actions $topic
  }
  8 { aws ec2 authorize-security-group-ingress --group-id $o.AdminSgId --protocol tcp --port 22 --cidr 0.0.0.0/0 | Out-Null }
  9 { aws ec2 allocate-address --region ap-southeast-1 --domain vpc --query PublicIp --output text | Out-Null }
  10 {
    $ud = aws ec2 describe-launch-template-versions --launch-template-id $o.LtId --versions 1 --query "LaunchTemplateVersions[0].LaunchTemplateData.UserData" --output text
    $text = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String($ud)).Replace([string][char]10, [string][char]13 + [string][char]10)
    '{ "UserData": "B64" }'.Replace('B64', [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($text))) | Set-Content lt-data.json
    $ver = aws ec2 create-launch-template-version --launch-template-id $o.LtId --source-version 1 --launch-template-data file://lt-data.json --query LaunchTemplateVersion.VersionNumber --output text
    aws autoscaling update-auto-scaling-group --auto-scaling-group-name $o.AsgName --launch-template "LaunchTemplateId=$($o.LtId),Version=$ver"
    aws autoscaling terminate-instance-in-auto-scaling-group --instance-id $ids[0] --no-should-decrement-desired-capacity | Out-Null
    Remove-Item lt-data.json
  }
}
"Scenario $Scenario is live. Find out what broke, then fix it."
'@
$b | Set-Content C:\aws-lab\Break-SaatShop.ps1
Get-ChildItem C:\aws-lab\*.ps1 | Select-Object -ExpandProperty Name

You should see something like this:

Break-SaatShop.ps1
Find-Spenders.ps1
Find-Waste.ps1
Get-RecentChanges.ps1
Remove-LabBucket.ps1
Start-SqlLab.ps1
Start-SsmEndpoints.ps1
Stop-SqlLab.ps1
Stop-SsmEndpoints.ps1
Test-Ec2Access.ps1
Test-NativeRestore.ps1
Test-RdsAccess.ps1

Don’t read the script yet if you want a fair test. Its switch block is the list of answers.

4Work through the scenarios

Run one scenario at a time with .\Break-SaatShop.ps1 -Scenario N, investigate, fix it, and confirm with Get-ShopHealth before the next. Here is what the “users” report for each one, and a hint about where to start:

✅ Checkpoint: saat-final is CREATE_COMPLETE, Get-ShopHealth shows 200, alarm OK and two healthy targets, and Break-SaatShop.ps1 is saved.