Standup Serverless Jenkins on Fargate with Terraform - Part 2: ECS Deployment

This tutorial assumes that you've completed Standup Serverless Jenkins on Fargate with Terraform - Part 1: Networking . If not, you need to do that first.

To start, create variables.tf and add the following variables.

variable "application_name" {
  description = "Name of the application"
  type        = string
}

variable "aws_vpc_id" {
  description = "VPC ID"
  type        = string
}

variable "jenkins_controller_identifier" {
  description = "Name of the jenkins controller"
  type        = string
}

variable "jenkins_agent_port" {
  description = "Port Jenkins agent uses to connect to controller"
  type        = number
}

variable "jenkins_controller_port" {
  description = "Port used to connect to Jenkins controller"
  type        = number
}

Next, create terraform.tfvars to give your variables values. Remember the network we created in lesson one. Assuming you made that, grab the VPC ID and set the value for aws_vpc_id.

application_name = "serverless-jenkins-on-ecs"
jenkins_controller_identifier = "jenkins-controller"
jenkins_agent_port            = 50000
jenkins_controller_port       = 8080
aws_vpc_id = "vpc-ID_Get-This-From-AWS"

Create data.tf, so you can get the necessary information to create your ECS resources. Note that we're grabbing the subnets that we want by filtering on VPC ID and tag names.

data "aws_region" "current" {}

# Current AWS account
data "aws_caller_identity" "this" {}

data "aws_subnets" "public" {
  filter {
    name   = "vpc-id"
    values = [var.aws_vpc_id]
  }
  filter {
    name   = "tag:Name"
    values = ["public-*"]
  }
}

data "aws_subnets" "private" {
  filter {
    name   = "vpc-id"
    values = [var.aws_vpc_id]
  }
  filter {
    name   = "tag:Name"
    values = ["private-*"]
  }
}

Create main.tf to bring in the AWS providers from the HashiCorp registry. Terraform providers are simply plugins that allow you to interact with APIs. In this case, we need to work with AWS APIs.

terraform {
  required_providers {
    aws = {
      source  = "hashicorp/aws"
    }
  }
}

We need storage for our Jenkins. Create efs.tf as we plan to use AWS Elastic File System (EFS). Note the comments above each snippet to understand what the code is doing.

# Elastic File System (EFS) 
resource "aws_efs_file_system" "this" {
  creation_token   = var.application_name
  encrypted        = true
  performance_mode = "generalPurpose"
  throughput_mode  = "bursting"
}

# EFS Mount Targets
resource "aws_efs_mount_target" "this" {
  for_each = toset(data.aws_subnets.private.ids)

  file_system_id  = aws_efs_file_system.this.id
  subnet_id       = each.value
  security_groups = [aws_security_group.efs.id]
}

# EFS security group
resource "aws_security_group" "efs" {
  name   = "efs"
  vpc_id = var.aws_vpc_id
}

resource "aws_security_group_rule" "ecs_ingress" {
  security_group_id = aws_security_group.efs.id
  type                     = "ingress"
  from_port                = 2049
  to_port                  = 2049
  protocol                 = "tcp"
  source_security_group_id = aws_security_group.ecs_service.id
}

# EFS Access Point
resource "aws_efs_access_point" "this" {
  file_system_id = aws_efs_file_system.this.id
  posix_user {
    gid = 1000
    uid = 1000
  }
  root_directory {
    path = "/home"
    creation_info {
      owner_gid   = 1000
      owner_uid   = 1000
      permissions = 755
    }
  }
}

# EFS Policy
data "aws_iam_policy_document" "this" {
  statement {
    actions = [
      "elasticfilesystem:ClientMount",
      "elasticfilesystem:ClientWrite"
    ]
    effect = "Allow"
    resources = [
      aws_efs_file_system.this.arn,
    ]

    principals {
      type        = "Service"
      identifiers = ["ecs-tasks.amazonaws.com"]
    }
    condition {
      test     = "Bool"
      variable = "aws:SecureTransport"
      values   = ["true"]
    }
  }
}

# EFS Policy Attachment
resource "aws_efs_file_system_policy" "this" {
  file_system_id = aws_efs_file_system.this.id
  policy         = data.aws_iam_policy_document.this.json
}

Here we go. This is why you're here and this is our largest Terraform file. Create ecs.tf to create our ECS cluster, service, etc. Like before, each section of code is annotated.

# ECS Cluster
resource "aws_ecs_cluster" "this" {
  name = var.application_name
}

# ECS Task Definition
resource "aws_ecs_task_definition" "this" {
  family = var.application_name
  container_definitions = templatefile("${path.module}/container_definition.tftpl", {
    container_name          = var.jenkins_controller_identifier,
    container_image         = "jenkins/jenkins:2.479.1", # latest version as of Oct. 11, 24
    jenkins_controller_port = var.jenkins_controller_port
    jenkins_agent_port      = var.jenkins_agent_port
    source_volume           = "home",
    awslogs_group           = aws_cloudwatch_log_group.this.name,
    awslogs_region          = data.aws_region.current.name,
    }
  )
  network_mode             = "awsvpc"
  cpu                      = 1024
  memory                   = 2048
  execution_role_arn       = aws_iam_role.execution.arn
  task_role_arn            = aws_iam_role.task.arn
  requires_compatibilities = ["FARGATE"]
  volume {
    name = "home"
    efs_volume_configuration {
      file_system_id     = aws_efs_file_system.this.id
      transit_encryption = "ENABLED"
      authorization_config {
        access_point_id = aws_efs_access_point.this.id
        iam             = "ENABLED"
      }
    }
  }
}

# Roles and Polices 
resource "aws_iam_role" "execution" {
  name = "ecs-execution"
  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ecs-tasks.amazonaws.com"
        }
      },
    ]
  })
}

resource "aws_iam_role_policy_attachment" "basic_execution_role" {
  role       = aws_iam_role.execution.name
  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
}

resource "aws_iam_role" "task" {
  name = "ecs-task"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ecs-tasks.amazonaws.com"
        }
      },
    ]
  })
}

data "aws_iam_policy_document" "efs_access" {
  statement {
    actions = [
      "elasticfilesystem:ClientMount",
      "elasticfilesystem:ClientWrite"
    ]

    resources = [
      aws_efs_file_system.this.arn
    ]
  }
}

resource "aws_iam_policy" "efs_access" {
  name   = "efs-access"
  policy = data.aws_iam_policy_document.efs_access.json
}

resource "aws_iam_role_policy_attachment" "efs_access" {
  role       = aws_iam_role.task.name
  policy_arn = aws_iam_policy.efs_access.arn
}

data "aws_iam_policy_document" "ecs_access" {
  statement {
    actions = [
      "ecs:RegisterTaskDefinition",
      "ecs:DeregisterTaskDefinition",
      "ecs:ListClusters",
      "ecs:ListTaskDefinitions",
      "ecs:DescribeContainerInstances",
      "ecs:DescribeTaskDefinition",
      "ecs:DescribeClusters",
      "ecs:ListTagsForResource"
    ]
    resources = [
      "*"
    ]
  }

  statement {
    actions = [
      "ecs:ListContainerInstances"
    ]
    resources = [
      aws_ecs_cluster.this.arn
    ]
  }

  statement {
    actions = [
      "ecs:RunTask",
      "ecs:StopTask",
      "ecs:DescribeTasks"
    ]
    resources = [
      "*"
    ]
    condition {
      test     = "ArnEquals"
      variable = "ecs:cluster"

      values = [
        aws_ecs_cluster.this.arn
      ]
    }
  }
}

resource "aws_iam_policy" "ecs_access" {
  name   = "ecs-access"
  policy = data.aws_iam_policy_document.ecs_access.json
}

resource "aws_iam_role_policy_attachment" "ecs_access" {
  role       = aws_iam_role.task.name
  policy_arn = aws_iam_policy.ecs_access.arn
}

data "aws_iam_policy_document" "iam_access" {
  statement {
    actions = [
      "iam:GetRole",
      "iam:PassRole"
    ]

    resources = [
      aws_iam_role.execution.arn,
      aws_iam_role.agent.arn
    ]
  }
}

resource "aws_iam_policy" "iam_access" {
  name   = "iam-access"
  policy = data.aws_iam_policy_document.iam_access.json
}

resource "aws_iam_role_policy_attachment" "iam_access" {
  role       = aws_iam_role.task.name
  policy_arn = aws_iam_policy.iam_access.arn
}

resource "aws_iam_role" "agent" {
  name = "ecs-agent"

  assume_role_policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Action = "sts:AssumeRole"
        Effect = "Allow"
        Principal = {
          Service = "ecs-tasks.amazonaws.com"
        }
      },
    ]
  })
}

resource "aws_iam_role_policy_attachment" "admin_access" {
  role       = aws_iam_role.agent.name
  policy_arn = "arn:aws:iam::aws:policy/AdministratorAccess"
}


# ECS Service 
resource "aws_ecs_service" "this" {
  name            = var.application_name
  launch_type     = "FARGATE"
  cluster         = aws_ecs_cluster.this.arn
  task_definition = aws_ecs_task_definition.this.arn
  desired_count   = 1

  network_configuration {
    subnets          = data.aws_subnets.private.ids
    security_groups  = [aws_security_group.ecs_service.id]
    assign_public_ip = false
  }

  load_balancer {
    target_group_arn = aws_lb_target_group.this.arn
    container_name   = var.jenkins_controller_identifier
    container_port   = var.jenkins_controller_port
  }

  service_registries {
    registry_arn = aws_service_discovery_service.this.arn
    port         = var.jenkins_agent_port
  }
}

# Security Group and Rules
resource "aws_security_group" "ecs_service" {
  name   = "ecs-jenkins-controller"
  vpc_id = var.aws_vpc_id
}

resource "aws_security_group_rule" "alb_ingress" {
  security_group_id = aws_security_group.ecs_service.id

  type                     = "ingress"
  from_port                = var.jenkins_controller_port
  to_port                  = var.jenkins_controller_port
  protocol                 = "tcp"
  source_security_group_id = aws_security_group.alb.id
}

resource "aws_security_group_rule" "service_all_egress" {
  security_group_id = aws_security_group.ecs_service.id

  type        = "egress"
  from_port   = 0
  to_port     = 65535
  protocol    = "tcp"
  cidr_blocks = ["0.0.0.0/0"]
}

resource "aws_security_group_rule" "jenkins_agent_ingress" {
  security_group_id = aws_security_group.ecs_service.id

  type                     = "ingress"
  from_port                = var.jenkins_agent_port
  to_port                  = var.jenkins_agent_port
  protocol                 = "tcp"
  source_security_group_id = aws_security_group.ecs_jenkins_agent.id
}

resource "aws_security_group" "ecs_jenkins_agent" {
  name   = "ecs-jenkins-agents"
  vpc_id = var.aws_vpc_id
}

resource "aws_security_group_rule" "agent_all_egress" {
  security_group_id = aws_security_group.ecs_jenkins_agent.id
  type              = "egress"
  from_port         = 0
  to_port           = 65535
  protocol          = "tcp"
  cidr_blocks       = ["0.0.0.0/0"]
}

# CloudWatch Log Group
resource "aws_cloudwatch_log_group" "this" {
  name              = var.application_name
  retention_in_days = 30
}

Create service-discovery.tf next. What if we want to add a Jenkins agent? Service Discovery will provide a good way to manage everything. When we launch a new task, it will register itself with discovery. When other resources, want to reference that task, they can query Service Discovery.

# Description: This file contains the terraform code to create a private DNS namespace and a service discovery service.

# Service Discovery namespace
resource "aws_service_discovery_private_dns_namespace" "this" {
  name = var.application_name
  vpc  = var.aws_vpc_id
}

# Service Discovery service
resource "aws_service_discovery_service" "this" {
  name = var.jenkins_controller_identifier

  dns_config {
    namespace_id   = aws_service_discovery_private_dns_namespace.this.id
    routing_policy = "MULTIVALUE"

    dns_records {
      ttl  = 60
      type = "A"
    }
    dns_records {
      ttl  = 60
      type = "SRV"
    }
  }
}

But how are we going to access our cluster? Aren't our tasks in private subnets? Good question and yes, they are. We need an Application Load Balancer (ALB). Create alb.tf for ALB configuration.

# ALB
resource "aws_lb" "this" {
  name = var.application_name

  internal           = false
  load_balancer_type = "application"
  security_groups    = [aws_security_group.alb.id]
  subnets            = data.aws_subnets.public.ids
}

# ALB Security Group
resource "aws_security_group" "alb" {
  name   = "alb"
  vpc_id = var.aws_vpc_id
}

# Open HTTP port 80 to the world
resource "aws_security_group_rule" "http_ingress" {
  security_group_id = aws_security_group.alb.id

  type        = "ingress"
  from_port   = 80
  to_port     = 80
  protocol    = "tcp"
  cidr_blocks = ["0.0.0.0/0"]
}

# Open HTTP port 8080 to allow ALB to communicate with ECS service
resource "aws_security_group_rule" "ecs_egress" {
  security_group_id        = aws_security_group.alb.id
  type                     = "egress"
  from_port                = 8080
  to_port                  = 8080
  protocol                 = "tcp"
  source_security_group_id = aws_security_group.ecs_service.id
}


# ALB Listener
resource "aws_lb_listener" "this" {
  load_balancer_arn = aws_lb.this.arn
  port              = "80"
  protocol          = "HTTP"

  default_action {
    type             = "forward"
    target_group_arn = aws_lb_target_group.this.arn
  }
}

# ALB Target Group
resource "aws_lb_target_group" "this" {
  name        = var.application_name
  port        = 8080
  protocol    = "HTTP"
  target_type = "ip"
  vpc_id      = var.aws_vpc_id

  health_check {
    path = "/login"
  }
}

Finally, create outputs.tf and grab the CloudWatch Log Group and Jenkins URL (ALB DNS).

output "ecs_cloudwatch_log_group_name" {
  description = "Name of the ECS CloudWatch Log group"
  value       = aws_cloudwatch_log_group.this.name
}

output "jenkins_url" {
  description = "URL of the Jenkins server"
  value       = "http://${aws_lb.this.dns_name}"
}

When you see this, you're done! Grab the jenkins_url output value and paste it into a browser.

Ah...you need a password. Let's go get that.

Navigate to Amazon Elastic Container Service -> Clusters -> serverless-jenkins-on-ecs -> Tasks -> your-task -> Logs and get the password from the logs.

Now, you're in! At this point, go ahead and install your plugins, create an admin user, etc. You're set. Jenkins is ready to use. Before we conclude, let's look at what we've done.