I wanna setup ECS EC2 Nodes in order to run my Jenkins slaves. I read the documentation of the AWS-ECS plugin and replicated the exact steps of configuring Jenkins Master and ECS Nodes with Auto Scaling Group as Capacity Providers, all with in the same VPC and Subnet.
As expected the agents are provisioning and tasks which is Jenkins inbound agents are connected to the master with JNLP.
But, the pipeline gets stuck and builds forever, either saying:
Jenkins doesn’t have label ‘…’, when the task defination is getting changed
Or,
Waiting for next executor.
Edit: Here’s the task defination generated by the plugin
{
"taskDefinitionArn": "arn:aws:ecs:us-east-1:971422682872:task-definition/testing-testing-td:4",
"containerDefinitions": [
{
"name": "testing-testing-td",
"image": "jenkins/inbound-agent",
"cpu": 1024,
"memoryReservation": 2048,
"portMappings": [],
"essential": true,
"environment": [],
"mountPoints": [
{
"sourceVolume": "docker",
"containerPath": "/var/run/docker.sock",
"readOnly": false
}
],
"volumesFrom": [],
"privileged": false,
"logConfiguration": {
"logDriver": "awslogs",
"options": {
"awslogs-group": "/ecs-jenkins-cluster/jenkins-agents",
"awslogs-region": "us-east-1",
"awslogs-stream-prefix": "jenkins-agents"
}
},
"systemControls": []
}
],
"family": "testing-testing-td",
"taskRoleArn": "arn:aws:iam::971422682872:role/ecsTaskExecutionRole",
"executionRoleArn": "arn:aws:iam::971422682872:role/ecsTaskExecutionRole",
"networkMode": "host",
"revision": 4,
"volumes": [
{
"name": "docker",
"host": {
"sourcePath": "/var/run/docker.sock"
}
}
],
"status": "ACTIVE",
"requiresAttributes": [
{
"name": "com.amazonaws.ecs.capability.logging-driver.awslogs"
},
{
"name": "ecs.capability.execution-role-awslogs"
},
{
"name": "com.amazonaws.ecs.capability.task-iam-role-network-host"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.19"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.21"
},
{
"name": "com.amazonaws.ecs.capability.task-iam-role"
},
{
"name": "com.amazonaws.ecs.capability.docker-remote-api.1.18"
}
],
"placementConstraints": [],
"compatibilities": [
"EXTERNAL",
"EC2"
],
"registeredAt": "2024-12-26T19:24:39.462Z",
"registeredBy": "arn:aws:sts::971422682872:assumed-role/ecs-jenkins-access/i-0fa22ce5559ab9423",
"enableFaultInjection": false,
"tags": [
{
"key": "jenkins.label",
"value": "testing"
},
{
"key": "jenkins.templatename",
"value": "testing-td"
}
]
}
Main Purpose: I need to use ECS EC2 launch type, which uses an Auto Scaling Group(spot instances under the hood) to run Jenkins inbound agents.
For the configuration, of ASG the launch template uses this user-data script:
#!/bin/bash
set -e
# Update and upgrade the system
sudo apt update -y && sudo apt upgrade -y
# Install Docker
sudo apt install -y docker.io
sudo systemctl start docker
sudo systemctl enable docker
# Install Java
sudo apt install -y openjdk-21-jdk
java --version
# Install Maven
sudo apt install -y maven
# Configure Maven environment
echo "export MAVEN_HOME=/usr/share/maven" | sudo tee /etc/profile.d/maven.sh
echo "export MAVEN_CONFIG=/etc/maven" | sudo tee -a /etc/profile.d/maven.sh
echo "export PATH=\$MAVEN_HOME/bin:\$PATH" | sudo tee -a /etc/profile.d/maven.sh
sudo chmod +x /etc/profile.d/maven.sh
source /etc/profile.d/maven.sh
# Add user to Docker group
sudo usermod -aG docker $USER
# Install AWS CLI
sudo snap install aws-cli --classic
# Restart Docker service
sudo systemctl restart docker
# Configure AWS ECS
export AWS_REGION="us-east-1"
export OS_PACKAGE="amd64.deb"
curl -O https://s3.${AWS_REGION}.amazonaws.com/amazon-ecs-agent-${AWS_REGION}/amazon-ecs-init-latest.${OS_PACKAGE}
sudo dpkg -i amazon-ecs-init-latest.${OS_PACKAGE}
sudo sed -i '/\[Unit\]/a After=cloud-final.service' /lib/systemd/system/ecs.service
echo "ECS_CLUSTER=new-cluster" | sudo tee /etc/ecs/ecs.config
sudo systemctl enable ecs
sudo systemctl daemon-reload
sudo systemctl restart ecs
# Reboot the system to apply kernel upgrades
sudo reboot
And here’s the pipeline:
pipeline {
agent {
label 'ecs-build-agents'
}
environment {
JAR_NAME = 'demo-spring-application.jar'
S3_BUCKET = 'jenkins-spring-boot-build'
AWS_REGION = 'us-east-1'
SPOT_INSTACES = 'ec2-spot-fleet-agents'
TERRAFORM_INSTANCES = 'terraform-agents'
FARGATE_INSTANCES = 'deepanshu-jenkins-agent'
MASTER_NODE = 'master-node'
}
stages {
stage('Checkout to Master') {
// agent {
// node "${MASTER_NODE}"
// }
steps {
git branch: 'master', url: 'https://github.com/deepanshu-rawat6/demo-spring-application'
}
}
stage('Validate Tools') {
// agent { label "${TERRAFORM_INSTANCES}" }
steps {
sh '''
echo "Validating Java and Maven tools:"
java --version || { echo "Java not found!"; exit 1; }
mvn --version || { echo "Maven not found!"; exit 1; }
'''
}
}
stage('Build Application') {
// agent { label "${TERRAFORM_INSTANCES}" }
steps {
sh '''
echo "Setting up JAR name dynamically in pom.xml"
sed -i 's/<finalName>.*<\\/finalName>/<finalName>${JAR_NAME}<\\/finalName>/' pom.xml
echo "Starting build process..."
mvn clean install -Djar.finalName=${JAR_NAME}
ls -la
'''
}
}
stage('Find Generated JAR') {
// agent { label "${TERRAFORM_INSTANCES}" }
steps {
script {
sh '''
echo "Searching for generated JAR:"
find target -name "*.jar" -exec ls -lh {} \\;
'''
}
}
}
stage('Verify and Run Docker') {
// agent { label "${TERRAFORM_INSTANCES}" }
steps {
sh '''
echo "Verifying Docker installation..."
sudo docker --version || { echo "Docker not found!"; exit 1; }
echo "Testing a secure Docker container:"
sudo docker run hello-world
'''
}
}
stage('Stress Test') {
steps {
sh '''
docker compose up
'''
}
}
stage('Upload JAR to S3') {
// agent { label "${TERRAFORM_INSTANCES}" }
steps {
sh '''
echo "Uploading JAR to secure S3 bucket..."
ls ./target
aws s3 cp ./target/SpringBootFirst-0.0.1-SNAPSHOT.jar s3://${S3_BUCKET}/my-builds/build.jar --sse AES256
'''
}
post {
success {
echo 'JAR uploaded to S3.'
}
failure {
echo 'JAR upload failed. Please check the logs.'
}
}
}
}
}