Managing Multiple S3 Buckets with Fine-Grained Access Policies Using Terraform

Terraform
#1

Hello Folks,

Terraform Level-3 Task-7 is getting failed- “due to policy missing for ‘datacenter-dev-bucket-26876’”. However, terraform show commands display policy is created for dev bucket.

main.tf

######################################

Create S3 Buckets (Dev, Staging, Prod)

######################################

resource “aws_s3_bucket” “bucket_creation_using_for_loop” {
for_each = var.KKE_ENV_TAGS
bucket = each.value.bucket_name

tags = merge(
{
Name = each.value.bucket_name
Environment = each.key

},
each.value.owner != "" ? { Owner = each.value.owner } : {}

)

lifecycle {
ignore_changes = [tags]
}

}

######################################

Lifecycle Rules (for buckets with backup = true)

######################################

resource “aws_s3_bucket_lifecycle_configuration” “kke_lifecycle” {
for_each = {
for env, config in var.KKE_ENV_TAGS :
env => config if lookup(config, “backup”, false)
}

bucket = aws_s3_bucket.bucket_creation_using_for_loop[each.key].id

rule {
id = “MoveToGlacier”
status = “Enabled”

filter {
  prefix = ""
}

transition {
  days          = 30
  storage_class = "GLACIER"
}

}
}

######################################

Public Read Bucket Policy

######################################

resource “aws_s3_bucket_policy” “kke_bucket_policy” {
for_each = var.KKE_ENV_TAGS

bucket = aws_s3_bucket.bucket_creation_using_for_loop[each.key].id

policy = jsonencode({
Version = “2012-10-17”
Statement = [
{
Sid = “PublicReadAccess”
Effect = “Allow”
Principal = “"
Action = [
"s3:Get”,
“s3:List*”,
“s3:Describe*”
]
Resource = [
aws_s3_bucket.bucket_creation_using_for_loop[each.key].arn,
“${aws_s3_bucket.bucket_creation_using_for_loop[each.key].arn}/*”
]
}
]
})

depends_on = [aws_s3_bucket.bucket_creation_using_for_loop]
}

############################################

variables.tf

############################################

variable “KKE_ENV_TAGS” {
description = “Environment specific metadata”
type = map(object({
bucket_name = string
owner = string
backup = optional(bool, false)
}))
default = {
Dev = {
bucket_name = “nautilus-dev-bucket-31091”
owner = “Alice”
backup = false
}
Staging = {
bucket_name = “nautilus-staging-bucket-31091”
owner = “Bob”
backup = true
}
Prod = {
bucket_name = “nautilus-prod-bucket-31091”
owner = “Carol”
backup = true
}
}
}

############################################

outputs.tf

############################################

output “kke_bucket_names” {
description = “Names of the created S3 buckets”
value = [for b in aws_s3_bucket.bucket_creation_using_for_loop : b.bucket]
}

Can someone what went wrong here?

Thanks,
Sohel Khan.

#2

Hi @Sohelk

I’ve just checked it, and it’s working properly on my end, so this issue seems to be user-specific. Could you please share your code again using the code block feature in this forum? I’ll review it and provide feedback.

image

#3

Please find below .tf files,

######################################
#main.tf
# Create S3 Buckets (Dev, Staging, Prod)
######################################

resource “aws_s3_bucket” “bucket_creation_using_for_loop” {
for_each = var.KKE_ENV_TAGS
bucket = each.value.bucket_name

tags = merge(
{
Name = each.value.bucket_name
Environment = each.key



},
each.value.owner != “” ? { Owner = each.value.owner } : {}

)

lifecycle {
ignore_changes = [tags]
}

}

######################################

# Lifecycle Rules (for buckets with backup = true)

######################################

resource “aws_s3_bucket_lifecycle_configuration” “kke_lifecycle” {
for_each = {
for env, config in var.KKE_ENV_TAGS :
env => config if lookup(config, “backup”, false)
}

bucket = aws_s3_bucket.bucket_creation_using_for_loop[each.key].id

rule {
id = “MoveToGlacier”
status = “Enabled”

filter {
prefix = “”
}

transition {
days = 30
storage_class = “GLACIER”
}

}
}

######################################

# Public Read Bucket Policy

######################################

resource “aws_s3_bucket_policy” “kke_bucket_policy” {
for_each = var.KKE_ENV_TAGS

bucket = aws_s3_bucket.bucket_creation_using_for_loop[each.key].id

policy = jsonencode({
Version = “2012-10-17”
Statement = [
{
Sid = “PublicReadAccess”
Effect = “Allow”
Principal = “"
Action = [
"s3:Get”,
“s3:List*”,
“s3:Describe*”
]
Resource = [
aws_s3_bucket.bucket_creation_using_for_loop[each.key].arn,
“${aws_s3_bucket.bucket_creation_using_for_loop[each.key].arn}/*”
]
}
]
})

depends_on = [aws_s3_bucket.bucket_creation_using_for_loop]
}

############################################

# variables.tf

############################################

variable “KKE_ENV_TAGS” {
description = “Environment specific metadata”
type = map(object({
bucket_name = string
owner = string
backup = optional(bool, false)
}))
default = {
Dev = {
bucket_name = “nautilus-dev-bucket-31091”
owner = “Alice”
backup = false
}
Staging = {
bucket_name = “nautilus-staging-bucket-31091”
owner = “Bob”
backup = true
}
Prod = {
bucket_name = “nautilus-prod-bucket-31091”
owner = “Carol”
backup = true
}
}
}

############################################

# outputs.tf

############################################

output “kke_bucket_names” {
description = “Names of the created S3 buckets”
value = [for b in aws_s3_bucket.bucket_creation_using_for_loop : b.bucket]
}
#4

Hi @Sohelk

The task asks to use the lifecycle block with ignore_changes to protect the tags, not aws_s3_bucket_lifecycle_configuration. Please try again.

#5

Hi @raymond.baoly, please review my code once if you get a chance. I am also facing similar issue. I can see the lifecycle policy in the state file for dev bucket.

resource “aws_s3_bucket” “buckets” {
for_each = var.KKE_ENV_TAGS
bucket = each.value.Name
tags = {
Name = each.value.Name
Environment = each.value.Environment
Owner = each.value.Owner
Backup = each.value.Backup ? “true” : “false”
}
lifecycle {
ignore_changes = [
tags
]
}
}

resource “aws_s3_bucket_lifecycle_configuration” “lifecycle” {
for_each = { for k, v in var.KKE_ENV_TAGS : k => v if v.Backup }
bucket = aws_s3_bucket.buckets[each.key].id
rule {
id = “MoveToGlacier”
status = “Enabled”
transition {
days = 30
storage_class = “GLACIER”
}
}
}

resource “aws_s3_bucket_policy” “policy” {
for_each = var.KKE_ENV_TAGS
bucket = aws_s3_bucket.buckets[each.key].id
policy = jsonencode({
Version = “2012-10-17”
Statement = [
{
Effect = “Allow”
Principal = “"
Action = “S3:GetObject”
Resource = "${aws_s3_bucket.buckets[each.key].arn}/”
}
]
})
depends_on = [aws_s3_bucket.buckets]
}

variable “KKE_ENV_TAGS” {
type = map(object({
Name = string
Environment = string
Owner = string
Backup = bool
}))
default = {
dev = {
Name = “devops-dev-bucket-31190”
Environment = “Dev”
Owner = “Alice”
Backup = false
}
staging = {
Name = “devops-staging-bucket-31190”
Environment = “Staging”
Owner = “Bob”
Backup = true
}
Prod = {
Name = “devops-prod-bucket-31190”
Environment = “Prod”
Owner = “Carol”
Backup = true
}
}
}

output “kke_bucket_name” {
value = [for b in aws_s3_bucket.buckets : b.bucket]
}

#6

@Venkata_Pavan

Please use a code block, it’s really hard to check the code without proper formatting.

#7

@raymond.baoly , would you mind helping me on how to use code block. i couldn’t figure it out. sorry for the inconvenience

#8

This guide will get you started. It’s pretty simple – 3 backticks on on line, 3 backticks on the line after your code, and your code in the middle. It looks like this:

image
image1508×268 4.45 KB

#9 
resource "aws_s3_bucket" "buckets" {
  for_each = var.KKE_ENV_TAGS
  bucket   = each.value.Name
  tags = {
    Name        = each.value.Name
    Environment = each.value.Environment
    Owner       = each.value.Owner
    Backup      = each.value.Backup ? "true" : "false"
  }
  lifecycle {
    ignore_changes = [
      tags
    ]
  }
}

resource "aws_s3_bucket_lifecycle_configuration" "lifecycle" {
  for_each = { for k, v in var.KKE_ENV_TAGS : k => v if v.Backup }
  bucket   = aws_s3_bucket.buckets[each.key].id
  rule {
    id     = "MoveToGlacier"
    status = "Enabled"
    transition {
      days          = 30
      storage_class = "GLACIER"
    }
  }
}

resource "aws_s3_bucket_policy" "policy" {
  for_each = var.KKE_ENV_TAGS
  bucket   = aws_s3_bucket.buckets[each.key].id
  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect    = "Allow"
        Principal = "*"
        Action    = "S3:GetObject"
        Resource  = "${aws_s3_bucket.buckets[each.key].arn}/*"
      }
    ]
  })
  depends_on = [aws_s3_bucket.buckets]
}

variable "KKE_ENV_TAGS" {
  type = map(object({
    Name        = string
    Environment = string
    Owner       = string
    Backup      = bool
  }))
  default = {
    dev = {
      Name        = "devops-dev-bucket-31190"
      Environment = "Dev"
      Owner       = "Alice"
      Backup      = false
    }
    staging = {
      Name        = "devops-staging-bucket-31190"
      Environment = "Staging"
      Owner       = "Bob"
      Backup      = true
    }
    Prod = {
      Name        = "devops-prod-bucket-31190"
      Environment = "Prod"
      Owner       = "Carol"
      Backup      = true
    }
  }
}

output "kke_bucket_name" {
  value = [for b in aws_s3_bucket.buckets : b.bucket]
}

Thanks @rob_kodekloud .

please help to review @rob_kodekloud @raymond.baoly

#10

Hi @Venkata_Pavan

You are facing the same issue as with the owner post. The task requires using the lifecycle block with ignore_changes to protect the tags, not aws_s3_bucket_lifecycle_configuration

image
image1907×937 104 KB

#11

Hi @raymond.baoly, changed the lifecycle rule as you mentioned but the error I am getting is policy is missing for dev bucket. But I can see that in state file it is created. can you please check what am i missing here

image
image1755×976 136 KB

#12

Resolved now. used below block in aws_s3_bucket_policy

lifecycle {
    create_before_destroy = true
}
#13

As you might have noticed, I have already included the lifecycle block to ignore tags in my code. I used aws_s3_bucket_lifecycle_configuration because the requirement mentioned moving data to Glacier storage after 30 days.

Could you please share the code that worked for you? That would really help me better understand the mistakes I might be making.

Thanks in advance for your support.

#14

Hi @Sohelk

Please try again without using aws_s3_bucket_lifecycle_configuration and instead add the lifecycle block to the aws_s3_bucket resource. If that doesn’t work, I’ll try it myself and share the code here.

#15 
resource "aws_s3_bucket" "bucket_creation_using_for_loop" {
  for_each = var.KKE_ENV_TAGS

  bucket = each.value.bucket_name

  tags = merge(
    {
      Name        = each.value.bucket_name
      Environment = each.key
    },
    each.value.owner != "" ? { Owner = each.value.owner } : {}
  )

  
  lifecycle {
    ignore_changes = [tags]
  }
}


resource "aws_s3_bucket_policy" "kke_bucket_policy" {
  for_each = var.KKE_ENV_TAGS

  bucket = aws_s3_bucket.bucket_creation_using_for_loop[each.key].id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect    = "Allow"
        Principal = "*"
        Action    = [
          "s3:Get*",
          "s3:List*",
          "s3:Describe*"
        ]
        Resource = [
          aws_s3_bucket.bucket_creation_using_for_loop[each.key].arn,
          "${aws_s3_bucket.bucket_creation_using_for_loop[each.key].arn}/*"
        ]
      }
    ]
  })

  depends_on = [aws_s3_bucket.bucket_creation_using_for_loop]
}

# variables.tf
variable "KKE_ENV_TAGS" {
  type = map(object({
    bucket_name = string
    owner       = string
    backup      = optional(bool, false)
  }))
  default = {
    Dev = {
      bucket_name = "datacenter-dev-bucket-31825"
      owner       = "Alice"
      backup      = false
    }
    Staging = {
      bucket_name = "datacenter-staging-bucket-31825"
      owner       = "Bob"
      backup      = true
    }
    Prod = {
      bucket_name = "datacenter-prod-bucket-31825"
      owner       = "Carol"
      backup      = true
    }
  }
}

#outputs.tf

output "kke_bucket_names" {
  description = "Names of all created S3 buckets"
  value       = [for b in aws_s3_bucket.bucket_creation_using_for_loop : b.bucket]
}

image

Still getting same error.

#16

@Sohelk

Please try again and refer to the solution.

variables.tf

variable "KKE_ENV_TAGS" {
  description = "Environment-specific metadata for S3 buckets"
  type = map(object({
    bucket_name = string
    owner       = string
    backup      = bool
  }))
  default = {
    Dev = {
      bucket_name = "nautilus-dev-bucket-29025"
      owner       = "Alice"
      backup      = false
    }
    Staging = {
      bucket_name = "nautilus-staging-bucket-29025"
      owner       = "Bob"
      backup      = true
    }
    Prod = {
      bucket_name = "nautilus-prod-bucket-29025"
      owner       = "Carol"
      backup      = true
    }
  }
}

main.tf

# Create S3 buckets per environment
resource "aws_s3_bucket" "kke_buckets" {
  for_each = var.KKE_ENV_TAGS

  bucket = each.value.bucket_name
  acl    = "private"

  # Add tags
  tags = {
    Name        = each.value.bucket_name
    Environment = each.key
    Owner       = each.value.owner
    Backup      = each.value.backup ? "true" : "false"
  }

  lifecycle {
    ignore_changes = [tags]
  }
}

# Add lifecycle rules for Staging and Prod buckets
resource "aws_s3_bucket_lifecycle_configuration" "kke_lifecycle" {
  for_each = { for k, v in var.KKE_ENV_TAGS : k => v if v.backup }

  bucket = aws_s3_bucket.kke_buckets[each.key].id

  rule {
    id     = "MoveToGlacier"
    status = "Enabled"

    transition {
      days          = 30
      storage_class = "GLACIER"
    }
  }
}

# Bucket policy to allow public read
resource "aws_s3_bucket_policy" "kke_policy" {
  for_each = var.KKE_ENV_TAGS

  bucket = aws_s3_bucket.kke_buckets[each.key].id

  policy = jsonencode({
    Version = "2012-10-17"
    Statement = [
      {
        Effect    = "Allow"
        Principal = "*"
        Action    = ["s3:GetObject"]
        Resource  = "${aws_s3_bucket.kke_buckets[each.key].arn}/*"
      }
    ]
  })

  depends_on = [aws_s3_bucket.kke_buckets]
}

outputs.tf

output "kke_bucket_names" {
  description = "Names of the created S3 buckets"
  value       = [for b in aws_s3_bucket.kke_buckets : b.bucket]
}

image
image1909×938 136 KB