Snowflake Pipe creation with SNS topic fails with the following error : Invalid Parameter - amazon-s3

I am trying to create a snowflake pipe and it fails with the following error
090040 (XX000): Pipe Notifications bind failure "Invalid parameter: TopicArn (Service: Sns, Status Code: 400, Request ID: d7fbc360-1faf-5216-bfab-9f037f6ac63d): AwsErrorDetails(errorMessage=Invalid parameter: TopicArn, errorCode=InvalidParameter, serviceName=Sns)"
create pipe PROJECT.PUBLIC.mypipe
auto_ingest=true
aws_sns_topic='arn:aws:sns:us-east-2:4xxxxxxxx:SNSexample'
as
copy into PROJECT.PUBLIC.LOADING_STATION(INFO)
from (select $1 from #REFERENCE_STATION)
Here is a documentation for reference https://docs.snowflake.com/en/user-guide/data-load-snowpipe-auto-s3.html#prerequisite-create-an-amazon-sns-topic-and-subscription
Can someone shed light on how to resolve this.
Here is the policy in SNS
> "Version": "2008-10-17", "Id": "__default_policy_ID",
> "Statement": [
> {
> "Sid": "__default_statement_ID",
> "Effect": "Allow",
> "Principal": {
> "AWS": "*"
> },
> "Action": [
> "SNS:GetTopicAttributes",
> "SNS:SetTopicAttributes",
> "SNS:AddPermission",
> "SNS:RemovePermission",
> "SNS:DeleteTopic",
> "SNS:ListSubscriptionsByTopic",
> "SNS:Receive"
> ],
> "Resource": "arn:aws:sns:us-east-2:accountid:SNSexample.fifo",
> "Condition": {
> "StringEquals": {
> "AWS:SourceOwner": "snowflakeaccountid"
> }
> }
> },
> {
> "Sid": "1",
> "Effect": "Allow",
> "Principal": {
> "AWS": "arn:aws:iam::snowflakeaccountid:user/cie40000-s"
> },
> "Action": "sns:Subscribe",
> "Resource": "arn:aws:sns:us-east-2:myaccountid:SNSexample.fifo"
> },
> {
> "Sid": "s3-event-notifier",
> "Effect": "Allow",
> "Principal": {
> "Service": "s3.amazonaws.com"
> },
> "Action": "SNS:Publish",
> "Resource": "arn:aws:sns:us-east-2:myaccountid:SNSexample.fifo",
> "Condition": {
> "ArnLike": {
> "aws:SourceArn": "arn:aws:s3:*:*:bucketname"
> }
> }
> } ] }

Related

how to upload files to s3 with server side encryption enabled by custom kms key

created 2 new kms key:
custom-client-side-encrypt-kms-key. (encrypt file on client side);
custom-server-side-encrypt-kms-key. (enable S3 server side encryption)
created new user:
test-user-encrypt
gave the user access to key and bucket;
set up new key policy to allow user to use the key
ran command(new user credential, with key id options)
aws s3api put-object --body newFile --bucket <bucket-name> --key inbound/newFile --server-side-encryption aws:kms --ssekms-key-id newKeyId
error: when calling the PutObject operation: Access Denied
ran command (new user credential, without key id options)
aws s3api put-object --body newFile --bucket <bucket-name> --key inbound/newFile --server-side-encryption aws:kms
it works, but the object was encrypted with AWS managed default kms key
based on the above test, it seems the user has correct permissions to S3, but kms key permission was not correct.
here is my policies in brief:
key policy
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "Enable IAM User Permissions",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::xxxxxxid:root"
},
"Action": "kms:*",
"Resource": "*"
},
{
"Sid": "Allow use of the key",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::xxxxxxid:user/test_user_encrypt"
},
"Action": [
"kms:ReEncrypt",
"kms:GenerateDataKey",
"kms:Encrypt",
"kms:DescribeKey",
"kms:Decrypt"
],
"Resource": "*",
"Condition": {
"ForAnyValue:IpAddress": {
"aws:SourceIp": [
"xx.xx.xx.xx",(IP addresses allowed)
"xx.xx.xx.xx",
]
}
}
},
{
"Sid": "Allow attachment of persistent resources",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::xxxxid:user/test_user_encrypt"
},
"Action": [
"kms:RevokeGrant",
"kms:ListGrants",
"kms:CreateGrant"
],
"Resource": "*",
"Condition": {
"Bool": {
"kms:GrantIsForAWSResource": "true"
}
}
"Condition": {
"ForAnyValue:IpAddress": {
"aws:SourceIp": [
"xx.xx.xx.xx", (IP addresses allowed)
"xx.xx.xx.xx",
]
}
}
}
]
}
IAM user policy
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kms:ReEncrypt",
"kms:GenerateDataKey",
"kms:Encrypt",
"kms:DescribeKey",
"kms:Decrypt"
],
"Resource": "arn:aws:kms:<region>:xxID:key/custom-client-side-encrypt-kms-key-id",
"Condition": {
"ForAnyValue:IpAddress": {
"aws:SourceIp": [
"xx.xx.xx.xx" (IP address allowed)
]
}
}
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"kms:RevokeGrant",
"kms:ReEncrypt",
"kms:ListGrants",
"kms:GenerateDataKey",
"kms:Encrypt",
"kms:DescribeKey",
"kms:Decrypt",
"kms:CreateGrant"
],
"Resource": "arn:aws:kms:<region>:xxxID:key/<custom-server-side-encrypt-key-id>"
"Condition": {
"ForAnyValue:IpAddress": {
"aws:SourceIp": [
"xx.xx.xx.xx", (IP address allowed)
]
}
}
},
{
"Sid": "",
"Effect": "Allow",
"Action": "s3:PutObject",
"Resource": "arn:aws:s3:::bucket/inbound/*",
"Condition": {
"ForAnyValue:IpAddress": {
"aws:SourceIp": [
"xx.xxx.xxx.xx", (IP address allowed)
]
}
}
},
{
"Sid": "",
"Effect": "Deny",
"Action": [
"s3:PutObjectAcl",
"s3:PutObject",
"s3:ListMultipartUploadParts"
],
"Resource": [
"arn:aws:s3:::bucket/*/*/*",
"arn:aws:s3:::bucket/*/*/",
"arn:aws:s3:::bucket/*/"
]
}
]
}
bucket policy
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::xxxid:user/test_user_encrypt"
},
"Action": "s3:PutObject",
"Resource": "arn:aws:s3:::bucket/inbound/*",
"Condition": {
"ForAnyValue:IpAddress": {
"aws:SourceIp": [
"xx.xx.xx.xx", (IP address allowed)
]
}
}
},
{
"Sid": "",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::xxxid:user/test_user_encrypt"
},
"Action": "s3:ListMultipartUploadParts",
"Resource": "arn:aws:s3:::bucket/inbound/*",
"Condition": {
"ForAnyValue:IpAddress": {
"aws:SourceIp": [
"xx.xx.xx.xx", (IP address allowed)
]
}
}
},
{
"Sid": "DenyIncorrectEncryptKey",
"Effect": "Deny",
"Principal": {
"AWS": "*"
},
"Action": "s3:PutObject",
"Resource": "arn:aws:s3:::bucket/*",
"Condition": {
"StringNotEquals": {
"s3:x-amz-server-side-encryption-aws-kms-key-id": "arn:aws:kms:region:xxxid:key/custom-server-side-encrypt-kms-key-id"
}
}
},
{
"Sid": "DenyUnEncryptedObjectUploads",
"Effect": "Deny",
"Principal": {
"AWS": "*"
},
"Action": "s3:PutObject",
"Resource": "arn:aws:s3:::bucket/*",
"Condition": {
"StringNotEquals": {
"s3:x-amz-server-side-encryption": "aws:kms"
}
}
},
{
"Sid": "AllowSSLRequestsOnly",
"Effect": "Deny",
"Principal": {
"AWS": "*"
},
"Action": "s3:*",
"Resource": [
"arn:aws:s3:::bucket/*",
"arn:aws:s3:::bucket"
],
"Condition": {
"Bool": {
"aws:SecureTransport": "false"
}
}
}
]
}
I figured out. I should NOT put condition "ip range" around kms:grant permissions.
"Condition": {
"ForAnyValue:IpAddress": {
"aws:SourceIp": [
"xx.xx.xx.xx", (IP address allowed)
]
}
}
that condition make kms:grant invalid. my guess....
after removed the condition, it works fine.

AccessDenied on aws cp for bucket that encrypts objects with SSE KMS

I'm trying to download an SSE-KMS encrypted object from my S3 bucket secret-bucket, where my AWS account id is XXXXXXXX with command
aws s3 cp s3://secret-bucket/file.json ./file.json
IAM user policy
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"kms:*"
],
"Resource": "*"
}
]
}
{
"Statement": [
{
"Effect": "Allow",
"Action": "s3:*",
"Resource": "*"
}
]
}
S3 bucket policy
{
"Version": "2012-10-17",
"Id": "access",
"Statement": [
{
"Sid": "get-access",
"Effect": "Allow",
"Principal": {
"AWS": [
"arn:aws:iam::XXXXXXXX:root"
]
},
"Action": [
"s3:GetObject*",
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::secret-bucket/*",
"arn:aws:s3:::secret-bucket"
]
}
]
}
KMS key policy
{
"Version": "2012-10-17",
"Id": "bucket-key-default-1",
"Statement": [
{
"Sid": "Allow administration of the key",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::XXXXXXXX:root"
},
"Action": "kms:*",
"Resource": "*"
}
]
}
Apparently I'm missing some permissions somewhere, but can't seem to figure out which one. Any ideas?

Why am I getting "AccessDenied" from S3 DeleteObjects?

I'm trying to delete a number of objects in an S3 bucket like so (Lambda / Node.js):
exports.handler = async function(event, context) {
const s3Params = {
Bucket: 'my-bucket',
Delete: {
Objects: [ 'my-bucket/dir/file1.json', 'my-bucket/dir/file2.json' ],
Quiet: False
}
}
const result = await s3.deleteObjects(s3Params).promise()
return result
}
But I get:
{
result: {
Deleted: [],
Errors: [
{Key: "my-bucket/dir/file1.json", Code: "AccessDenied", Message: "Access Denied"},
{Key: "my-bucket/dir/file1.json", Code: "AccessDenied", Message: "Access Denied"}
]
}
}
Here is my role policy:
{
"RoleName": "S3CleanupRole",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"s3:ListBucket",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::my-bucket"
],
"Effect": "Allow"
},
{
"Action": [
"DynamoDB:Query"
],
"Resource": [
"arn:aws:dynamodb:us-east-1:514141358776:table/buckets-to-clean-out"
],
"Effect": "Allow"
}
]
},
"PolicyName": "S3CleanupPolicy"
}
What's the problem?
(Kudos to Ravi Ramanujam for this - his answer to AWS S3 Access Denied on delete helped me solve this problem.)
My policy is wrong. The permission for s3:DeleteObject applies only to the bucket arn:aws:dynamodb:us-east-1:514141358776:table/buckets-to-clean-out and not any of its contents. The correct policy allows s3:DeleteObject on the bucket contents, viz:
{
"RoleName": "S3CleanupRole",
"PolicyDocument": {
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::my-bucket"
],
"Effect": "Allow"
},
{
"Action": [
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::my-bucket/*"
],
"Effect": "Allow"
},
{
"Action": [
"DynamoDB:Query"
],
"Resource": [
"arn:aws:dynamodb:us-east-1:514141358776:table/buckets-to-clean-out"
],
"Effect": "Allow"
}
]
},
"PolicyName": "S3CleanupPolicy"
}
The crucial piece is below. Note that the "Resource" section contains arn:aws:s3:::my-bucket/* rather than just arn:aws:s3:::my-bucket
{
"Action": [
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::my-bucket/*"
],
"Effect": "Allow"
}

S3 policy for allowing requests with Cognito credentials: AWS

I'm trying to policy json into my S3 bucket which will make my bucket accessible using Cognito credentials. Following is the policy:
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"Federated": "cognito-identity.amazonaws.com"
},
"Action": "s3:ListBucket",
"Resource": "arn:aws:s3:::bucketName12345",
"Condition": {
"StringLike": {
"s3:prefix": "cognito/angularApplicationName/"
}
}
},
{
"Effect": "Allow",
"Principal": {
"Federated": "cognito-identity.amazonaws.com"
},
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::bucketName12345/cognito/angularApplicationName/${cognito-identity.amazonaws.com:sub}",
"arn:aws:s3:::bucketName12345/cognito/angularApplicationName/${cognito-identity.amazonaws.com:sub}/*"
]
}
]
}
but I get a warning saying "your bucket is public....". Since I'm very new to AWS, can somebody point out what am I missing here?

How do I grant access to all subfolders of a folder in Amazon S3?

Here is the policy I wrote in Amazon S3. I thought it should give access to subfolders because of the * but it is giving access denied errors when the user tries to create or view subfolders. How can I change this to work?
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "AllowUserToSeeBucketListInTheConsole",
"Action": [
"s3:ListAllMyBuckets",
"s3:GetBucketLocation"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::*"
]
},
{
"Sid": "AllowRootAndMediaListingOfCompanyBucket",
"Action": [
"s3:ListBucket"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::mycoolbucket"
],
"Condition": {
"StringEquals": {
"s3:prefix": [
"",
"media/"
],
"s3:delimiter": [
"/"
]
}
}
},
{
"Sid": "AllowAllS3ActionsInMediaFolder",
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": [
"arn:aws:s3:::mycoolbucket/media/*"
]
}
]
}
More details:
I logged into the console as the user. I went to the media folder. I then click on a folder inside of media and got the message "Error access denied".
You are missing permissions to list the contents of the media folder. Add the following statement to your policy.
Note: Your policy should be added to the user(s) and not to the bucket itself. A better choice is to create an IAM group, attach the policy to the group and then add each user to the group (which you mentioned that you are doing).
{
"Sid": "AllowListingOfMediaFolder",
"Action": ["s3:ListBucket"],
"Effect": "Allow",
"Resource": ["arn:aws:s3:::mycoolbucket"],
"Condition":{"StringLike":{"s3:prefix":["media/*"]}}
},
With this policy, I'm able to grant access to all subfolders of a folder in Amazon S3
{
"Version": "2012-10-17",
"Statement": [
{
"Sid": "",
"Effect": "Allow",
"Action": "s3:ListBucket",
"Resource": "arn:aws:s3:::<<bucketname>>",
"Condition": {
"StringLike": {
"s3:prefix": "foldername/*"
}
}
},
{
"Sid": "",
"Effect": "Allow",
"Action": [
"s3:GetObject*",
"s3:PutObject*",
"s3:ListBucket",
"s3:DeleteObject*"
],
"Resource": "arn:aws:s3:::<<bucketname>>/foldername/*"
}
]
}