GKE - Automatic replacement of Instance Group Template - virtual-machine

Question

Current Situation: We're having a Kubernetes Cluster in Google (GKE). There is an instance group that refers an instance template defining the VMs.
Problem: Some days ago we had a spontaneous outage. I could find out that our instance group referred a new instance template, that was missing some access rights leading to that outage. Changing the instance template back to the original one solved the issue.
What I'm still wondering: We did neither change the template for the instance group nor did we create the instance template that was used.
I'm the only person managing the cluster, so I'm pretty sure that this happened automatically. How could this happen?
Searching for a reason:
I was researching on that topic but I could not find any related issues. I discovered three logs that might be interesting, but I cannot find a reason for that.
gke-<MY_PROJECT>-default-pool-4f1a9017 is the new instance template created.
All three logs are direct subsequent logs:
Log1 - ERROR - beta.compute.instanceGroupManagers.insert:
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"status": {
"code": 6,
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists"
},
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1",
"requestAttributes": {
"time": "<TIMESTAMP>",
"auth": {}
},
"destinationAttributes": {}
},
"serviceName": "compute.googleapis.com",
"methodName": "beta.compute.instanceGroupManagers.insert",
"authorizationInfo": [
{
"permission": "compute.instanceGroupManagers.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"type": "compute.instanceGroupManagers"
}
},
{
"permission": "compute.instances.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.disks.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.disks.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.subnetworks.use",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.subnetworks.useExternalIp",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.instances.setMetadata",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setTags",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-4f1a9017-0000",
"type": "compute.instances"
}
}
],
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"instanceTemplate": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/global/instanceTemplates/<INSTANCE_TEMPLATE_ID>",
"#type": "type.googleapis.com/compute.instanceGroupManagers.insert",
"targetSize": "0",
"baseInstanceName": "gke-<MY_PROJECT>-default-pool-4f1a9017"
},
"response": {
"#type": "type.googleapis.com/error",
"error": {
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists",
"errors": [
{
"reason": "alreadyExists",
"domain": "global",
"message": "The resource 'projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp' already exists"
}
],
"code": 409
}
},
"resourceLocation": {
"currentLocations": [
"<MY_ZONE>"
]
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"project_id": "<MY_PROJECT>",
"location": "<MY_ZONE>",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "ERROR",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"receiveTimestamp": "<TIMESTAMP>"
}
Log2 - NOTICE - v1.compute.instanceGroupManagers.setInstanceTemplate
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1",
"requestAttributes": {
"time": "<TIMESTAMP>",
"auth": {}
},
"destinationAttributes": {}
},
"serviceName": "compute.googleapis.com",
"methodName": "v1.compute.instanceGroupManagers.setInstanceTemplate",
"authorizationInfo": [
{
"permission": "compute.instanceGroupManagers.update",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"type": "compute.instanceGroupManagers"
}
},
{
"permission": "compute.instances.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.disks.create",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.disks.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/disks/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.disks"
}
},
{
"permission": "compute.subnetworks.use",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.subnetworks.useExternalIp",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/regions/<MY_REGION>/subnetworks/default",
"type": "compute.subnetworks"
}
},
{
"permission": "compute.instances.setMetadata",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setTags",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
},
{
"permission": "compute.instances.setLabels",
"granted": true,
"resourceAttributes": {
"service": "compute",
"name": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instances/gke-<MY_PROJECT>-default-pool-19b04905-0000",
"type": "compute.instances"
}
}
],
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"instanceTemplate": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/global/instanceTemplates/<INSTANCE_TEMPLATE_ID>",
"#type": "type.googleapis.com/compute.instanceGroupManagers.setInstanceTemplate"
},
"response": {
"targetId": "8142394286096021440",
"#type": "type.googleapis.com/operation",
"id": "<OPERATION_ID2>",
"insertTime": "<TIMESTAMP>",
"operationType": "compute.instanceGroupManagers.setInstanceTemplate",
"targetLink": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"zone": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>",
"name": "<OPERATION_ID>",
"progress": "0",
"selfLink": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/operations/<OPERATION_ID>",
"status": "RUNNING",
"user": "service-<ID>#container-engine-robot.iam.gserviceaccount.com",
"selfLinkWithId": "https://www.googleapis.com/compute/v1/projects/<MY_PROJECT>/zones/<MY_ZONE>/operations/<OPERATION_ID2>",
"startTime": "<TIMESTAMP>"
},
"resourceLocation": {
"currentLocations": [
"<MY_ZONE>"
]
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"location": "<MY_ZONE>",
"project_id": "<MY_PROJECT>",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "NOTICE",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"operation": {
"id": "<OPERATION_ID>",
"producer": "compute.googleapis.com",
"first": true
},
"receiveTimestamp": "<TIMESTAMP>"
}
Log3 - NOTICE - v1.compute.instanceGroupManagers.setInstanceTemplate
{
"protoPayload": {
"#type": "type.googleapis.com/google.cloud.audit.AuditLog",
"authenticationInfo": {
"principalEmail": "service-<ID>#container-engine-robot.iam.gserviceaccount.com"
},
"requestMetadata": {
"callerIp": "<IP_V6>",
"callerSuppliedUserAgent": "google-api-go-client/0.5 GoogleContainerEngine/v1"
},
"serviceName": "compute.googleapis.com",
"methodName": "v1.compute.instanceGroupManagers.setInstanceTemplate",
"resourceName": "projects/<MY_PROJECT>/zones/<MY_ZONE>/instanceGroupManagers/gke-<MY_PROJECT>-default-pool-19b04905-grp",
"request": {
"#type": "type.googleapis.com/compute.instanceGroupManagers.setInstanceTemplate"
}
},
"insertId": "<INSERT_ID>",
"resource": {
"type": "gce_instance_group_manager",
"labels": {
"project_id": "<MY_PROJECT>",
"instance_group_manager_name": "gke-<MY_PROJECT>-default-pool-19b04905-grp",
"instance_group_manager_id": "<INSTANCE_GROUP_MANAGER_ID>",
"location": "<MY_ZONE>"
}
},
"timestamp": "<TIMESTAMP>",
"severity": "NOTICE",
"logName": "projects/<MY_PROJECT>/logs/cloudaudit.googleapis.com%2Factivity",
"operation": {
"id": "<OPERATION_ID>",
"producer": "compute.googleapis.com",
"last": true
},
"receiveTimestamp": "<TIMESTAMP>"
}