import boto3 from cloud_governance.policy.policy_operations.aws.zombie_cluster.zombie_cluster_common_methods import ZombieClusterCommonMethods from cloud_governance.common.clouds.aws.ec2.ec2_operations import EC2Operations from cloud_governance.common.logger.init_logger import logger from cloud_governance.common.clouds.aws.utils.utils import Utils # @todo add next token # response = client.get_servers() # results = response["serverList"] # while "NextToken" in response: # response = client.get_servers(NextToken=response["NextToken"]) # results.extend(response["serverList"]) from cloud_governance.policy.policy_operations.aws.zombie_cluster.delete_ec2_resources import DeleteEC2Resources from cloud_governance.policy.policy_operations.aws.zombie_cluster.delete_iam_resources import DeleteIAMResources from cloud_governance.policy.policy_operations.aws.zombie_cluster.delete_s3_resources import DeleteS3Resources class ZombieClusterResources(ZombieClusterCommonMethods): """ This class filter and deletes zombie cluster resources, which are not deleted while cleanup the clusters, alert user after 4 days of cluster deleted. and delete the resources after 7 days of cluster deleted. """ def __init__(self, cluster_prefix: str = None, delete: bool = False, region: str = 'us-east-2', cluster_tag: str = '', resource_name: str = '', force_delete: bool = False): super().__init__(region=region, force_delete=force_delete) self.cluster_prefix = cluster_prefix self.delete = delete self.cluster_tag = cluster_tag self.resource_name = resource_name self.delete_ec2_resource = DeleteEC2Resources(self.ec2_client, self.elb_client, self.elbv2_client, region=region) self.delete_iam_resource = DeleteIAMResources(iam_client=self.iam_client) self.delete_s3_resource = DeleteS3Resources(s3_client=self.s3_client, s3_resource=self.s3_resource) self.ec2_operations = EC2Operations(region=region) self.__get_details_resource_list = Utils().get_details_resource_list def all_cluster_instance(self): """ This method returns list of cluster's instance tag name that contains openshift tag prefix from all regions :return: list of cluster's instance tag name """ instances_list = [] result_instance = {} regions_data = self.ec2_client.describe_regions() for region in regions_data['Regions']: __ec2 = boto3.client('ec2', region_name=region['RegionName']) ec2s_data = self.__get_details_resource_list(func_name=__ec2.describe_instances, input_tag='Reservations', check_tag='NextToken') for items in ec2s_data: if items.get('Instances'): instances_list.append(items['Instances']) for instance in instances_list: for item in instance: if item.get('InstanceId'): instance_id = item['InstanceId'] if item.get('Tags'): for tag in item['Tags']: if tag['Key'].startswith(self.cluster_prefix): result_instance[instance_id] = tag['Key'] return result_instance def _cluster_instance(self): """ This method returns list of cluster's instance tag name that contains openshift tag prefix :return: list of cluster's instance tag name """ instances_list = [] result_instance = {} ec2s_data = self.ec2_operations.get_instances() for items in ec2s_data: if items.get('Instances'): instances_list.append(items['Instances']) for instance in instances_list: for item in instance: if item.get('InstanceId'): instance_id = item['InstanceId'] if item.get('Tags'): for tag in item['Tags']: if tag['Key'].startswith(self.cluster_prefix): result_instance[instance_id] = tag['Key'] return result_instance def __get_cluster_resources(self, resources_list: list, input_resource_id: str, tags: str = 'Tags'): """ This method returns all cluster resources keys that start with cluster prefix :param resources_list: :param tags: :return: dictionary of the resources key and id """ result_resources_key_id = {} for resource in resources_list: if resource.get(input_resource_id): resource_id = resource[input_resource_id] # skip when input_resource_id no found else: continue if resource.get(tags): for tag in resource[tags]: if tag['Key'].startswith(self.cluster_prefix): # when input a specific cluster, return resource id of the input cluster for inner_tag in resource[tags]: if self.cluster_tag: if self.cluster_tag == inner_tag['Key']: result_resources_key_id[resource_id] = tag['Key'] elif self.resource_name: if self.resource_name == inner_tag['Value']: result_resources_key_id[resource_id] = tag['Key'] else: result_resources_key_id[resource_id] = tag['Key'] break return result_resources_key_id def __extract_vpc_id_from_resource_data(self, zombie_id: str, resource_data: list, input_tag: str, output_tag: str = ''): """ This method extract vpc_id from the resource data :param zombie_id: :param resource_data: :param input_tag: :param output_tag: :return: """ vpc_id = '' for resource in resource_data: if resource.get(input_tag) == zombie_id: if output_tag: if resource.get(output_tag): vpc_id = resource.get(output_tag)[0].get('VpcId') elif resource[input_tag] == zombie_id: vpc_id = resource.get('VpcId') return vpc_id def __get_cluster_resources_by_vpc_id(self, vpc_id: str, resource_data: list, output_tag: str, input_tag: str = ''): """ this method list all the resources by vpc_id :param vpc_id: :param resource_data: :param output_tag: :param input_tag: :return: """ ids = [] for resource in resource_data: if input_tag: for attachment in resource.get(input_tag): if attachment.get('VpcId') == vpc_id: ids.append(resource.get(output_tag)) elif resource.get('VpcId') == vpc_id: ids.append(resource.get(output_tag)) return ids def __get_zombie_resources(self, exist_resources: dict): """ This method filter zombie resource, meaning no active instance for this cluster """ zombie_resources = {} zombies_values = set(exist_resources.values()) - set(self._cluster_instance().values()) for zombie_value in zombies_values: for key, value in exist_resources.items(): if zombie_value == value: zombie_resources[key] = value return zombie_resources def __get_all_zombie_resources(self, exist_resources: dict): """ This method filter zombie resource, meaning no active instance for this cluster in all regions """ zombie_resources = {} zombies_values = set(exist_resources.values()) - set(self.all_cluster_instance().values()) for zombie_value in zombies_values: for key, value in exist_resources.items(): if zombie_value == value: zombie_resources[key] = value return zombie_resources def zombie_cluster_volume(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's volume according to cluster tag name and cluster name data delete only available resource that related to cluster """ available_volumes = [] volumes_data = self.ec2_operations.get_volumes() # filter in-use resource for volume in volumes_data: if volume['State'] == 'available': available_volumes.append(volume) exist_volume = self.__get_cluster_resources(resources_list=available_volumes, input_resource_id='VolumeId') zombies = self.__get_zombie_resources(exist_volume) resources = self._get_tags_of_zombie_resources(resources=volumes_data, resource_id_name='VolumeId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource_id=zombie, resource='ec2_volume') else: if self._force_delete and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource_id=zombie, resource='ec2_volume') return zombies, cluster_left_out_days def zombie_cluster_ami(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's ami according to cluster tag name and cluster name data """ images_data = self.ec2_operations.get_images() exist_ami = self.__get_cluster_resources(resources_list=images_data, input_resource_id='ImageId') zombies = self.__get_zombie_resources(exist_ami) resources = self._get_tags_of_zombie_resources(resources=images_data, resource_id_name='ImageId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) try: if delete_cluster_resource and self.delete: self.ec2_client.deregister_image(ImageId=zombie) logger.info(f'deregister_image: {zombie}') else: if self._force_delete and self.delete: self.ec2_client.deregister_image(ImageId=zombie) logger.info(f'deregister_image: {zombie}') except Exception as err: logger.exception(f'Cannot deregister_image: {zombie}, {err}') return zombies, cluster_left_out_days def zombie_cluster_snapshot(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's snapshot according to cluster tag name and cluster name data """ snapshots_data = self.ec2_operations.get_snapshots() exist_snapshot = self.__get_cluster_resources(resources_list=snapshots_data, input_resource_id='SnapshotId') zombies = self.__get_zombie_resources(exist_snapshot) resources = self._get_tags_of_zombie_resources(resources=snapshots_data, resource_id_name='SnapshotId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='ebs_snapshots', resource_id=zombie) else: if self._force_delete and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='ebs_snapshots', resource_id=zombie) return zombies, cluster_left_out_days def __get_tag_from_resource_tags(self, tags: list, cluster_tag: str): """ This method retunrs the cluster tag @param tags: @param cluster_tag: @return: """ if cluster_tag: for tag in tags: if cluster_tag == tag.get('Key'): return tag.get('Key') return '' return '' def __get_vpc_tags(self, vpc_id: str, cluster_tag: str): """ This method gets tags of vpc @param vpc_id: @return: """ if vpc_id: vpcs = self.ec2_client.describe_vpcs(VpcIds=[vpc_id])['Vpcs'] if vpcs: vpc = vpcs[0] if vpc.get('Tags'): for tag in vpc.get('Tags'): if cluster_tag in tag.get('Key'): return tag.get('Key') return '' def __get_zombies_by_vpc_id(self, vpc_id: str, resources: list, output_tag: str, cluster_tag: str = '', tags: str = 'Tags', input_tag: str = ''): """ This method extracts zombies from vpc_id @param vpc_id: @param resources: @param output_tag: @param cluster_tag: @param tags: @return: """ ids = {} for resource in resources: if input_tag: for attachment in resource.get(input_tag): if attachment.get('VpcId') == vpc_id: if resource.get(tags): tag = self.__get_tag_from_resource_tags(resource.get(tags), cluster_tag) if tag: ids[resource.get(output_tag)] = tag else: tag = self.__get_vpc_tags(vpc_id=vpc_id, cluster_tag=cluster_tag) if tag: ids[resource.get(output_tag)] = tag elif resource.get('VpcId') == vpc_id: if resource.get(tags): tag = self.__get_tag_from_resource_tags(resource.get(tags), cluster_tag) if tag: ids[resource.get(output_tag)] = tag else: tag = self.__get_vpc_tags(vpc_id=vpc_id, cluster_tag=cluster_tag) if tag: ids[resource.get(output_tag)] = tag return ids def zombie_cluster_security_group(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of zombie cluster's security groups compare to existing instances and cluster name data :return: list of zombie cluster's security groups """ security_groups = self.ec2_operations.get_security_groups() exist_security_group = self.__get_cluster_resources(resources_list=security_groups, input_resource_id='GroupId') zombies = self.__get_zombie_resources(exist_security_group) if vpc_id and not zombies: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=security_groups, output_tag='GroupId', cluster_tag=cluster_tag_vpc) resources = self._get_tags_of_zombie_resources(resources=security_groups, resource_id_name='GroupId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): security_groups = self.ec2_operations.get_security_groups() vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=security_groups, input_tag='GroupId') zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=security_groups, output_tag='GroupId', cluster_tag=cluster_tag) cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource('security_group', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource('security_group', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_elastic_ip(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of zombie cluster's elastic ip according to existing instances and cluster name data """ exist_elastic_ip_association = [] exist_elastic_ip_allocation = [] elastic_ips_data = self.ec2_operations.get_elastic_ips() for elastic_ip in elastic_ips_data: if elastic_ip.get('AssociationId'): exist_elastic_ip_association.append(elastic_ip) if elastic_ip.get('AllocationId'): exist_elastic_ip_allocation.append(elastic_ip) exist_elastic_ip_ass = self.__get_cluster_resources(resources_list=exist_elastic_ip_association, input_resource_id='AssociationId') exist_elastic_ip_all = self.__get_cluster_resources(resources_list=exist_elastic_ip_allocation, input_resource_id='AllocationId') zombies_ass = self.__get_zombie_resources(exist_elastic_ip_ass) zombies_all = self.__get_zombie_resources(exist_elastic_ip_all) resources_ass = self._get_tags_of_zombie_resources(resources=elastic_ips_data, resource_id_name='AllocationId', zombies=zombies_ass, aws_service='ec2') resources_all = self._get_tags_of_zombie_resources(resources=elastic_ips_data, resource_id_name='AllocationId', zombies=zombies_all, aws_service='ec2') cluster_left_out_days = {} if zombies_ass: for zombie, cluster_tag in zombies_ass.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources_ass, cluster_left_out_days=cluster_left_out_days, zombie=zombie,cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, deletion_type='disassociate') else: if self._force_delete: self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, deletion_type='disassociate') if zombies_all: for zombie, cluster_tag in zombies_all.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources_all, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='elastic_ip', resource_id=zombie, cluster_tag=cluster_tag) zombies = {**zombies_all} return zombies, cluster_left_out_days def zombie_cluster_network_interface(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of zombie cluster's network interface according to existing instances and cluster name data """ network_interfaces_data = self.ec2_operations.get_network_interface() exist_network_interface = self.__get_cluster_resources(resources_list=network_interfaces_data, input_resource_id='NetworkInterfaceId', tags='TagSet') zombies = self.__get_zombie_resources(exist_network_interface) if not zombies and vpc_id: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_interfaces_data, output_tag='NetworkInterfaceId', tags='TagSet', cluster_tag=cluster_tag_vpc) resources = self._get_tags_of_zombie_resources(resources=network_interfaces_data, resource_id_name='NetworkInterfaceId', zombies=zombies, aws_service='ec2', aws_tag='TagSet') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): network_interfaces_data = self.ec2_operations.get_network_interface() vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=network_interfaces_data, input_tag='NetworkInterfaceId') if vpc_id: zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_interfaces_data, output_tag='NetworkInterfaceId', cluster_tag=cluster_tag, tags='TagSet') else: zombie_ids = [zombie] cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days( resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='network_interface', resource_id=zombie_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='network_interface', resource_id=zombie_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_load_balancer(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's load balancer according to cluster vpc and cluster name data """ exist_load_balancer = {} load_balancers_data = self.ec2_operations.get_load_balancers() for resource in load_balancers_data: resource_id = resource['LoadBalancerName'] tags = self.elb_client.describe_tags(LoadBalancerNames=[resource_id]) for item in tags['TagDescriptions']: if item.get('Tags'): for tag in item['Tags']: if tag['Key'].startswith(self.cluster_prefix): # when input a specific cluster, return resource id of the input cluster if self.cluster_tag: if self.cluster_tag == tag['Key']: exist_load_balancer[resource_id] = tag['Key'] else: exist_load_balancer[resource_id] = tag['Key'] break zombies = self.__get_zombie_resources(exist_load_balancer) resources = self._get_tags_of_zombie_resources(resources=load_balancers_data, resource_id_name='LoadBalancerName', zombies=zombies, aws_service='elbv1') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer', resource_id=zombie, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer', resource_id=zombie, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_load_balancer_v2(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's load balancer according to cluster vpc and cluster name data """ exist_load_balancer = {} load_balancers_data = self.ec2_operations.get_load_balancers_v2() for resource in load_balancers_data: resource_id = resource['LoadBalancerArn'] tags = self.elbv2_client.describe_tags(ResourceArns=[resource_id]) for item in tags['TagDescriptions']: if item.get('Tags'): for tag in item['Tags']: if tag['Key'].startswith(self.cluster_prefix): # when input a specific cluster, return resource id of the input cluster if self.cluster_tag: if self.cluster_tag == tag['Key']: exist_load_balancer[resource_id] = tag['Key'] else: exist_load_balancer[resource_id] = tag['Key'] break zombies = self.__get_zombie_resources(exist_load_balancer) resources = self._get_tags_of_zombie_resources(resources=load_balancers_data, resource_id_name='LoadBalancerArn', zombies=zombies, aws_service='elbv2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer_v2', resource_id=zombie, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='load_balancer_v2', resource_id=zombie, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def __get_all_exist_vpcs(self): """ This method returns all exist vpc ids (for supporting Network ACL - missing OpenShift tags) :return: """ vpcs_data = self.ec2_operations.get_vpcs() vpcs_list = [] for resource in vpcs_data: vpcs_list.append(resource['VpcId']) return vpcs_list def zombie_cluster_vpc(self): """ This method returns list of cluster's vpc according to cluster tag name and cluster name data """ vpcs_data = self.ec2_operations.get_vpcs() exist_vpc = self.__get_cluster_resources(resources_list=vpcs_data, input_resource_id='VpcId') zombies = self.__get_zombie_resources(exist_vpc) delete_dict = {"ELB": self.zombie_cluster_load_balancer, "ELBV2": self.zombie_cluster_load_balancer_v2, "VPCE": self.zombie_cluster_vpc_endpoint, "DHCP": self.zombie_cluster_dhcp_option, "RT": self.zombie_cluster_route_table, "SG": self.zombie_cluster_security_group, "NATG": self.zombie_cluster_nat_gateway, "NACL": self.zombie_cluster_network_acl, "ENI": self.zombie_cluster_network_interface, "IGW": self.zombie_cluster_internet_gateway, "SUB": self.zombie_cluster_subnet, "EIP": self.zombie_cluster_elastic_ip} resources = self._get_tags_of_zombie_resources(resources=vpcs_data, resource_id_name='VpcId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='vpc', resource_id=zombie, pending_resources=delete_dict, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: self.delete_ec2_resource.delete_zombie_resource(resource='vpc', resource_id=zombie, pending_resources=delete_dict, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_subnet(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's subnet according to cluster tag name and cluster name data """ subnets_data = self.ec2_operations.get_subnets() exist_subnet = self.__get_cluster_resources(resources_list=subnets_data, input_resource_id='SubnetId') zombies = self.__get_zombie_resources(exist_subnet) if not zombies and vpc_id: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=subnets_data, output_tag='SubnetId', cluster_tag=cluster_tag_vpc) resources = self._get_tags_of_zombie_resources(resources=subnets_data, resource_id_name='SubnetId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): subnets_data = self.ec2_operations.get_subnets() vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=subnets_data, input_tag='SubnetId') zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=subnets_data, output_tag='SubnetId', cluster_tag=cluster_tag) cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='subnet', resource_id=zombie_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='subnet', resource_id=zombie_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_route_table(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's route table according to cluster tag name and cluster name data """ route_tables_data = self.ec2_operations.get_route_tables() exist_route_table = self.__get_cluster_resources(resources_list=route_tables_data, input_resource_id='RouteTableId') zombies = self.__get_zombie_resources(exist_route_table) if not zombies and vpc_id: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=route_tables_data, output_tag='RouteTableId', cluster_tag=cluster_tag_vpc) resources = self._get_tags_of_zombie_resources(resources=route_tables_data, resource_id_name='RouteTableId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): route_tables_data = self.ec2_operations.get_route_tables() vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=route_tables_data, input_tag='RouteTableId') zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=route_tables_data, output_tag='RouteTableId', cluster_tag=cluster_tag) cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='route_table', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='route_table', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_internet_gateway(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's route table internet gateway according to cluster tag name and cluster name data """ internet_gateways_data = self.ec2_operations.get_internet_gateways() exist_internet_gateway = self.__get_cluster_resources(resources_list=internet_gateways_data, input_resource_id='InternetGatewayId') zombies = self.__get_zombie_resources(exist_internet_gateway) if not zombies and vpc_id: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=internet_gateways_data, output_tag='InternetGatewayId', input_tag='Attachments', cluster_tag=cluster_tag_vpc) resources = self._get_tags_of_zombie_resources(resources=internet_gateways_data, resource_id_name='InternetGatewayId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): internet_gateways_data = self.ec2_operations.get_internet_gateways() vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=internet_gateways_data, input_tag='InternetGatewayId', output_tag='Attachments') if vpc_id: zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=internet_gateways_data, output_tag='InternetGatewayId', cluster_tag=cluster_tag, input_tag='Attachments') else: zombie_ids = [zombie] cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='internet_gateway', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='internet_gateway', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_dhcp_option(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's dhcp option according to cluster tag name and cluster name data """ dhcp_options_data = self.ec2_operations.get_dhcp_options() exist_dhcp_option = self.__get_cluster_resources(resources_list=dhcp_options_data, input_resource_id='DhcpOptionsId') zombies = self.__get_zombie_resources(exist_dhcp_option) resources = self._get_tags_of_zombie_resources(resources=dhcp_options_data, resource_id_name='DhcpOptionsId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} vpcs = self.ec2_client.describe_vpcs()['Vpcs'] if zombies: for zombie, cluster_tag in zombies.items(): vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=vpcs, input_tag='DhcpOptionsId') cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: if vpc_id: self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie, vpc_id=vpc_id) else: self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie) else: if self._force_delete and self.delete: if vpc_id: self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie, vpc_id=vpc_id) else: self.delete_ec2_resource.delete_zombie_resource(resource='dhcp_options', resource_id=zombie) return zombies, cluster_left_out_days def zombie_cluster_vpc_endpoint(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of cluster's vpc endpoint according to cluster tag name and cluster name data """ vpc_endpoints_data = self.ec2_operations.get_vpce() exist_vpc_endpoint = self.__get_cluster_resources(resources_list=vpc_endpoints_data, input_resource_id='VpcEndpointId') zombies = self.__get_zombie_resources(exist_vpc_endpoint) if not zombies and vpc_id: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=vpc_endpoints_data, output_tag='VpcEndpointId', cluster_tag=cluster_tag_vpc) resources = self._get_tags_of_zombie_resources(resources=vpc_endpoints_data, resource_id_name='VpcEndpointId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): vpc_endpoints_data = self.ec2_operations.get_vpce() vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=vpc_endpoints_data, input_tag='VpcEndpointId') if vpc_id: zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=vpc_endpoints_data, output_tag='VpcEndpointId', cluster_tag=cluster_tag) else: zombie_ids = [zombies] cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='vpc_endpoints', resource_id=zombie_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='vpc_endpoints', resource_id=zombie_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_nat_gateway(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of zombie cluster's nat gateway according to cluster tag name and cluster name data """ nat_gateways_data = self.ec2_operations.get_nat_gateways() exist_nat_gateway = self.__get_cluster_resources(resources_list=nat_gateways_data, input_resource_id='NatGatewayId') zombies = self.__get_zombie_resources(exist_nat_gateway) if not zombies and vpc_id: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=nat_gateways_data, output_tag='NatGatewayId', cluster_tag=cluster_tag_vpc) resources = self._get_tags_of_zombie_resources(resources=nat_gateways_data, resource_id_name='NatGatewayId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): nat_gateways_data = self.ec2_operations.get_nat_gateways() vpc_id = self.__extract_vpc_id_from_resource_data(zombie, nat_gateways_data, input_tag='NatGatewayId') zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=nat_gateways_data, output_tag='NatGatewayId', cluster_tag=cluster_tag) cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='nat_gateways', resource_id=zombie_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='nat_gateways', resource_id=zombie_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_network_acl(self, vpc_id: str = '', cluster_tag_vpc: str = ''): """ This method returns list of zombie cluster's network acl according to existing vpc id and cluster name data """ exist_network_acl = {} network_acls_data = self.ec2_operations.get_nacls() for resource in network_acls_data: exist_network_acl[resource['NetworkAclId']] = resource['VpcId'] zombie_resources = {} all_exist_vpcs = self.__get_all_exist_vpcs() zombies_values = set(exist_network_acl.values()) - set(all_exist_vpcs) for zombie_value in zombies_values: for key, value in exist_network_acl.items(): if zombie_value == value: zombie_resources[key] = value zombies = zombie_resources if not zombies and vpc_id: zombies = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_acls_data, output_tag='NetworkAclId', cluster_tag=cluster_tag_vpc) resources = self._get_tags_of_zombie_resources(resources=network_acls_data, resource_id_name='NetworkAclId', zombies=zombies, aws_service='ec2') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): network_acls_data = self.ec2_operations.get_nacls() if not vpc_id: vpc_id = self.__extract_vpc_id_from_resource_data(zombie_id=zombie, resource_data=network_acls_data, input_tag='NetworkAclId') zombie_ids = self.__get_zombies_by_vpc_id(vpc_id=vpc_id, resources=network_acls_data, output_tag='NetworkAclId', cluster_tag=cluster_tag) cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='network_acl', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) else: if self._force_delete and self.delete: for zombie_id in zombie_ids: self.delete_ec2_resource.delete_zombie_resource(resource='network_acl', resource_id=zombie_id, vpc_id=vpc_id, cluster_tag=cluster_tag) return zombies, cluster_left_out_days def zombie_cluster_role(self): """ This method returns list of cluster's role in all regions according to cluster name and cluster name data * Role is a global resource, need to scan for live cluster in all regions """ exist_role_name_tag = {} roles_data = self.__get_details_resource_list(func_name=self.iam_client.list_roles, input_tag='Roles', check_tag='Marker') for role in roles_data: role_name = role['RoleName'] if 'worker-role' in role_name or 'master-role' in role_name: role_data = self.iam_client.get_role(RoleName=role_name) data = role_data['Role'] if data.get('Tags'): for tag in data['Tags']: if tag['Key'].startswith(self.cluster_prefix): # when input a specific cluster, return resource id of the input cluster if self.cluster_tag: if self.cluster_tag == tag['Key']: exist_role_name_tag[role_name] = tag['Key'] else: exist_role_name_tag[role_name] = tag['Key'] break zombies = [] cluster_left_out_days = {} if exist_role_name_tag: zombies = self.__get_all_zombie_resources(exist_role_name_tag) resources = self._get_tags_of_zombie_resources(resources=roles_data, resource_id_name='RoleName', zombies=zombies, aws_service='role') if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_role') else: if self._force_delete and self.delete: self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_role') return zombies, cluster_left_out_days def zombie_cluster_user(self): """ This method returns list of cluster's user according to cluster name and cluster name data * User is a global resource, need to scan for live cluster in all regions """ exist_user_name_tag = {} users_data = self.__get_details_resource_list(func_name=self.iam_client.list_users, input_tag='Users', check_tag='Marker') for user in users_data: user_name = user['UserName'] user_data = self.iam_client.get_user(UserName=user_name) data = user_data['User'] if data.get('Tags'): for tag in data['Tags']: if tag['Key'].startswith(self.cluster_prefix): # when input a specific cluster, return resource id of the input cluster if self.cluster_tag: if self.cluster_tag == tag['Key']: exist_user_name_tag[user_name] = tag['Key'] else: exist_user_name_tag[user_name] = tag['Key'] break zombies = self.__get_all_zombie_resources(exist_user_name_tag) resources = self._get_tags_of_zombie_resources(resources=users_data, resource_id_name='UserName', zombies=zombies, aws_service='user') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_user') else: if self._force_delete and self.delete: self.delete_iam_resource.delete_iam_zombie_resource(resource_id=zombie, resource_type='iam_user') return zombies, cluster_left_out_days def zombie_cluster_s3_bucket(self, cluster_stamp: str = 'image-registry'): """ This method returns list of cluster's s3 bucket according to cluster name and cluster name data * S3 is a global resource, need to scan for live cluster in all regions """ exist_bucket_name_tag = {} response = self.s3_client.list_buckets() # Fetch all bucket with stamp for bucket in response['Buckets']: # fast filter if cluster_stamp in bucket['Name']: try: tags = self.s3_client.get_bucket_tagging(Bucket=bucket['Name']) except Exception as e: # continue when no bucket tags continue for tag in tags['TagSet']: if tag['Key'].startswith(self.cluster_prefix): # when input a specific cluster, return resource id of the input cluster if self.cluster_tag: if self.cluster_tag == tag['Key']: exist_bucket_name_tag[bucket['Name']] = tag['Key'] else: exist_bucket_name_tag[bucket['Name']] = tag['Key'] break zombies = self.__get_all_zombie_resources(exist_bucket_name_tag) resources = self._get_tags_of_zombie_resources(resources=response['Buckets'], resource_id_name='Name', zombies=zombies, aws_service='bucket', aws_tag='TagSet') cluster_left_out_days = {} if zombies: for zombie, cluster_tag in zombies.items(): cluster_left_out_days, delete_cluster_resource = self._check_zombie_cluster_deleted_days(resources=resources, cluster_left_out_days=cluster_left_out_days, zombie=zombie, cluster_tag=cluster_tag) if delete_cluster_resource and self.delete: self.delete_s3_resource.delete_zombie_s3_resource(resource_type='s3_bucket', resource_id=zombie) else: if self._force_delete and self.delete: self.delete_s3_resource.delete_zombie_s3_resource(resource_type='s3_bucket', resource_id=zombie) return zombies, cluster_left_out_days # zombie_cluster_resources = ZombieClusterResources(cluster_prefix='kubernetes.io/cluster/', delete=False, region='us-east-2') # print(zombie_cluster_resources.zombie_cluster_subnet())