Skip to content

Added support to read multiple accounts #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ntkathole wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added support to read multiple accounts #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ settings.yaml
.idea
.cache/
.eggs
.env/
*.egg-info
**/cleanup*.log
*.pyc
Expand Down
37 changes: 6 additions & 31 deletions cloudwash/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,44 +1,19 @@
import json
from contextlib import contextmanager

import wrapanapi

from cloudwash.config import settings


@contextmanager
def compute_client(compute_resource, **kwargs):
def compute_client(compute_resource, client=None):
"""The context manager for compute resource client to initiate and disconnect
:param str compute_resource: The compute resource name
:param obj client: Client object
"""
if compute_resource == "azure":
client = wrapanapi.AzureSystem(
username=settings.azure.auth.client_id,
password=settings.azure.auth.secret_id,
tenant_id=settings.azure.auth.tenant_id,
subscription_id=settings.azure.auth.subscription_id,
provisioning={
"resource_group": kwargs['resource_group'],
"template_container": None,
"region_api": kwargs['azure_region'],
},
)
elif compute_resource == "gce":
client = wrapanapi.GoogleCloudSystem(
project=settings.gce.auth.project_id,
service_account=json.loads(settings.gce.auth.service_account),
)
elif compute_resource == "aws":
client = wrapanapi.EC2System(
username=settings.aws.auth.access_key,
password=settings.aws.auth.secret_key,
region=kwargs['aws_region'],
)
supported_providers = ["azure", "gce", "aws"]
if compute_resource in supported_providers:
client = client
else:
raise ValueError(
f"{compute_resource} is an incorrect value. It should be one of azure or gce or ec2"
f"{compute_resource} is an incorrect value. It should be one of {supported_providers}"
)

try:
yield client
finally:
Expand Down
26 changes: 17 additions & 9 deletions cloudwash/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import logging
from pathlib import Path
from pathlib import PurePath

Expand All @@ -21,12 +22,19 @@

def validate_provider(provider_name):
provider = provider_name.upper()
provider_settings = [
f"{provider}.{setting_key}" for setting_key in settings.to_dict().get(provider)
]
settings.validators.register(Validator(*provider_settings, ne=None))
try:
settings.validators.validate()
logger.info(f"The {provider} providers settings are initialized and validated !")
except Exception:
raise
provider_settings = settings.to_dict().get(provider)
if not provider_settings:
logging.error("No provider settings found! Please check settings file!")
exit()
for account in provider_settings:
account_settings = [f"{account}.{account_key}" for account_key in provider_settings]
settings.validators.register(Validator(*account_settings, ne=None))
try:
settings.validators.validate()
logger.info(
f"The {provider} provider settings for account {account['NAME']}"
f" are initialized and validated!"
)
except Exception as e:
logging.error(f"Settings validation failed for {provider} account {account['NAME']}")
raise e
211 changes: 113 additions & 98 deletions cloudwash/providers/aws.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,120 +1,135 @@
"""ec2 CR Cleanup Utilities"""
import wrapanapi

from cloudwash.client import compute_client
from cloudwash.config import settings
from cloudwash.logger import logger
from cloudwash.utils import dry_data
from cloudwash.utils import echo_dry
from cloudwash.utils import pprint
from cloudwash.utils import total_running_time


def cleanup(**kwargs):

is_dry_run = kwargs["dry_run"]
data = ['VMS', 'NICS', 'DISCS', 'PIPS', 'RESOURCES', 'STACKS']
regions = settings.aws.auth.regions
if "all" in regions:
with compute_client("aws", aws_region="us-west-2") as client:
regions = client.list_regions()
for region in regions:
dry_data['VMS']['stop'] = []
dry_data['VMS']['skip'] = []
for items in data:
dry_data[items]['delete'] = []
with compute_client("aws", aws_region=region) as aws_client:
# Dry Data Collection Defs
def dry_vms():
all_vms = aws_client.list_vms()
for vm in all_vms:
if vm.name in settings.aws.exceptions.vm.vm_list:
dry_data["VMS"]["skip"].append(vm.name)
continue
elif total_running_time(vm).minutes >= settings.aws.criteria.vm.sla_minutes:
if vm.name in settings.aws.exceptions.vm.stop_list:
dry_data["VMS"]["stop"].append(vm.name)
accounts = settings.aws
for account in accounts:
pprint("green", f"\nCloudwash execution for account: {account['NAME']}")
regions = account.auth.regions
if "all" in regions:
client = wrapanapi.EC2System(
username=account.auth.access_key,
password=account.auth.secret_key,
region="us-west-2",
)
with compute_client("aws", client=client) as client:
regions = client.list_regions()
for region in regions:
client = wrapanapi.EC2System(
username=account.auth.access_key,
password=account.auth.secret_key,
region=region,
)
dry_data['VMS']['stop'] = []
dry_data['VMS']['skip'] = []
for items in data:
dry_data[items]['delete'] = []
with compute_client("aws", client=client) as aws_client:
# Dry Data Collection Defs
def dry_vms():
all_vms = aws_client.list_vms()
for vm in all_vms:
if vm.name in account.exceptions.vm.vm_list:
dry_data["VMS"]["skip"].append(vm.name)
continue
elif vm.name.startswith(settings.aws.criteria.vm.delete_vm):
dry_data["VMS"]["delete"].append(vm.name)
return dry_data["VMS"]
elif total_running_time(vm).minutes >= account.criteria.vm.sla_minutes:
if vm.name in account.exceptions.vm.stop_list:
dry_data["VMS"]["stop"].append(vm.name)
continue
elif vm.name.startswith(account.criteria.vm.delete_vm):
dry_data["VMS"]["delete"].append(vm.name)
return dry_data["VMS"]

def dry_nics():
rnics = []
if settings.aws.criteria.nic.unassigned:
rnics = aws_client.get_all_unused_network_interfaces()
[
dry_data["NICS"]["delete"].append(dnic["NetworkInterfaceId"])
for dnic in rnics
]
return rnics
def dry_nics():
rnics = []
if account.criteria.nic.unassigned:
rnics = aws_client.get_all_unused_network_interfaces()
[
dry_data["NICS"]["delete"].append(dnic["NetworkInterfaceId"])
for dnic in rnics
]
return rnics

def dry_discs():
rdiscs = []
if settings.aws.criteria.disc.unassigned:
rdiscs = aws_client.get_all_unattached_volumes()
[dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs]
return rdiscs
def dry_discs():
rdiscs = []
if account.criteria.disc.unassigned:
rdiscs = aws_client.get_all_unattached_volumes()
[dry_data["DISCS"]["delete"].append(ddisc["VolumeId"]) for ddisc in rdiscs]
return rdiscs

def dry_pips():
rpips = []
if settings.aws.criteria.public_ip.unassigned:
rpips = aws_client.get_all_disassociated_addresses()
[dry_data["PIPS"]["delete"].append(dpip["AllocationId"]) for dpip in rpips]
return rpips
def dry_pips():
rpips = []
if account.criteria.public_ip.unassigned:
rpips = aws_client.get_all_disassociated_addresses()
[dry_data["PIPS"]["delete"].append(dpip["AllocationId"]) for dpip in rpips]
return rpips

def dry_stacks():
rstacks = []
[
rstacks.append(stack.name)
for stack in aws_client.list_stacks()
if (
total_running_time(stack).minutes
>= settings.aws.criteria.stacks.sla_minutes
and stack.name.startswith(settings.aws.criteria.stacks.delete_stack)
)
and stack.name not in settings.aws.exceptions.stacks.stack_list
]
[dry_data['STACKS']['delete'].append(stack) for stack in rstacks]
def dry_stacks():
rstacks = []
[
rstacks.append(stack.name)
for stack in aws_client.list_stacks()
if (
total_running_time(stack).minutes >= account.criteria.stacks.sla_minutes
and stack.name.startswith(account.criteria.stacks.delete_stack)
)
and stack.name not in account.exceptions.stacks.stack_list
]
[dry_data['STACKS']['delete'].append(stack) for stack in rstacks]

return rstacks
return rstacks

# Remove / Stop VMs
def remove_vms(avms):
# Remove VMs
[aws_client.get_vm(vm_name).delete() for vm_name in avms["delete"]]
# Stop VMs
[aws_client.get_vm(vm_name).stop() for vm_name in avms["stop"]]
# Remove / Stop VMs
def remove_vms(avms):
# Remove VMs
[aws_client.get_vm(vm_name).delete() for vm_name in avms["delete"]]
# Stop VMs
[aws_client.get_vm(vm_name).stop() for vm_name in avms["stop"]]

# Delete CloudFormations
def remove_stacks(stacks):
[aws_client.get_stack(stack_name).delete() for stack_name in stacks]
# Delete CloudFormations
def remove_stacks(stacks):
[aws_client.get_stack(stack_name).delete() for stack_name in stacks]

# Actual Cleaning and dry execution
logger.info(f"\nResources from the region: {region}")
if kwargs["vms"] or kwargs["_all"]:
avms = dry_vms()
if not is_dry_run:
remove_vms(avms=avms)
logger.info(f"Stopped VMs: \n{avms['stop']}")
logger.info(f"Removed VMs: \n{avms['delete']}")
logger.info(f"Skipped VMs: \n{avms['skip']}")
if kwargs["nics"] or kwargs["_all"]:
rnics = dry_nics()
if not is_dry_run and rnics:
aws_client.remove_all_unused_nics()
logger.info(f"Removed NICs: \n{rnics}")
if kwargs["discs"] or kwargs["_all"]:
rdiscs = dry_discs()
if not is_dry_run and rdiscs:
aws_client.remove_all_unused_volumes()
logger.info(f"Removed Discs: \n{rdiscs}")
if kwargs["pips"] or kwargs["_all"]:
rpips = dry_pips()
if not is_dry_run and rpips:
aws_client.remove_all_unused_ips()
logger.info(f"Removed PIPs: \n{rpips}")
if kwargs["stacks"] or kwargs["_all"]:
rstacks = dry_stacks()
if not is_dry_run:
remove_stacks(stacks=rstacks)
logger.info(f"Removed Stacks: \n{rstacks}")
if is_dry_run:
echo_dry(dry_data)
# Actual Cleaning and dry execution
logger.info(f"\nResources from the region: {region}")
if kwargs["vms"] or kwargs["_all"]:
avms = dry_vms()
if not is_dry_run:
remove_vms(avms=avms)
logger.info(f"Stopped VMs: \n{avms['stop']}")
logger.info(f"Removed VMs: \n{avms['delete']}")
logger.info(f"Skipped VMs: \n{avms['skip']}")
if kwargs["nics"] or kwargs["_all"]:
rnics = dry_nics()
if not is_dry_run and rnics:
aws_client.remove_all_unused_nics()
logger.info(f"Removed NICs: \n{rnics}")
if kwargs["discs"] or kwargs["_all"]:
rdiscs = dry_discs()
if not is_dry_run and rdiscs:
aws_client.remove_all_unused_volumes()
logger.info(f"Removed Discs: \n{rdiscs}")
if kwargs["pips"] or kwargs["_all"]:
rpips = dry_pips()
if not is_dry_run and rpips:
aws_client.remove_all_unused_ips()
logger.info(f"Removed PIPs: \n{rpips}")
if kwargs["stacks"] or kwargs["_all"]:
rstacks = dry_stacks()
if not is_dry_run:
remove_stacks(stacks=rstacks)
logger.info(f"Removed Stacks: \n{rstacks}")
if is_dry_run:
echo_dry(dry_data)
Loading