diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7e0b63f..0bfd8f2 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -9,9 +9,9 @@ python setup.py develop If you do not plan on modifying the code and will simply be using it, instead run: ``` python setup.py install -``` +``` -If you have the AWS CLI, you can run `aws configure` to generate the credentials files in the appropriate place. If you have already configured the AWS CLI, then no further steps are necessary. +If you have the AWS CLI, you can run `aws configure` to generate the credentials files in the appropriate place. If you have already configured the AWS CLI, then no further steps are necessary. You must ensure that the account you are authenticating with has at least the following permissions: @@ -20,7 +20,7 @@ You must ensure that the account you are authenticating with has at least the fo "ec2:DescribeRegions"], "Effect": "Allow", "Resource": "*" }]} ``` -This is required to perform the VPC lookups. +This is required to perform the VPC lookups. Run/Test/Clean @@ -40,7 +40,7 @@ python setup.py test python setup.py clean —-all ``` -Note: *.pyc files will be regenerated in src whenever you run the test suite but as long as they are git ignored it’s not a big deal. You can still remove them with `rm src/**/*.pyc` +Note: *.pyc files will be regenerated in src whenever you run the test suite but as long as they are git ignored it’s not a big deal. You can still remove them with `rm src/**/*.pyc` Getting started @@ -61,24 +61,23 @@ class MyEnv(NetworkBase): self.add_child_template(Bastion()) if __name__ == '__main__': - my_config = EnvConfig(config_handlers=[Bastion]) - MyEnv(env_config=my_config) + MyEnv() ``` To generate the cloudformation template for this python code, save the above snippet in a file called `my_env.py` and run `python my_env.py init`. -This will look at the patterns passed into the EnvConfig object and generate a config.json file with the relevant fields added. Fill this config file out, adding values for at least the following fields: +This will generate a config.json using the default config (and any loaded subclasses of Template which extend the default config) file with the relevant fields added. Fill this config file out, adding values for at least the following fields: -`template : ec2_key_default` - SSH key used to log into your EC2 instances -`template : s3_bucket` - S3 bucket used to upload the generated cloudformation templates +`template : ec2_key_default` - SSH key used to log into your EC2 instances +`template : s3_bucket` - S3 bucket used to upload the generated cloudformation templates Next run `python my_env.py create` to generate the cloudformation template using the updated config. Since we overrode environmentbase's `create_hook` function, this will hook into environmentbase's create action and add the bastion stack and any other resources you specified. -NOTE: You can also override config values using environment variables. You can create env variables using the format: -`
_` in all caps (e.g. `TEMPLATE_EC2_KEY_DEFAULT`) +NOTE: You can also override config values using environment variables. You can create env variables using the format: +`
_` in all caps (e.g. `TEMPLATE_EC2_KEY_DEFAULT`) -These are read in after the config file is loaded, so will override any values in your config.json +These are read in after the config file is loaded, so will override any values in your config.json Then run `python my_env.py deploy` to create the stack on [cloudformation](https://console.aws.amazon.com/cloudformation/) @@ -102,12 +101,12 @@ By default, networkbase will create one public and one private subnet for each a ], "subnet_config": [ { - "type": "public", + "type": "public", "size": "18", "name": "public" }, { - "type": "private", + "type": "private", "size": "22", "name": "private" }, @@ -136,18 +135,18 @@ Extension point for modifying behavior of delete action. Called after config is Extension point for reacting to the cloudformation stack event stream. If global.monitor_stack is enabled in config this function is used to react to stack events. Once a stack is created a notification topic will begin emitting events to a queue. Each event is passed to this call for further processing. The return value is used to indicate whether processing is complete (true indicates processing is complete, false indicates you are not yet done). Details about the event data can be read [here](http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-listing-event-history.html) -The event_data hash provided the following mappings from the raw cloudformation event: -status = ResourceStatus -type = ResourceType -name = LogicalResourceId -reason = ResourceStatusReason -props = ResourceProperties +The event_data hash provided the following mappings from the raw cloudformation event: +status = ResourceStatus +type = ResourceType +name = LogicalResourceId +reason = ResourceStatusReason +props = ResourceProperties ## Dealing with versions 1. Basic commands - Reading tag information + Reading tag information - List all tags (apply across all branches): `git tag -l` @@ -197,7 +196,7 @@ props = ResourceProperties 2. Make code changes, test .. etc (merge changes from master into this branch) Remember to update: src/environmentbase/version.py to the same value as the branch name. 3. Merge into master - 4. Delete the version branch + 4. Delete the version branch 5. Create tag on master with same name as branch you just deleted. - + Note that the branch is deleted **before** the tag is created because they share the same name. Otherwise referring to things by the version name may be ambigious. diff --git a/src/environmentbase/environmentbase.py b/src/environmentbase/environmentbase.py index 5a59c93..d5440be 100644 --- a/src/environmentbase/environmentbase.py +++ b/src/environmentbase/environmentbase.py @@ -24,14 +24,6 @@ class ValidationError(Exception): pass -class EnvConfig(object): - - def __init__(self, config_handlers=None): - self.config_handlers = config_handlers if config_handlers else [] - # self.stack_event_handlers = stack_event_handlers if stack_event_handlers else [] - # self.deploy_handlers = deploy_handlers if deploy_handlers else {} - - class EnvironmentBase(object): """ EnvironmentBase encapsulates functionality required to build and deploy a network and common resources for object storage within a specified region @@ -39,9 +31,9 @@ class EnvironmentBase(object): def __init__(self, view=None, - env_config=EnvConfig(), config_filename=res.R.CONFIG_FILENAME, - config_file_override=None): + config_file_override=None, + is_silent=False): """ Init method for environment base creates all common objects for a given environment within the CloudFormation template including a network, s3 bucket and requisite policies to allow ELB Access log aggregation and @@ -53,7 +45,6 @@ def __init__(self, """ self.config_filename = config_filename - self.env_config = env_config self.config_file_override = config_file_override self.config = {} self.globals = {} @@ -62,7 +53,6 @@ def __init__(self, self.deploy_parameter_bindings = [] self.ignore_outputs = ['templateValidationHash', 'dateGenerated'] self.stack_outputs = {} - self._config_handlers = [] self.stack_monitor = None self._ami_cache = None self.cfn_connection = None @@ -70,10 +60,9 @@ def __init__(self, self.boto_session = None - # self.env_config = env_config - for config_handler in env_config.config_handlers: - self._add_config_handler(config_handler) - self.add_config_hook() + # Show names of Template subclasses + if not is_silent: + print "Using patterns: %s" % [cls.__name__ for cls in utility.get_pattern_list()] # Load the user interface self.view = view if view else cli.CLI() @@ -92,14 +81,6 @@ def create_hook(self): """ pass - def add_config_hook(self): - """ - Override in your subclass for adding custom config handlers. - Called after the other config handlers have been added. - After the hook completes the view is loaded and started. - """ - pass - def deploy_hook(self): """ Extension point for modifying behavior of deploy action. Called after config is loaded and before @@ -148,8 +129,7 @@ def init_action(self, is_silent=False): Override in your subclass for custom initialization steps @param is_silent [boolean], supress console output (for testing) """ - config_handlers = self.env_config.config_handlers - res.R.generate_config(prompt=True, is_silent=is_silent, output_filename=self.config_filename, config_handlers=config_handlers) + res.R.generate_config(prompt=True, is_silent=is_silent, output_filename=self.config_filename) def s3_prefix(self): """ @@ -250,6 +230,22 @@ def _root_template_url(self): bucket_name=self.template_args.get('s3_bucket'), resource_path=self._root_template_path()) + def load_runtime_config(self): + """ + For patterns defining custom config sections bind the loaded config values to associated class. + For class TestPattern whose get_factory_defaults() returns { "fav_color": "red" } will be able + to retreive the loaded value from the build_hook() (e.g. TestPattern.runtime_config['fav_color']) + """ + pattern_classes = utility.get_pattern_list() + for cls in pattern_classes: + runtime_config = {} + default_config = cls.get_factory_defaults() + for key in default_config.keys(): + value = self.config[key] + runtime_config[key] = value + + cls.runtime_config = runtime_config + def create_action(self): """ Default create_action invoked by the CLI @@ -257,6 +253,7 @@ def create_action(self): Override the create_hook in your environment to inject all of your cloudformation resources """ self.load_config() + self.load_runtime_config() self.initialize_template() # Do custom troposphere resource creation in your overridden copy of this method @@ -461,27 +458,13 @@ def _validate_config(self, config, factory_schema=None): config_reqs_copy = copy.deepcopy(factory_schema) # Merge in any requirements provided by config handlers - for handler in self._config_handlers: - config_reqs_copy.update(handler.get_config_schema()) + utility.update_schema_from_patterns(config_reqs_copy) self._validate_config_helper(config_reqs_copy, config, '') # Validate region self._validate_region(config) - def _add_config_handler(self, handler): - """ - Register classes that will augment the configuration defaults and/or validation logic here - """ - - if not hasattr(handler, 'get_factory_defaults') or not callable(getattr(handler, 'get_factory_defaults')): - raise ValidationError('Class %s cannot be a config handler, missing get_factory_defaults()' % type(handler).__name__ ) - - if not hasattr(handler, 'get_config_schema') or not callable(getattr(handler, 'get_config_schema')): - raise ValidationError('Class %s cannot be a config handler, missing get_config_schema()' % type(handler).__name__ ) - - self._config_handlers.append(handler) - @staticmethod def _config_env_override(config, path, print_debug=False): """ diff --git a/src/environmentbase/networkbase.py b/src/environmentbase/networkbase.py index 48162d0..ebbd5f1 100755 --- a/src/environmentbase/networkbase.py +++ b/src/environmentbase/networkbase.py @@ -10,19 +10,10 @@ class NetworkBase(EnvironmentBase): for a common deployment within AWS. This is intended to be the 'base' stack for deploying child stacks """ - def add_config_hook(self): - super(NetworkBase, self).add_config_hook() - self._add_config_handler(BaseNetwork) - def create_hook(self): super(NetworkBase, self).create_hook() - network_config = self.config.get('network', {}) - boto_config = self.config.get('boto', {}) - nat_config = self.config.get('nat') - region_name = boto_config['region_name'] - - base_network_template = BaseNetwork('BaseNetwork', network_config, region_name, nat_config) + base_network_template = BaseNetwork('BaseNetwork') self.add_child_template(base_network_template) self.template._subnets = base_network_template._subnets.copy() diff --git a/src/environmentbase/patterns/base_network.py b/src/environmentbase/patterns/base_network.py index 2ef94d8..02ce153 100644 --- a/src/environmentbase/patterns/base_network.py +++ b/src/environmentbase/patterns/base_network.py @@ -73,13 +73,12 @@ def get_factory_defaults(): def get_config_schema(): return BaseNetwork.CONFIG_SCHEMA - def __init__(self, template_name, network_config, region_name, nat_config): + def __init__(self, template_name): super(BaseNetwork, self).__init__(template_name) - self.network_config = network_config - self.nat_config = nat_config - self.az_count = int(network_config.get('az_count', '2')) - self.region_name = region_name + self.network_config = self.runtime_config['network'] + self.nat_config = self.runtime_config['nat'] + self.az_count = int(self.network_config.get('az_count', '2')) self.stack_outputs = {} # Simple mapping of AZs to NATs, to prevent creating duplicates diff --git a/src/environmentbase/patterns/bastion.py b/src/environmentbase/patterns/bastion.py index b4c08dd..9ce6898 100644 --- a/src/environmentbase/patterns/bastion.py +++ b/src/environmentbase/patterns/bastion.py @@ -9,18 +9,12 @@ class Bastion(Template): Adds a bastion host within a given deployment based on environemntbase. """ - SUGGESTED_INSTANCE_TYPES = [ - "m1.small", "t2.micro", "t2.small", "t2.medium", - "m3.medium", - "c3.large", "c3.2xlarge" - ] - def __init__(self, name='bastion', - ingress_port='2222', - access_cidr='0.0.0.0/0', - default_instance_type='t2.micro', - suggested_instance_types=SUGGESTED_INSTANCE_TYPES, + ingress_port=None, + access_cidr=None, + default_instance_type=None, + suggested_instance_types=None, user_data=None): """ Method initializes bastion host in a given environment deployment @@ -29,15 +23,20 @@ def __init__(self, More info here: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-elb-listener.html @param access_cidr [string] - CIDR notation for external access to this tier. @param user_data [string] - User data to to initialize the bastion hosts. + @param default_instance_type [string] EC2 instance type name + @param suggested_instance_types [list] List of EC2 instance types available for selection in CloudFormation """ self.name = name - self.ingress_port = ingress_port - self.access_cidr = access_cidr - self.default_instance_type = default_instance_type - self.suggested_instance_types = suggested_instance_types self.user_data = user_data + # Let constructor parameters override runtime config settings + cfg = self.runtime_config['bastion'] + self.ingress_port = ingress_port or cfg['ingress_port'] + self.access_cidr = access_cidr or cfg['remote_access_cidr'] + self.default_instance_type = default_instance_type or cfg['default_instance_type'] + self.suggested_instance_types = suggested_instance_types or cfg['suggested_instance_types'] + super(Bastion, self).__init__(template_name=name) # Called after add_child_template() has attached common parameters and some instance attributes: @@ -72,7 +71,7 @@ def build_hook(self): utility_bucket=self.utility_bucket ) - bastion_asg = self.add_asg( + self.add_asg( layer_name=self.name, security_groups=[security_groups['bastion'], self.common_security_group], load_balancer=bastion_elb, @@ -88,7 +87,7 @@ def build_hook(self): self.add_output(Output( 'BastionELBDNSZoneId', - Value=GetAtt(bastion_elb, 'CanonicalHostedZoneNameID') + Value=GetAtt(bastion_elb, 'CanonicalHostedZoneNameID') )) self.add_output(Output( @@ -99,7 +98,12 @@ def build_hook(self): @staticmethod def get_factory_defaults(): return {"bastion": { - "instance_type": "t2.micro", + "default_instance_type": "t2.micro", + "suggested_instance_types": [ + "m1.small", "t2.micro", "t2.small", "t2.medium", + "m3.medium", + "c3.large", "c3.2xlarge" + ], "remote_access_cidr": "0.0.0.0/0", "ingress_port": 2222 }} @@ -107,7 +111,8 @@ def get_factory_defaults(): @staticmethod def get_config_schema(): return {"bastion": { - "instance_type": "str", + "default_instance_type": "str", + "suggested_instance_types": "list", "remote_access_cidr": "str", "ingress_port": "int" }} diff --git a/src/environmentbase/patterns/rds.py b/src/environmentbase/patterns/rds.py index 82c2a4f..2439eb8 100644 --- a/src/environmentbase/patterns/rds.py +++ b/src/environmentbase/patterns/rds.py @@ -1,7 +1,6 @@ from environmentbase.template import Template import environmentbase.resources as res from environmentbase.networkbase import NetworkBase -from environmentbase.environmentbase import EnvConfig from troposphere import Ref, Parameter, GetAtt, Output, Join, rds, ec2 @@ -59,7 +58,7 @@ def __init__(self, connect_from_cidr=None, connect_from_sg=None, subnet_set='private', - config_map=DEFAULT_CONFIG['db']): + config_map=None): """ Method initializes host in a given environment deployment @param tier_name: [string] - name of the tier to assign @@ -70,7 +69,7 @@ def __init__(self, """ self.tier_name = tier_name - self.config_map = config_map + self.config_map = config_map or self.runtime_config['db'] self.subnet_set = subnet_set if connect_from_cidr and connect_from_sg: @@ -247,64 +246,62 @@ class Controller(NetworkBase): > mysql -h -P -u -p """ + # For this example override the loaded config with the config below + db_config = { + "label1": { + "db_instance_type_default": "db.m1.small", + "rds_user_name": "defaultusername", + # Actual database name, cannot include non-alphanumeric characters (e.g. "-") + "master_db_name": "mydb", + "volume_size": 100, + "backup_retention_period": 30, + "rds_engine": "MySQL", + # 5.6.19 is no longer supported + "rds_engine_version": "5.6.22", + "preferred_backup_window": "02:00-02:30", + "preferred_maintenance_window": "sun:03:00-sun:04:00", + # Name of vm snapshot to use, empty string ("") means don't use an old snapshot + # Note: "master_db_name" value will be overridden if snapshot_id is non-empty + "snapshot_id": "", + "password": "changeme111111111111" + }, + "label2": { + "db_instance_type_default": "db.m1.small", + "rds_user_name": "defaultusername", + # Actual database name, cannot include non-alphanumeric characters (e.g. "-") + "master_db_name": "mydb2", + "volume_size": 100, + "backup_retention_period": 30, + "rds_engine": "MySQL", + # 5.6.19 is no longer supported + "rds_engine_version": "5.6.22", + "preferred_backup_window": "02:00-02:30", + "preferred_maintenance_window": "sun:03:00-sun:04:00", + # Name of vm snapshot to use, empty string ("") means don't use an old snapshot + # Note: "master_db_name" value will be overridden if snapshot_id is non-empty + "snapshot_id": "", + "password": "changeme1111111111111" + } + } + def create_hook(self): + super(Controller, self).create_hook() + # Create the rds instance pattern (includes standard standard parameters) my_db = RDS( 'dbTier', - subnet_set=self.template.subnets['private'].keys()[0], - config_map=db_config) + subnet_set='private', + config_map=self.db_config) # Attach pattern as a child template self.add_child_template(my_db) def deploy_hook(self): - for db_label, db_config in self.config['db'].iteritems(): + for db_label, db_config in self.db_config.iteritems(): db_resource_name = db_label.lower() + 'dbTier'.title() + 'RdsMasterUserPassword' print "adding ", db_resource_name self.add_parameter_binding(key=db_resource_name, value=db_config['password']) if __name__ == '__main__': - - db_config = { - 'label1': { - 'db_instance_type_default': 'db.m1.small', - 'rds_user_name': 'defaultusername', - # Actual database name, cannot include non-alphanumeric characters (e.g. '-') - 'master_db_name': 'mydb', - 'volume_size': 100, - 'backup_retention_period': 30, - 'rds_engine': 'MySQL', - # 5.6.19 is no longer supported - 'rds_engine_version': '5.6.22', - 'preferred_backup_window': '02:00-02:30', - 'preferred_maintenance_window': 'sun:03:00-sun:04:00', - # Name of vm snapshot to use, empty string ('') means don't use an old snapshot - # Note: 'master_db_name' value will be overridden if snapshot_id is non-empty - 'snapshot_id': '', - 'password': 'changeme111111111111' - }, - 'label2': { - 'db_instance_type_default': 'db.m1.small', - 'rds_user_name': 'defaultusername', - # Actual database name, cannot include non-alphanumeric characters (e.g. '-') - 'master_db_name': 'mydb2', - 'volume_size': 100, - 'backup_retention_period': 30, - 'rds_engine': 'MySQL', - # 5.6.19 is no longer supported - 'rds_engine_version': '5.6.22', - 'preferred_backup_window': '02:00-02:30', - 'preferred_maintenance_window': 'sun:03:00-sun:04:00', - # Name of vm snapshot to use, empty string ('') means don't use an old snapshot - # Note: 'master_db_name' value will be overridden if snapshot_id is non-empty - 'snapshot_id': '', - 'password': 'changeme1111111111111' - } - } - - my_config = res.FACTORY_DEFAULT_CONFIG - my_config['db'] = db_config - - env_config = EnvConfig(config_handlers=[RDS]) - Controller(env_config=env_config, config_file_override=my_config) + Controller() diff --git a/src/environmentbase/resources.py b/src/environmentbase/resources.py index 35758bc..8ad50f0 100644 --- a/src/environmentbase/resources.py +++ b/src/environmentbase/resources.py @@ -4,6 +4,7 @@ import json import os import re +import utility # Declare R to be the singleton Resource instance @@ -159,28 +160,18 @@ def _extract_config_section(self, config, config_key, filename, prompt=False): def generate_config(self, config_file=CONFIG_FILENAME, output_filename=None, - config_handlers=list(), extract_map=_EXTRACTED_CONFIG_SECTIONS, prompt=False, is_silent=False): """ Copies specified yaml/json file from the EGG resource to current directory, default is 'conifg.json'. Optionally - split out specific sections into separate files using extract_map. Additionally us config_handlers to add in - additional conifg content before serializing content to file. + split out specific sections into separate files using extract_map. @param config_file [string] Name of file within resource path to load. @param output_file [string] Name of generated config file (default is same as 'config_file') @param prompt [boolean] block for user input to abort file output if file already exists @param is_silent [boolena] supress console output (primarly for testing) @param extract_map [map] Specifies top-level sections of config to externalize to separate file. Where key=config section name, value=filename. - @param config_handlers [list(objects)] Config handlers should resemble the following: - class CustomHandler(object): - @staticmethod - def get_factory_defaults(): - return custom_config_addition - @staticmethod - def get_config_schema(): - return custom_config_validation """ # Output same file name as the input unless specified otherwise if not output_filename: @@ -190,8 +181,7 @@ def get_config_schema(): config = self.parse_file(config_file, from_file=False) # Merge in any defaults provided by registered config handlers - for handler in config_handlers: - config.update(handler.get_factory_defaults()) + utility.update_config_from_patterns(config) # Make changes to a new copy of the config config_copy = copy.deepcopy(config) diff --git a/src/environmentbase/template.py b/src/environmentbase/template.py index 6812543..67487ac 100644 --- a/src/environmentbase/template.py +++ b/src/environmentbase/template.py @@ -47,6 +47,9 @@ class Template(t.Template): # Region to deploy to region = '' + # Custom config loaded by EnvironmentBase.load_pattern_configs() + runtime_config = {} + ARCH_MAP = "InstanceTypeToArch" IMAGE_MAP_PREFIX = "ImageMapFor" diff --git a/src/environmentbase/utility.py b/src/environmentbase/utility.py index d0a24e3..63cb35d 100644 --- a/src/environmentbase/utility.py +++ b/src/environmentbase/utility.py @@ -119,6 +119,7 @@ def get_stack_depends_on_from_parent_template(parent_template_contents, stack_na # Otherwise return the DependsOn list that the stack was deployed with return stack_reference.get('DependsOn') + def get_template_s3_resource_path(prefix, template_name, include_timestamp=True): """ Constructs s3 resource path for provided template name @@ -143,3 +144,34 @@ def get_template_s3_url(bucket_name, resource_path): """ return 'https://%s.s3.amazonaws.com/%s' % (bucket_name, resource_path) + +def _get_subclasses_of(parent_import_path, parent_classname): + # Import environmentbase.template.Template (in case it's not already) + _module = __import__(parent_import_path, fromlist=[parent_classname]) + parent_class = getattr(_module, parent_classname) + subclasses = parent_class.__subclasses__() + return subclasses + + +def get_pattern_list(): + """ + Returns list of all imported subclasses of environmentbase.template.Template + """ + return _get_subclasses_of('environmentbase.template', 'Template') + + +def _update_from_patterns(_dict, fun_name): + class_list = get_pattern_list() + for template_subclass in class_list: + additional_dict = getattr(template_subclass, fun_name)() + _dict.update(additional_dict) + + return _dict + + +def update_schema_from_patterns(config_schema, class_list=None): + return _update_from_patterns(config_schema, 'get_config_schema') + + +def update_config_from_patterns(config, class_list=None): + return _update_from_patterns(config, 'get_factory_defaults') diff --git a/src/examples/basic.py b/src/examples/basic.py index 6c29af6..f1e2d40 100755 --- a/src/examples/basic.py +++ b/src/examples/basic.py @@ -1,8 +1,6 @@ -from environmentbase.environmentbase import EnvConfig from environmentbase.networkbase import NetworkBase from environmentbase.patterns.bastion import Bastion from environmentbase.patterns.ha_cluster import HaCluster -from environmentbase.patterns.base_network import BaseNetwork class MyEnvClass(NetworkBase): @@ -26,5 +24,4 @@ def create_hook(self): suggested_instance_types=['t2.micro'])) if __name__ == '__main__': - env_config = EnvConfig(config_handlers=[BaseNetwork]) - MyEnvClass(env_config=env_config) + MyEnvClass() diff --git a/src/examples/child_stack.py b/src/examples/child_stack.py index 3f4250f..d8451ae 100644 --- a/src/examples/child_stack.py +++ b/src/examples/child_stack.py @@ -1,6 +1,5 @@ from environmentbase.networkbase import NetworkBase from environmentbase.template import Template -from environmentbase.environmentbase import EnvConfig from troposphere import ec2 @@ -19,7 +18,7 @@ class MyChildTemplate(Template): # Called from add_child_template() after some common parameters are attached to this instance, see docs for details def build_hook(self): - self.add_resource(ec2.Instance("ec2instance", InstanceType="m3.medium", ImageId="ami-e7527ed7") ) + self.add_resource(ec2.Instance("ec2instance", InstanceType="m3.medium", ImageId="ami-e7527ed7")) # When no config.json file exists a new one is created using the 'factory default' file. This function # augments the factory default before it is written to file with the config values required @@ -35,10 +34,4 @@ def get_config_schema(): return {'my_child_template': {'favorite_color': 'str'}} if __name__ == '__main__': - - # EnvConfig holds references to handler classes used to extend certain functionality - # of EnvironmentBase. The config_handlers list takes any class that implements - # get_factory_defaults() and get_config_schema(). - env_config = EnvConfig(config_handlers=[MyChildTemplate]) - - MyRootTemplate(env_config=env_config) + MyRootTemplate() diff --git a/src/examples/nested_child_stack.py b/src/examples/nested_child_stack.py index 96f20df..c74cd37 100644 --- a/src/examples/nested_child_stack.py +++ b/src/examples/nested_child_stack.py @@ -1,6 +1,5 @@ from environmentbase.networkbase import NetworkBase from environmentbase.template import Template -from environmentbase.environmentbase import EnvConfig from environmentbase.patterns.bastion import Bastion from troposphere import ec2 @@ -10,6 +9,9 @@ class Controller(NetworkBase): Class creates a VPC and common network components for the environment """ def create_hook(self): + super(Controller, self).create_hook() + + # print self.template.subnets self.add_child_template(ChildTemplate('Child')) self.add_child_template(Bastion('Bastion')) @@ -26,10 +28,10 @@ def build_hook(self): InstanceType="m3.medium", ImageId="ami-e7527ed7", KeyName=self.ec2_key, - SubnetId=self.subnets['private'][0], + SubnetId=self.subnets['private']['private'][0], SecurityGroupIds=[self.common_security_group] )) - self.add_child_template(GrandchildTemplate('Grandchild')) + # self.add_child_template(GrandchildTemplate('Grandchild')) # When no config.json file exists a new one is created using the 'factory default' file. This function # augments the factory default before it is written to file with the config values required @@ -52,14 +54,8 @@ def build_hook(self): InstanceType="m3.medium", ImageId="ami-e7527ed7", KeyName=self.ec2_key, - SubnetId=self.subnets['private'][0], + SubnetId=self.subnets['private']['private'][0], SecurityGroupIds=[self.common_security_group])) if __name__ == '__main__': - - # EnvConfig holds references to handler classes used to extend certain functionality - # of EnvironmentBase. The config_handlers list takes any class that implements - # get_factory_defaults() and get_config_schema(). - env_config = EnvConfig(config_handlers=[ChildTemplate]) - - Controller(env_config=env_config) + Controller() diff --git a/src/tests/test_environmentbase.py b/src/tests/test_environmentbase.py index a529f96..985a829 100644 --- a/src/tests/test_environmentbase.py +++ b/src/tests/test_environmentbase.py @@ -8,10 +8,21 @@ import sys import copy from tempfile import mkdtemp -from environmentbase import cli, resources as res, environmentbase as eb +from environmentbase import cli, resources as res, environmentbase as eb, utility from environmentbase import networkbase import environmentbase.patterns.ha_nat from troposphere import ec2 +from environmentbase.template import Template + + +class MyTemplate(Template): + @staticmethod + def get_factory_defaults(): + return {'new_section': {'new_key': 'value'}} + + @staticmethod + def get_config_schema(): + return {'new_section': {'new_key': 'basestring'}} class EnvironmentBaseTestCase(TestCase): @@ -36,17 +47,14 @@ def fake_cli(self, extra_args): return my_cli - def _create_dummy_config(self, env_base=None): + def _create_dummy_config(self): dummy_string = 'dummy' dummy_bool = False dummy_int = 3 dummy_list = ['A', 'B', 'C'] config_requirements = res.R.parse_file(res.Res.CONFIG_REQUIREMENTS_FILENAME, from_file=False) - - if env_base: - for handler in env_base.config_handlers: - config_requirements.update(handler.get_config_schema()) + utility.update_schema_from_patterns(config_requirements) config = {} for (section, keys) in config_requirements.iteritems(): @@ -76,7 +84,7 @@ def _create_local_file(self, name, content): def test_constructor(self): """Make sure EnvironmentBase passes control to view to process user requests""" fake_cli = self.fake_cli(['init']) - env_base = eb.EnvironmentBase(fake_cli) + env_base = eb.EnvironmentBase(fake_cli, is_silent=True) # Check that EnvironmentBase started the CLI fake_cli.process_request.assert_called_once_with(env_base) @@ -110,7 +118,7 @@ def process_request(self, controller): actions_called[action] += 1 - eb.EnvironmentBase(MyView()) + eb.EnvironmentBase(MyView(), is_silent=True) self.assertEqual(actions_called['init'], 1) self.assertEqual(actions_called['create'], 1) @@ -121,13 +129,14 @@ def test_config_yaml(self): """ Verify load_config can load non-default files """ alt_config_filename = 'config.yaml' config = res.R.parse_file(res.Res.CONFIG_FILENAME, from_file=False) + utility.update_config_from_patterns(config) with open(alt_config_filename, 'w') as f: f.write(yaml.dump(config, default_flow_style=False)) f.flush() fake_cli = self.fake_cli(['create', '--config-file', 'config.yaml']) - base = eb.EnvironmentBase(fake_cli, config_filename=alt_config_filename) + base = eb.EnvironmentBase(fake_cli, config_filename=alt_config_filename, is_silent=True) base.load_config() self.assertEqual(base.config['global']['environment_name'], 'environmentbase') @@ -147,7 +156,7 @@ def test_config_override(self): f.flush() fake_cli = self.fake_cli(['create']) - base = eb.EnvironmentBase(fake_cli) + base = eb.EnvironmentBase(fake_cli, is_silent=True) base.load_config() self.assertNotEqual(base.config['global']['environment_name'], original_value) @@ -157,7 +166,7 @@ def test_config_override(self): # existence check with self.assertRaises(Exception): - base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', config_filename])) + base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', config_filename]), is_silent=True) base.load_config() # remove config.json and create the alternate config file @@ -167,7 +176,7 @@ def test_config_override(self): with open(config_filename, 'w') as f: f.write(yaml.dump(config)) f.flush() - base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', config_filename])) + base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', config_filename]), is_silent=True) base.load_config() self.assertNotEqual(base.config['global']['environment_name'], original_value) @@ -177,7 +186,7 @@ def test_config_validation(self): environmentbase.TEMPLATE_REQUIREMENTS defines the required sections and keys for a valid input config file This test ensures that EnvironmentBase._validate_config() enforces the TEMPLATE_REQUIREMENTS contract """ - cntrl = eb.EnvironmentBase(self.fake_cli(['create'])) + cntrl = eb.EnvironmentBase(self.fake_cli(['create']), is_silent=True) valid_config = self._create_dummy_config() cntrl._validate_config(valid_config) @@ -240,9 +249,7 @@ def test_config_validation(self): }}}}) def test_extending_config(self): - - # Typically this would subclass eb.Template - class MyConfigHandler(object): + class MyTemplate(eb.Template): @staticmethod def get_factory_defaults(): return {'new_section': {'new_key': 'value'}} @@ -255,11 +262,11 @@ class MyEnvBase(eb.EnvironmentBase): pass view = self.fake_cli(['init']) - env_config = eb.EnvConfig(config_handlers=[MyConfigHandler]) controller = MyEnvBase( view=view, - env_config=env_config + is_silent=True ) + controller.init_action(is_silent=True) controller.load_config() @@ -274,17 +281,18 @@ class MyEnvBase(eb.EnvironmentBase): # recreate config file without 'new_section' and make sure it fails validation os.remove(res.Res.CONFIG_FILENAME) dummy_config = self._create_dummy_config() + del dummy_config['new_section'] self._create_local_file(res.Res.CONFIG_FILENAME, json.dumps(dummy_config, indent=4)) with self.assertRaises(eb.ValidationError): - base = MyEnvBase(view=view, env_config=env_config) + base = MyEnvBase(view=view, is_silent=True) base.load_config() def test_generate_config(self): """ Verify cli flags update config object """ # Verify that debug and output are set to the factory default - base = eb.EnvironmentBase(self.fake_cli(['init'])) + base = eb.EnvironmentBase(self.fake_cli(['init']), is_silent=True) res.R.generate_config(prompt=True, is_silent=True) base.load_config() @@ -298,20 +306,20 @@ def test_generate_config(self): def test_template_file_flag(self): # verify that the --template-file flag changes the config value dummy_value = 'dummy' - base = eb.EnvironmentBase(self.fake_cli(['create', '--template-file', dummy_value])) + base = eb.EnvironmentBase(self.fake_cli(['create', '--template-file', dummy_value]), is_silent=True) base.init_action(is_silent=True) base.load_config() self.assertEqual(base.config['global']['environment_name'], dummy_value) def test_config_file_flag(self): dummy_value = 'dummy' - base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', dummy_value])) + base = eb.EnvironmentBase(self.fake_cli(['create', '--config-file', dummy_value]), is_silent=True) base.init_action(is_silent=True) self.assertTrue(os.path.isfile(dummy_value)) def test_factory_default(self): with self.assertRaises(Exception): - base = eb.EnvironmentBase(self.fake_cli(['init'])) + base = eb.EnvironmentBase(self.fake_cli(['init']), is_silent=True) base.load_config() # Create refs to files that should be created and make sure they don't already exists @@ -321,14 +329,24 @@ def test_factory_default(self): self.assertFalse(os.path.isfile(ami_cache_file)) # Verify that create_missing_files works as intended - base = eb.EnvironmentBase(self.fake_cli(['init'])) + base = eb.EnvironmentBase(self.fake_cli(['init']), is_silent=True) base.init_action(is_silent=True) self.assertTrue(os.path.isfile(config_file)) # TODO: After ami_cache is updated change 'create_missing_files' to be singular # self.assertTrue(os.path.isfile(ami_cache_file)) # Verify that the previously created files are loaded up correctly - eb.EnvironmentBase(self.fake_cli(['create'])) + eb.EnvironmentBase(self.fake_cli(['create']), is_silent=True) + + def test_load_runtime_config(self): + base = eb.EnvironmentBase(self.fake_cli(['create']), is_silent=True) + base.init_action(is_silent=True) + base.load_config() + + # verify that config section is attached to the class + base.config['new_section']['new_key'] = 'different_value' + base.load_runtime_config() + self.assertTrue(MyTemplate.runtime_config['new_section']['new_key'], 'different_value') # The following two tests use a create_action, which currently doesn't test correctly diff --git a/src/tests/test_resources.py b/src/tests/test_resources.py index e75519c..6801837 100644 --- a/src/tests/test_resources.py +++ b/src/tests/test_resources.py @@ -1,5 +1,6 @@ from unittest2 import TestCase from environmentbase.resources import Res +from environmentbase.template import Template from tempfile import mkdtemp import shutil import os @@ -101,7 +102,7 @@ def test_generate_config(self): } } - class CustomHandler(object): + class CustomHandler(Template): @staticmethod def get_factory_defaults(): return custom_config_addition @@ -120,8 +121,7 @@ def get_config_schema(): "Mappings": "mappings.json", "Resources": "resources.json", "Outputs": "output.json" - }, - config_handlers=[CustomHandler()] + } ) # Make sure all the extracted files exist diff --git a/src/tests/test_utility.py b/src/tests/test_utility.py new file mode 100644 index 0000000..db4e082 --- /dev/null +++ b/src/tests/test_utility.py @@ -0,0 +1,60 @@ +from unittest2 import TestCase +from environmentbase import utility +from environmentbase.template import Template + + +class Parent(object): + pass + + +class A(Parent): + pass + + +class B(Parent): + pass + + +class C(Parent): + pass + + +class MyTemplate(Template): + @staticmethod + def get_factory_defaults(): + return {'new_section': {'new_key': 'value'}} + + @staticmethod + def get_config_schema(): + return {'new_section': {'new_key': 'basestring'}} + + +class UtilityTestCase(TestCase): + + def test__get_subclasses_of(self): + actual_subclasses = [A, B, C] + retreived_subclasses = utility._get_subclasses_of('tests.test_utility', 'Parent') + self.assertEqual(actual_subclasses, retreived_subclasses) + + def test_get_pattern_list(self): + # Count patterns from previous test runs (no way to unload classes as far as I know) + num_patterns = len(utility.get_pattern_list()) + + # Verify that a loaded a pattern is identified + mod = __import__('environmentbase.patterns.bastion', fromlist=['Bastion']) + klazz = getattr(mod, 'Bastion') + patterns = utility.get_pattern_list() + + self.assertGreater(len(patterns), num_patterns) + self.assertIn(klazz, patterns) + + def test__update_from_patterns(self): + _dict = {} + utility._update_from_patterns(_dict, 'get_factory_defaults') + self.assertIn('new_section', _dict) + self.assertEqual('value', _dict['new_section']['new_key']) + + _dict = {} + utility._update_from_patterns(_dict, 'get_config_schema') + self.assertIn('new_section', _dict) + self.assertEqual('basestring', _dict['new_section']['new_key'])