D7net
Home
Console
Upload
information
Create File
Create Folder
About
Tools
:
/
proc
/
3
/
task
/
3
/
root
/
opt
/
imh-python
/
lib
/
python3.9
/
site-packages
/
openstack
/
cloud
/
Filename :
openstackcloud.py
back
Copy
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import copy import functools import queue # import types so that we can reference ListType in sphinx param declarations. # We can't just use list, because sphinx gets confused by # openstack.resource.Resource.list and openstack.resource2.Resource.list import types # noqa import warnings import dogpile.cache import munch import requests.models import requestsexceptions import keystoneauth1.exceptions import keystoneauth1.session from openstack import _log from openstack.cloud import exc from openstack.cloud import _floating_ip from openstack.cloud import _object_store from openstack.cloud import meta from openstack.cloud import _utils import openstack.config from openstack.config import cloud_region as cloud_region_mod from openstack import proxy from openstack import utils DEFAULT_SERVER_AGE = 5 DEFAULT_PORT_AGE = 5 DEFAULT_FLOAT_AGE = 5 _CONFIG_DOC_URL = _floating_ip._CONFIG_DOC_URL DEFAULT_OBJECT_SEGMENT_SIZE = _object_store.DEFAULT_OBJECT_SEGMENT_SIZE # This halves the current default for Swift DEFAULT_MAX_FILE_SIZE = _object_store.DEFAULT_MAX_FILE_SIZE OBJECT_CONTAINER_ACLS = _object_store.OBJECT_CONTAINER_ACLS class _OpenStackCloudMixin: """Represent a connection to an OpenStack Cloud. OpenStackCloud is the entry point for all cloud operations, regardless of which OpenStack service those operations may ultimately come from. The operations on an OpenStackCloud are resource oriented rather than REST API operation oriented. For instance, one will request a Floating IP and that Floating IP will be actualized either via neutron or via nova depending on how this particular cloud has decided to arrange itself. :param bool strict: Only return documented attributes for each resource as per the Data Model contract. (Default False) """ _OBJECT_MD5_KEY = 'x-object-meta-x-sdk-md5' _OBJECT_SHA256_KEY = 'x-object-meta-x-sdk-sha256' _OBJECT_AUTOCREATE_KEY = 'x-object-meta-x-sdk-autocreated' _OBJECT_AUTOCREATE_CONTAINER = 'images' # NOTE(shade) shade keys were x-object-meta-x-shade-md5 - we need to check # those in freshness checks so that a shade->sdk transition # doesn't result in a re-upload _SHADE_OBJECT_MD5_KEY = 'x-object-meta-x-shade-md5' _SHADE_OBJECT_SHA256_KEY = 'x-object-meta-x-shade-sha256' _SHADE_OBJECT_AUTOCREATE_KEY = 'x-object-meta-x-shade-autocreated' def __init__(self): super(_OpenStackCloudMixin, self).__init__() self.log = _log.setup_logging('openstack') self.name = self.config.name self.auth = self.config.get_auth_args() self.default_interface = self.config.get_interface() self.force_ipv4 = self.config.force_ipv4 (self.verify, self.cert) = self.config.get_requests_verify_args() # Turn off urllib3 warnings about insecure certs if we have # explicitly configured requests to tell it we do not want # cert verification if not self.verify: self.log.debug( "Turning off Insecure SSL warnings since verify=False") category = requestsexceptions.InsecureRequestWarning if category: # InsecureRequestWarning references a Warning class or is None warnings.filterwarnings('ignore', category=category) self._disable_warnings = {} cache_expiration_time = int(self.config.get_cache_expiration_time()) cache_class = self.config.get_cache_class() cache_arguments = self.config.get_cache_arguments() self._resource_caches = {} if cache_class != 'dogpile.cache.null': self.cache_enabled = True self._cache = self._make_cache( cache_class, cache_expiration_time, cache_arguments) expirations = self.config.get_cache_expirations() for expire_key in expirations.keys(): # Only build caches for things we have list operations for if getattr( self, 'list_{0}'.format(expire_key), None): self._resource_caches[expire_key] = self._make_cache( cache_class, expirations[expire_key], cache_arguments) self._SERVER_AGE = DEFAULT_SERVER_AGE self._PORT_AGE = DEFAULT_PORT_AGE self._FLOAT_AGE = DEFAULT_FLOAT_AGE else: self.cache_enabled = False def _fake_invalidate(unused): pass class _FakeCache: def invalidate(self): pass # Don't cache list_servers if we're not caching things. # Replace this with a more specific cache configuration # soon. self._SERVER_AGE = 0 self._PORT_AGE = 0 self._FLOAT_AGE = 0 self._cache = _FakeCache() # Undecorate cache decorated methods. Otherwise the call stacks # wind up being stupidly long and hard to debug for method in _utils._decorated_methods: meth_obj = getattr(self, method, None) if not meth_obj: continue if (hasattr(meth_obj, 'invalidate') and hasattr(meth_obj, 'func')): new_func = functools.partial(meth_obj.func, self) new_func.invalidate = _fake_invalidate setattr(self, method, new_func) # If server expiration time is set explicitly, use that. Otherwise # fall back to whatever it was before self._SERVER_AGE = self.config.get_cache_resource_expiration( 'server', self._SERVER_AGE) self._PORT_AGE = self.config.get_cache_resource_expiration( 'port', self._PORT_AGE) self._FLOAT_AGE = self.config.get_cache_resource_expiration( 'floating_ip', self._FLOAT_AGE) self._container_cache = dict() self._file_hash_cache = dict() # self.__pool_executor = None self._raw_clients = {} self._local_ipv6 = ( _utils.localhost_supports_ipv6() if not self.force_ipv4 else False) def connect_as(self, **kwargs): """Make a new OpenStackCloud object with new auth context. Take the existing settings from the current cloud and construct a new OpenStackCloud object with some of the auth settings overridden. This is useful for getting an object to perform tasks with as another user, or in the context of a different project. .. code-block:: python conn = openstack.connect(cloud='example') # Work normally servers = conn.list_servers() conn2 = conn.connect_as(username='different-user', password='') # Work as different-user servers = conn2.list_servers() :param kwargs: keyword arguments can contain anything that would normally go in an auth dict. They will override the same settings from the parent cloud as appropriate. Entries that do not want to be overridden can be ommitted. """ if self.config._openstack_config: config = self.config._openstack_config else: # TODO(mordred) Replace this with from_session config = openstack.config.OpenStackConfig( app_name=self.config._app_name, app_version=self.config._app_version, load_yaml_config=False) params = copy.deepcopy(self.config.config) # Remove profile from current cloud so that overridding works params.pop('profile', None) # Utility function to help with the stripping below. def pop_keys(params, auth, name_key, id_key): if name_key in auth or id_key in auth: params['auth'].pop(name_key, None) params['auth'].pop(id_key, None) # If there are user, project or domain settings in the incoming auth # dict, strip out both id and name so that a user can say: # cloud.connect_as(project_name='foo') # and have that work with clouds that have a project_id set in their # config. for prefix in ('user', 'project'): if prefix == 'user': name_key = 'username' else: name_key = 'project_name' id_key = '{prefix}_id'.format(prefix=prefix) pop_keys(params, kwargs, name_key, id_key) id_key = '{prefix}_domain_id'.format(prefix=prefix) name_key = '{prefix}_domain_name'.format(prefix=prefix) pop_keys(params, kwargs, name_key, id_key) for key, value in kwargs.items(): params['auth'][key] = value cloud_region = config.get_one(**params) # Attach the discovery cache from the old session so we won't # double discover. cloud_region._discovery_cache = self.session._discovery_cache # Override the cloud name so that logging/location work right cloud_region._name = self.name cloud_region.config['profile'] = self.name # Use self.__class__ so that we return whatever this if, like if it's # a subclass in the case of shade wrapping sdk. return self.__class__(config=cloud_region) def connect_as_project(self, project): """Make a new OpenStackCloud object with a new project. Take the existing settings from the current cloud and construct a new OpenStackCloud object with the project settings overridden. This is useful for getting an object to perform tasks with as another user, or in the context of a different project. .. code-block:: python cloud = openstack.connect(cloud='example') # Work normally servers = cloud.list_servers() cloud2 = cloud.connect_as_project('different-project') # Work in different-project servers = cloud2.list_servers() :param project: Either a project name or a project dict as returned by `list_projects`. """ auth = {} if isinstance(project, dict): auth['project_id'] = project.get('id') auth['project_name'] = project.get('name') if project.get('domain_id'): auth['project_domain_id'] = project['domain_id'] else: auth['project_name'] = project return self.connect_as(**auth) def global_request(self, global_request_id): """Make a new Connection object with a global request id set. Take the existing settings from the current Connection and construct a new Connection object with the global_request_id overridden. .. code-block:: python from oslo_context import context cloud = openstack.connect(cloud='example') # Work normally servers = cloud.list_servers() cloud2 = cloud.global_request(context.generate_request_id()) # cloud2 sends all requests with global_request_id set servers = cloud2.list_servers() Additionally, this can be used as a context manager: .. code-block:: python from oslo_context import context c = openstack.connect(cloud='example') # Work normally servers = c.list_servers() with c.global_request(context.generate_request_id()) as c2: # c2 sends all requests with global_request_id set servers = c2.list_servers() :param global_request_id: The `global_request_id` to send. """ params = copy.deepcopy(self.config.config) cloud_region = cloud_region_mod.from_session( session=self.session, app_name=self.config._app_name, app_version=self.config._app_version, discovery_cache=self.session._discovery_cache, **params) # Override the cloud name so that logging/location work right cloud_region._name = self.name cloud_region.config['profile'] = self.name # Use self.__class__ so that we return whatever this is, like if it's # a subclass in the case of shade wrapping sdk. new_conn = self.__class__(config=cloud_region) new_conn.set_global_request_id(global_request_id) return new_conn def _make_cache(self, cache_class, expiration_time, arguments): return dogpile.cache.make_region( function_key_generator=self._make_cache_key ).configure( cache_class, expiration_time=expiration_time, arguments=arguments) def _make_cache_key(self, namespace, fn): fname = fn.__name__ if namespace is None: name_key = self.name else: name_key = '%s:%s' % (self.name, namespace) def generate_key(*args, **kwargs): arg_key = ','.join(args) kw_keys = sorted(kwargs.keys()) kwargs_key = ','.join( ['%s:%s' % (k, kwargs[k]) for k in kw_keys if k != 'cache']) ans = "_".join( [str(name_key), fname, arg_key, kwargs_key]) return ans return generate_key def _get_cache(self, resource_name): if resource_name and resource_name in self._resource_caches: return self._resource_caches[resource_name] else: return self._cache def _get_major_version_id(self, version): if isinstance(version, int): return version elif isinstance(version, (str, tuple)): return int(version[0]) return version def _get_versioned_client( self, service_type, min_version=None, max_version=None): config_version = self.config.get_api_version(service_type) config_major = self._get_major_version_id(config_version) max_major = self._get_major_version_id(max_version) min_major = self._get_major_version_id(min_version) # TODO(shade) This should be replaced with use of Connection. However, # we need to find a sane way to deal with this additional # logic - or we need to give up on it. If we give up on it, # we need to make sure we can still support it in the shade # compat layer. # NOTE(mordred) This logic for versions is slightly different # than the ksa Adapter constructor logic. openstack.cloud knows the # versions it knows, and uses them when it detects them. However, if # a user requests a version, and it's not found, and a different one # openstack.cloud does know about is found, that's a warning in # openstack.cloud. if config_version: if min_major and config_major < min_major: raise exc.OpenStackCloudException( "Version {config_version} requested for {service_type}" " but shade understands a minimum of {min_version}".format( config_version=config_version, service_type=service_type, min_version=min_version)) elif max_major and config_major > max_major: raise exc.OpenStackCloudException( "Version {config_version} requested for {service_type}" " but openstack.cloud understands a maximum of" " {max_version}".format( config_version=config_version, service_type=service_type, max_version=max_version)) request_min_version = config_version request_max_version = '{version}.latest'.format( version=config_major) adapter = proxy._ShadeAdapter( session=self.session, service_type=self.config.get_service_type(service_type), service_name=self.config.get_service_name(service_type), interface=self.config.get_interface(service_type), endpoint_override=self.config.get_endpoint(service_type), region_name=self.config.get_region_name(service_type), statsd_prefix=self.config.get_statsd_prefix(), statsd_client=self.config.get_statsd_client(), prometheus_counter=self.config.get_prometheus_counter(), prometheus_histogram=self.config.get_prometheus_histogram(), influxdb_client=self.config.get_influxdb_client(), min_version=request_min_version, max_version=request_max_version) if adapter.get_endpoint(): return adapter adapter = proxy._ShadeAdapter( session=self.session, service_type=self.config.get_service_type(service_type), service_name=self.config.get_service_name(service_type), interface=self.config.get_interface(service_type), endpoint_override=self.config.get_endpoint(service_type), region_name=self.config.get_region_name(service_type), min_version=min_version, max_version=max_version) # data.api_version can be None if no version was detected, such # as with neutron api_version = adapter.get_api_major_version( endpoint_override=self.config.get_endpoint(service_type)) api_major = self._get_major_version_id(api_version) # If we detect a different version that was configured, warn the user. # shade still knows what to do - but if the user gave us an explicit # version and we couldn't find it, they may want to investigate. if api_version and config_version and (api_major != config_major): warning_msg = ( '{service_type} is configured for {config_version}' ' but only {api_version} is available. shade is happy' ' with this version, but if you were trying to force an' ' override, that did not happen. You may want to check' ' your cloud, or remove the version specification from' ' your config.'.format( service_type=service_type, config_version=config_version, api_version='.'.join([str(f) for f in api_version]))) self.log.debug(warning_msg) warnings.warn(warning_msg) return adapter # TODO(shade) This should be replaced with using openstack Connection # object. def _get_raw_client( self, service_type, api_version=None, endpoint_override=None): return proxy._ShadeAdapter( session=self.session, service_type=self.config.get_service_type(service_type), service_name=self.config.get_service_name(service_type), interface=self.config.get_interface(service_type), endpoint_override=self.config.get_endpoint( service_type) or endpoint_override, region_name=self.config.get_region_name(service_type)) def _is_client_version(self, client, version): client_name = '_{client}_client'.format( client=client.replace('-', '_')) client = getattr(self, client_name) return client._version_matches(version) @property def _application_catalog_client(self): if 'application-catalog' not in self._raw_clients: self._raw_clients['application-catalog'] = self._get_raw_client( 'application-catalog') return self._raw_clients['application-catalog'] @property def _database_client(self): if 'database' not in self._raw_clients: self._raw_clients['database'] = self._get_raw_client('database') return self._raw_clients['database'] @property def _raw_image_client(self): if 'raw-image' not in self._raw_clients: image_client = self._get_raw_client('image') self._raw_clients['raw-image'] = image_client return self._raw_clients['raw-image'] def pprint(self, resource): """Wrapper around pprint that groks munch objects""" # import late since this is a utility function import pprint new_resource = _utils._dictify_resource(resource) pprint.pprint(new_resource) def pformat(self, resource): """Wrapper around pformat that groks munch objects""" # import late since this is a utility function import pprint new_resource = _utils._dictify_resource(resource) return pprint.pformat(new_resource) @property def _keystone_catalog(self): return self.session.auth.get_access(self.session).service_catalog @property def service_catalog(self): return self._keystone_catalog.catalog def endpoint_for(self, service_type, interface=None, region_name=None): """Return the endpoint for a given service. Respects config values for Connection, including ``*_endpoint_override``. For direct values from the catalog regardless of overrides, see :meth:`~openstack.config.cloud_region.CloudRegion.get_endpoint_from_catalog` :param service_type: Service Type of the endpoint to search for. :param interface: Interface of the endpoint to search for. Optional, defaults to the configured value for interface for this Connection. :param region_name: Region Name of the endpoint to search for. Optional, defaults to the configured value for region_name for this Connection. :returns: The endpoint of the service, or None if not found. """ endpoint_override = self.config.get_endpoint(service_type) if endpoint_override: return endpoint_override return self.config.get_endpoint_from_catalog( service_type=service_type, interface=interface, region_name=region_name) @property def auth_token(self): # Keystone's session will reuse a token if it is still valid. # We don't need to track validity here, just get_token() each time. return self.session.get_token() @property def current_user_id(self): """Get the id of the currently logged-in user from the token.""" return self.session.auth.get_access(self.session).user_id @property def current_project_id(self): """Get the current project ID. Returns the project_id of the current token scope. None means that the token is domain scoped or unscoped. :raises keystoneauth1.exceptions.auth.AuthorizationFailure: if a new token fetch fails. :raises keystoneauth1.exceptions.auth_plugins.MissingAuthPlugin: if a plugin is not available. """ return self.session.get_project_id() @property def current_project(self): """Return a ``munch.Munch`` describing the current project""" return self._get_project_info() def _get_project_info(self, project_id=None): project_info = munch.Munch( id=project_id, name=None, domain_id=None, domain_name=None, ) if not project_id or project_id == self.current_project_id: # If we don't have a project_id parameter, it means a user is # directly asking what the current state is. # Alternately, if we have one, that means we're calling this # from within a normalize function, which means the object has # a project_id associated with it. If the project_id matches # the project_id of our current token, that means we can supplement # the info with human readable info about names if we have them. # If they don't match, that means we're an admin who has pulled # an object from a different project, so adding info from the # current token would be wrong. auth_args = self.config.config.get('auth', {}) project_info['id'] = self.current_project_id project_info['name'] = auth_args.get('project_name') project_info['domain_id'] = auth_args.get('project_domain_id') project_info['domain_name'] = auth_args.get('project_domain_name') return project_info @property def current_location(self): """Return a ``munch.Munch`` explaining the current cloud location.""" return self._get_current_location() def _get_current_location(self, project_id=None, zone=None): return munch.Munch( cloud=self.name, # TODO(efried): This is wrong, but it only seems to be used in a # repr; can we get rid of it? region_name=self.config.get_region_name(), zone=zone, project=self._get_project_info(project_id), ) def _get_identity_location(self): '''Identity resources do not exist inside of projects.''' return munch.Munch( cloud=self.name, region_name=None, zone=None, project=munch.Munch( id=None, name=None, domain_id=None, domain_name=None)) def _get_project_id_param_dict(self, name_or_id): if name_or_id: project = self.get_project(name_or_id) if not project: return {} if self._is_client_version('identity', 3): return {'default_project_id': project['id']} else: return {'tenant_id': project['id']} else: return {} def _get_domain_id_param_dict(self, domain_id): """Get a useable domain.""" # Keystone v3 requires domains for user and project creation. v2 does # not. However, keystone v2 does not allow user creation by non-admin # users, so we can throw an error to the user that does not need to # mention api versions if self._is_client_version('identity', 3): if not domain_id: raise exc.OpenStackCloudException( "User or project creation requires an explicit" " domain_id argument.") else: return {'domain_id': domain_id} else: return {} def _get_identity_params(self, domain_id=None, project=None): """Get the domain and project/tenant parameters if needed. keystone v2 and v3 are divergent enough that we need to pass or not pass project or tenant_id or domain or nothing in a sane manner. """ ret = {} ret.update(self._get_domain_id_param_dict(domain_id)) ret.update(self._get_project_id_param_dict(project)) return ret def range_search(self, data, filters): """Perform integer range searches across a list of dictionaries. Given a list of dictionaries, search across the list using the given dictionary keys and a range of integer values for each key. Only dictionaries that match ALL search filters across the entire original data set will be returned. It is not a requirement that each dictionary contain the key used for searching. Those without the key will be considered non-matching. The range values must be string values and is either a set of digits representing an integer for matching, or a range operator followed by a set of digits representing an integer for matching. If a range operator is not given, exact value matching will be used. Valid operators are one of: <,>,<=,>= :param data: List of dictionaries to be searched. :param filters: Dict describing the one or more range searches to perform. If more than one search is given, the result will be the members of the original data set that match ALL searches. An example of filtering by multiple ranges:: {"vcpus": "<=5", "ram": "<=2048", "disk": "1"} :returns: A list subset of the original data set. :raises: OpenStackCloudException on invalid range expressions. """ filtered = [] for key, range_value in filters.items(): # We always want to operate on the full data set so that # calculations for minimum and maximum are correct. results = _utils.range_filter(data, key, range_value) if not filtered: # First set of results filtered = results else: # The combination of all searches should be the intersection of # all result sets from each search. So adjust the current set # of filtered data by computing its intersection with the # latest result set. filtered = [r for r in results for f in filtered if r == f] return filtered def _get_and_munchify(self, key, data): """Wrapper around meta.get_and_munchify. Some of the methods expect a `meta` attribute to be passed in as part of the method signature. In those methods the meta param is overriding the meta module making the call to meta.get_and_munchify to fail. """ if isinstance(data, requests.models.Response): data = proxy._json_response(data) return meta.get_and_munchify(key, data) def get_name(self): return self.name def get_session_endpoint(self, service_key): try: return self.config.get_session_endpoint(service_key) except keystoneauth1.exceptions.catalog.EndpointNotFound as e: self.log.debug( "Endpoint not found in %s cloud: %s", self.name, str(e)) endpoint = None except exc.OpenStackCloudException: raise except Exception as e: raise exc.OpenStackCloudException( "Error getting {service} endpoint on {cloud}:{region}:" " {error}".format( service=service_key, cloud=self.name, region=self.config.get_region_name(service_key), error=str(e))) return endpoint def has_service(self, service_key): if not self.config.has_service(service_key): # TODO(mordred) add a stamp here so that we only report this once if not (service_key in self._disable_warnings and self._disable_warnings[service_key]): self.log.debug( "Disabling %(service_key)s entry in catalog" " per config", {'service_key': service_key}) self._disable_warnings[service_key] = True return False try: endpoint = self.get_session_endpoint(service_key) except exc.OpenStackCloudException: return False if endpoint: return True else: return False def project_cleanup( self, dry_run=True, wait_timeout=120, status_queue=None, filters=None, resource_evaluation_fn=None ): """Cleanup the project resources. Cleanup all resources in all services, which provide cleanup methods. :param bool dry_run: Cleanup or only list identified resources. :param int wait_timeout: Maximum amount of time given to each service to comlete the cleanup. :param queue status_queue: a threading queue object used to get current process status. The queue contain processed resources. :param dict filters: Additional filters for the cleanup (only resources matching all filters will be deleted, if there are no other dependencies). :param resource_evaluation_fn: A callback function, which will be invoked for each resurce and must return True/False depending on whether resource need to be deleted or not. """ dependencies = {} get_dep_fn_name = '_get_cleanup_dependencies' cleanup_fn_name = '_service_cleanup' if not status_queue: status_queue = queue.Queue() for service in self.config.get_enabled_services(): if hasattr(self, service): proxy = getattr(self, service) if ( proxy and hasattr(proxy, get_dep_fn_name) and hasattr(proxy, cleanup_fn_name) ): deps = getattr(proxy, get_dep_fn_name)() if deps: dependencies.update(deps) dep_graph = utils.TinyDAG() for k, v in dependencies.items(): dep_graph.add_node(k) for dep in v['before']: dep_graph.add_node(dep) dep_graph.add_edge(k, dep) for dep in v.get('after', []): dep_graph.add_edge(dep, k) cleanup_resources = dict() for service in dep_graph.walk(timeout=wait_timeout): fn = None if hasattr(self, service): proxy = getattr(self, service) cleanup_fn = getattr(proxy, cleanup_fn_name, None) if cleanup_fn: fn = functools.partial( cleanup_fn, dry_run=dry_run, client_status_queue=status_queue, identified_resources=cleanup_resources, filters=filters, resource_evaluation_fn=resource_evaluation_fn ) if fn: self._pool_executor.submit( cleanup_task, dep_graph, service, fn ) else: dep_graph.node_done(service) for count in utils.iterate_timeout( timeout=wait_timeout, message="Timeout waiting for cleanup to finish", wait=1): if dep_graph.is_complete(): return def cleanup_task(graph, service, fn): try: fn() except Exception: log = _log.setup_logging('openstack.project_cleanup') log.exception('Error in the %s cleanup function' % service) finally: graph.node_done(service)