# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import concurrent import time import testtools from testscenarios import load_tests_apply_scenarios as load_tests # noqa import openstack from openstack.block_storage.v3 import volume as _volume import openstack.cloud from openstack.cloud import meta from openstack.compute.v2 import flavor as _flavor from openstack import exceptions from openstack.identity.v3 import project as _project from openstack.identity.v3 import user as _user from openstack.image.v2 import image as _image from openstack.network.v2 import port as _port from openstack.tests import fakes from openstack.tests.unit import base from openstack.tests.unit.cloud import test_port # Mock out the gettext function so that the task schema can be copypasta def _(msg): return msg _TASK_PROPERTIES = { "id": { "description": _("An identifier for the task"), "pattern": _( '^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}' '-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$' ), "type": "string", }, "type": { "description": _("The type of task represented by this content"), "enum": [ "import", ], "type": "string", }, "status": { "description": _("The current status of this task"), "enum": ["pending", "processing", "success", "failure"], "type": "string", }, "input": { "description": _("The parameters required by task, JSON blob"), "type": ["null", "object"], }, "result": { "description": _("The result of current task, JSON blob"), "type": ["null", "object"], }, "owner": { "description": _("An identifier for the owner of this task"), "type": "string", }, "message": { "description": _( "Human-readable informative message only included" " when appropriate (usually on failure)" ), "type": "string", }, "expires_at": { "description": _( "Datetime when this resource would be subject to removal" ), "type": ["null", "string"], }, "created_at": { "description": _("Datetime when this resource was created"), "type": "string", }, "updated_at": { "description": _("Datetime when this resource was updated"), "type": "string", }, 'self': {'type': 'string'}, 'schema': {'type': 'string'}, } _TASK_SCHEMA = dict( name='Task', properties=_TASK_PROPERTIES, additionalProperties=False, ) class TestMemoryCache(base.TestCase): def setUp(self): super(TestMemoryCache, self).setUp( cloud_config_fixture='clouds_cache.yaml' ) def _compare_images(self, exp, real): self.assertDictEqual( _image.Image(**exp).to_dict(computed=False), real.to_dict(computed=False), ) def _compare_volumes(self, exp, real): self.assertDictEqual( _volume.Volume(**exp).to_dict(computed=False), real.to_dict(computed=False), ) def test_openstack_cloud(self): self.assertIsInstance(self.cloud, openstack.connection.Connection) def _compare_projects(self, exp, real): self.assertDictEqual( _project.Project(**exp).to_dict(computed=False), real.to_dict(computed=False), ) def _compare_users(self, exp, real): self.assertDictEqual( _user.User(**exp).to_dict(computed=False), real.to_dict(computed=False), ) def test_list_projects_v3(self): project_one = self._get_project_data() project_two = self._get_project_data() project_list = [project_one, project_two] first_response = {'projects': [project_one.json_response['project']]} second_response = { 'projects': [p.json_response['project'] for p in project_list] } mock_uri = self.get_mock_url( service_type='identity', resource='projects', base_url_append='v3' ) self.register_uris( [ dict( method='GET', uri=mock_uri, status_code=200, json=first_response, ), dict( method='GET', uri=mock_uri, status_code=200, json=second_response, ), ] ) for a, b in zip( first_response['projects'], self.cloud.list_projects() ): self._compare_projects(a, b) # invalidate the list_projects cache self.cloud.list_projects.invalidate(self.cloud) for a, b in zip( second_response['projects'], self.cloud.list_projects() ): self._compare_projects(a, b) self.assert_calls() def test_list_servers_no_herd(self): self.cloud._SERVER_AGE = 2 fake_server = fakes.make_fake_server('1234', 'name') self.register_uris( [ self.get_nova_discovery_mock_dict(), dict( method='GET', uri=self.get_mock_url( 'compute', 'public', append=['servers', 'detail'] ), json={'servers': [fake_server]}, ), ] ) with concurrent.futures.ThreadPoolExecutor(16) as pool: for i in range(16): pool.submit(lambda: self.cloud.list_servers(bare=True)) # It's possible to race-condition 16 threads all in the # single initial lock without a tiny sleep time.sleep(0.001) self.assert_calls() def test_list_volumes(self): fake_volume = fakes.FakeVolume( 'volume1', 'available', 'Volume 1 Display Name' ) fake_volume_dict = meta.obj_to_munch(fake_volume) fake_volume2 = fakes.FakeVolume( 'volume2', 'available', 'Volume 2 Display Name' ) fake_volume2_dict = meta.obj_to_munch(fake_volume2) self.register_uris( [ self.get_cinder_discovery_mock_dict(), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', 'detail'] ), json={'volumes': [fake_volume_dict]}, ), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', 'detail'] ), json={'volumes': [fake_volume_dict, fake_volume2_dict]}, ), ] ) for a, b in zip([fake_volume_dict], self.cloud.list_volumes()): self._compare_volumes(a, b) # this call should hit the cache for a, b in zip([fake_volume_dict], self.cloud.list_volumes()): self._compare_volumes(a, b) self.cloud.list_volumes.invalidate(self.cloud) for a, b in zip( [fake_volume_dict, fake_volume2_dict], self.cloud.list_volumes() ): self._compare_volumes(a, b) self.assert_calls() def test_list_volumes_creating_invalidates(self): fake_volume = fakes.FakeVolume( 'volume1', 'creating', 'Volume 1 Display Name' ) fake_volume_dict = meta.obj_to_munch(fake_volume) fake_volume2 = fakes.FakeVolume( 'volume2', 'available', 'Volume 2 Display Name' ) fake_volume2_dict = meta.obj_to_munch(fake_volume2) self.register_uris( [ self.get_cinder_discovery_mock_dict(), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', 'detail'] ), json={'volumes': [fake_volume_dict]}, ), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', 'detail'] ), json={'volumes': [fake_volume_dict, fake_volume2_dict]}, ), ] ) for a, b in zip([fake_volume_dict], self.cloud.list_volumes()): self._compare_volumes(a, b) for a, b in zip( [fake_volume_dict, fake_volume2_dict], self.cloud.list_volumes() ): self._compare_volumes(a, b) self.assert_calls() def test_create_volume_invalidates(self): fake_volb4 = meta.obj_to_munch( fakes.FakeVolume('volume1', 'available', '') ) _id = '12345' fake_vol_creating = meta.obj_to_munch( fakes.FakeVolume(_id, 'creating', '') ) fake_vol_avail = meta.obj_to_munch( fakes.FakeVolume(_id, 'available', '') ) def now_deleting(request, context): fake_vol_avail['status'] = 'deleting' self.register_uris( [ self.get_cinder_discovery_mock_dict(), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', 'detail'] ), json={'volumes': [fake_volb4]}, ), dict( method='POST', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes'] ), json={'volume': fake_vol_creating}, ), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', _id] ), json={'volume': fake_vol_creating}, ), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', _id] ), json={'volume': fake_vol_avail}, ), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', 'detail'] ), json={'volumes': [fake_volb4, fake_vol_avail]}, ), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', _id] ), json={'volume': fake_vol_avail}, ), dict( method='DELETE', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', _id] ), json=now_deleting, ), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', _id] ), status_code=404, ), dict( method='GET', uri=self.get_mock_url( 'volumev3', 'public', append=['volumes', 'detail'] ), json={'volumes': [fake_volb4, fake_vol_avail]}, ), ] ) for a, b in zip([fake_volb4], self.cloud.list_volumes()): self._compare_volumes(a, b) volume = dict( display_name='junk_vol', size=1, display_description='test junk volume', ) self.cloud.create_volume(wait=True, timeout=2, **volume) # If cache was not invalidated, we would not see our own volume here # because the first volume was available and thus would already be # cached. for a, b in zip( [fake_volb4, fake_vol_avail], self.cloud.list_volumes() ): self._compare_volumes(a, b) self.cloud.delete_volume(_id) # And now delete and check same thing since list is cached as all # available for a, b in zip([fake_volb4], self.cloud.list_volumes()): self._compare_volumes(a, b) self.assert_calls() def test_list_users(self): user_data = self._get_user_data(email='test@example.com') self.register_uris( [ dict( method='GET', uri=self.get_mock_url( service_type='identity', resource='users', base_url_append='v3', ), status_code=200, json={'users': [user_data.json_response['user']]}, ) ] ) users = self.cloud.list_users() self.assertEqual(1, len(users)) self.assertEqual(user_data.user_id, users[0]['id']) self.assertEqual(user_data.name, users[0]['name']) self.assertEqual(user_data.email, users[0]['email']) self.assert_calls() def test_modify_user_invalidates_cache(self): self.use_keystone_v2() user_data = self._get_user_data(email='test@example.com') new_resp = {'user': user_data.json_response['user'].copy()} new_resp['user']['email'] = 'Nope@Nope.Nope' new_req = {'user': {'email': new_resp['user']['email']}} mock_users_url = self.get_mock_url( service_type='identity', interface='admin', resource='users' ) mock_user_resource_url = self.get_mock_url( service_type='identity', interface='admin', resource='users', append=[user_data.user_id], ) empty_user_list_resp = {'users': []} users_list_resp = {'users': [user_data.json_response['user']]} updated_users_list_resp = {'users': [new_resp['user']]} # Password is None in the original create below del user_data.json_request['user']['password'] uris_to_mock = [ # Inital User List is Empty dict( method='GET', uri=mock_users_url, status_code=200, json=empty_user_list_resp, ), # POST to create the user # GET to get the user data after POST dict( method='POST', uri=mock_users_url, status_code=200, json=user_data.json_response, validate=dict(json=user_data.json_request), ), # List Users Call dict( method='GET', uri=mock_users_url, status_code=200, json=users_list_resp, ), # List users to get ID for update # Get user using user_id from list # Update user # Get updated user dict( method='GET', uri=mock_users_url, status_code=200, json=users_list_resp, ), dict( method='PUT', uri=mock_user_resource_url, status_code=200, json=new_resp, validate=dict(json=new_req), ), # List Users Call dict( method='GET', uri=mock_users_url, status_code=200, json=updated_users_list_resp, ), # List User to get ID for delete # delete user dict( method='GET', uri=mock_users_url, status_code=200, json=updated_users_list_resp, ), dict(method='DELETE', uri=mock_user_resource_url, status_code=204), # List Users Call (empty post delete) dict( method='GET', uri=mock_users_url, status_code=200, json=empty_user_list_resp, ), ] self.register_uris(uris_to_mock) # first cache an empty list self.assertEqual([], self.cloud.list_users()) # now add one created = self.cloud.create_user( name=user_data.name, email=user_data.email ) self.assertEqual(user_data.user_id, created['id']) self.assertEqual(user_data.name, created['name']) self.assertEqual(user_data.email, created['email']) # Cache should have been invalidated users = self.cloud.list_users() self.assertEqual(user_data.user_id, users[0]['id']) self.assertEqual(user_data.name, users[0]['name']) self.assertEqual(user_data.email, users[0]['email']) # Update and check to see if it is updated updated = self.cloud.update_user( user_data.user_id, email=new_resp['user']['email'] ) self.assertEqual(user_data.user_id, updated.id) self.assertEqual(user_data.name, updated.name) self.assertEqual(new_resp['user']['email'], updated.email) users = self.cloud.list_users() self.assertEqual(1, len(users)) self.assertEqual(user_data.user_id, users[0]['id']) self.assertEqual(user_data.name, users[0]['name']) self.assertEqual(new_resp['user']['email'], users[0]['email']) # Now delete and ensure it disappears self.cloud.delete_user(user_data.user_id) self.assertEqual([], self.cloud.list_users()) self.assert_calls() def test_list_flavors(self): mock_uri = '{endpoint}/flavors/detail?is_public=None'.format( endpoint=fakes.COMPUTE_ENDPOINT ) uris_to_mock = [ dict( method='GET', uri=mock_uri, validate=dict( headers={'OpenStack-API-Version': 'compute 2.53'} ), json={'flavors': []}, ), dict( method='GET', uri=mock_uri, validate=dict( headers={'OpenStack-API-Version': 'compute 2.53'} ), json={'flavors': fakes.FAKE_FLAVOR_LIST}, ), ] self.use_compute_discovery() self.register_uris(uris_to_mock) self.assertEqual([], self.cloud.list_flavors()) self.assertEqual([], self.cloud.list_flavors()) self.cloud.list_flavors.invalidate(self.cloud) self.assertResourceListEqual( self.cloud.list_flavors(), fakes.FAKE_FLAVOR_LIST, _flavor.Flavor ) self.assert_calls() def test_list_images(self): self.use_glance() fake_image = fakes.make_fake_image(image_id='42') self.register_uris( [ dict( method='GET', uri=self.get_mock_url( 'image', 'public', append=['v2', 'images'] ), json={'images': []}, ), dict( method='GET', uri=self.get_mock_url( 'image', 'public', append=['v2', 'images'] ), json={'images': [fake_image]}, ), ] ) self.assertEqual([], self.cloud.list_images()) self.assertEqual([], self.cloud.list_images()) self.cloud.list_images.invalidate(self.cloud) [ self._compare_images(a, b) for a, b in zip([fake_image], self.cloud.list_images()) ] self.assert_calls() def test_list_images_caches_deleted_status(self): self.use_glance() deleted_image_id = self.getUniqueString() deleted_image = fakes.make_fake_image( image_id=deleted_image_id, status='deleted' ) active_image_id = self.getUniqueString() active_image = fakes.make_fake_image(image_id=active_image_id) list_return = {'images': [active_image, deleted_image]} self.register_uris( [ dict( method='GET', uri='https://image.example.com/v2/images', json=list_return, ), ] ) [ self._compare_images(a, b) for a, b in zip([active_image], self.cloud.list_images()) ] [ self._compare_images(a, b) for a, b in zip([active_image], self.cloud.list_images()) ] # We should only have one call self.assert_calls() def test_cache_no_cloud_name(self): self.use_glance() self.cloud.name = None fi = fakes.make_fake_image(image_id=self.getUniqueString()) fi2 = fakes.make_fake_image(image_id=self.getUniqueString()) self.register_uris( [ dict( method='GET', uri='https://image.example.com/v2/images', json={'images': [fi]}, ), dict( method='GET', uri='https://image.example.com/v2/images', json={'images': [fi, fi2]}, ), ] ) [ self._compare_images(a, b) for a, b in zip([fi], self.cloud.list_images()) ] # Now test that the list was cached [ self._compare_images(a, b) for a, b in zip([fi], self.cloud.list_images()) ] # Invalidation too self.cloud.list_images.invalidate(self.cloud) [ self._compare_images(a, b) for a, b in zip([fi, fi2], self.cloud.list_images()) ] def test_list_ports_filtered(self): down_port = test_port.TestPort.mock_neutron_port_create_rep['port'] active_port = down_port.copy() active_port['status'] = 'ACTIVE' # We're testing to make sure a query string is passed when we're # caching (cache by url), and that the results are still filtered. self.register_uris( [ dict( method='GET', uri=self.get_mock_url( 'network', 'public', append=['v2.0', 'ports'], qs_elements=['status=DOWN'], ), json={ 'ports': [ down_port, active_port, ] }, ), ] ) ports = self.cloud.list_ports(filters={'status': 'DOWN'}) for a, b in zip([down_port], ports): self.assertDictEqual( _port.Port(**a).to_dict(computed=False), b.to_dict(computed=False), ) self.assert_calls() class TestCacheIgnoresQueuedStatus(base.TestCase): scenarios = [ ('queued', dict(status='queued')), ('saving', dict(status='saving')), ('pending_delete', dict(status='pending_delete')), ] def setUp(self): super(TestCacheIgnoresQueuedStatus, self).setUp( cloud_config_fixture='clouds_cache.yaml' ) self.use_glance() active_image_id = self.getUniqueString() self.active_image = fakes.make_fake_image( image_id=active_image_id, status=self.status ) self.active_list_return = {'images': [self.active_image]} steady_image_id = self.getUniqueString() self.steady_image = fakes.make_fake_image(image_id=steady_image_id) self.steady_list_return = { 'images': [self.active_image, self.steady_image] } def _compare_images(self, exp, real): self.assertDictEqual( _image.Image(**exp).to_dict(computed=False), real.to_dict(computed=False), ) def test_list_images_ignores_pending_status(self): self.register_uris( [ dict( method='GET', uri='https://image.example.com/v2/images', json=self.active_list_return, ), dict( method='GET', uri='https://image.example.com/v2/images', json=self.steady_list_return, ), ] ) [ self._compare_images(a, b) for a, b in zip([self.active_image], self.cloud.list_images()) ] # Should expect steady_image to appear if active wasn't cached [ self._compare_images(a, b) for a, b in zip( [self.active_image, self.steady_image], self.cloud.list_images(), ) ] class TestCacheSteadyStatus(base.TestCase): scenarios = [ ('active', dict(status='active')), ('killed', dict(status='killed')), ] def setUp(self): super(TestCacheSteadyStatus, self).setUp( cloud_config_fixture='clouds_cache.yaml' ) self.use_glance() active_image_id = self.getUniqueString() self.active_image = fakes.make_fake_image( image_id=active_image_id, status=self.status ) self.active_list_return = {'images': [self.active_image]} def _compare_images(self, exp, real): self.assertDictEqual( _image.Image(**exp).to_dict(computed=False), real.to_dict(computed=False), ) def test_list_images_caches_steady_status(self): self.register_uris( [ dict( method='GET', uri='https://image.example.com/v2/images', json=self.active_list_return, ), ] ) [ self._compare_images(a, b) for a, b in zip([self.active_image], self.cloud.list_images()) ] [ self._compare_images(a, b) for a, b in zip([self.active_image], self.cloud.list_images()) ] # We should only have one call self.assert_calls() class TestBogusAuth(base.TestCase): def setUp(self): super(TestBogusAuth, self).setUp( cloud_config_fixture='clouds_cache.yaml' ) def test_get_auth_bogus(self): with testtools.ExpectedException(exceptions.ConfigException): openstack.connect(cloud='_bogus_test_', config=self.config)