impuls/lib/python3.11/site-packages/openstack/tests/unit/cloud/test_caching.py

833 lines
27 KiB
Python
Raw Normal View History

# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import concurrent
import time
import testtools
from testscenarios import load_tests_apply_scenarios as load_tests # noqa
import openstack
from openstack.block_storage.v3 import volume as _volume
import openstack.cloud
from openstack.cloud import meta
from openstack.compute.v2 import flavor as _flavor
from openstack import exceptions
from openstack.identity.v3 import project as _project
from openstack.identity.v3 import user as _user
from openstack.image.v2 import image as _image
from openstack.network.v2 import port as _port
from openstack.tests import fakes
from openstack.tests.unit import base
from openstack.tests.unit.cloud import test_port
# Mock out the gettext function so that the task schema can be copypasta
def _(msg):
return msg
_TASK_PROPERTIES = {
"id": {
"description": _("An identifier for the task"),
"pattern": _(
'^([0-9a-fA-F]){8}-([0-9a-fA-F]){4}-([0-9a-fA-F]){4}'
'-([0-9a-fA-F]){4}-([0-9a-fA-F]){12}$'
),
"type": "string",
},
"type": {
"description": _("The type of task represented by this content"),
"enum": [
"import",
],
"type": "string",
},
"status": {
"description": _("The current status of this task"),
"enum": ["pending", "processing", "success", "failure"],
"type": "string",
},
"input": {
"description": _("The parameters required by task, JSON blob"),
"type": ["null", "object"],
},
"result": {
"description": _("The result of current task, JSON blob"),
"type": ["null", "object"],
},
"owner": {
"description": _("An identifier for the owner of this task"),
"type": "string",
},
"message": {
"description": _(
"Human-readable informative message only included"
" when appropriate (usually on failure)"
),
"type": "string",
},
"expires_at": {
"description": _(
"Datetime when this resource would be subject to removal"
),
"type": ["null", "string"],
},
"created_at": {
"description": _("Datetime when this resource was created"),
"type": "string",
},
"updated_at": {
"description": _("Datetime when this resource was updated"),
"type": "string",
},
'self': {'type': 'string'},
'schema': {'type': 'string'},
}
_TASK_SCHEMA = dict(
name='Task',
properties=_TASK_PROPERTIES,
additionalProperties=False,
)
class TestMemoryCache(base.TestCase):
def setUp(self):
super(TestMemoryCache, self).setUp(
cloud_config_fixture='clouds_cache.yaml'
)
def _compare_images(self, exp, real):
self.assertDictEqual(
_image.Image(**exp).to_dict(computed=False),
real.to_dict(computed=False),
)
def _compare_volumes(self, exp, real):
self.assertDictEqual(
_volume.Volume(**exp).to_dict(computed=False),
real.to_dict(computed=False),
)
def test_openstack_cloud(self):
self.assertIsInstance(self.cloud, openstack.connection.Connection)
def _compare_projects(self, exp, real):
self.assertDictEqual(
_project.Project(**exp).to_dict(computed=False),
real.to_dict(computed=False),
)
def _compare_users(self, exp, real):
self.assertDictEqual(
_user.User(**exp).to_dict(computed=False),
real.to_dict(computed=False),
)
def test_list_projects_v3(self):
project_one = self._get_project_data()
project_two = self._get_project_data()
project_list = [project_one, project_two]
first_response = {'projects': [project_one.json_response['project']]}
second_response = {
'projects': [p.json_response['project'] for p in project_list]
}
mock_uri = self.get_mock_url(
service_type='identity', resource='projects', base_url_append='v3'
)
self.register_uris(
[
dict(
method='GET',
uri=mock_uri,
status_code=200,
json=first_response,
),
dict(
method='GET',
uri=mock_uri,
status_code=200,
json=second_response,
),
]
)
for a, b in zip(
first_response['projects'], self.cloud.list_projects()
):
self._compare_projects(a, b)
# invalidate the list_projects cache
self.cloud.list_projects.invalidate(self.cloud)
for a, b in zip(
second_response['projects'], self.cloud.list_projects()
):
self._compare_projects(a, b)
self.assert_calls()
def test_list_servers_no_herd(self):
self.cloud._SERVER_AGE = 2
fake_server = fakes.make_fake_server('1234', 'name')
self.register_uris(
[
self.get_nova_discovery_mock_dict(),
dict(
method='GET',
uri=self.get_mock_url(
'compute', 'public', append=['servers', 'detail']
),
json={'servers': [fake_server]},
),
]
)
with concurrent.futures.ThreadPoolExecutor(16) as pool:
for i in range(16):
pool.submit(lambda: self.cloud.list_servers(bare=True))
# It's possible to race-condition 16 threads all in the
# single initial lock without a tiny sleep
time.sleep(0.001)
self.assert_calls()
def test_list_volumes(self):
fake_volume = fakes.FakeVolume(
'volume1', 'available', 'Volume 1 Display Name'
)
fake_volume_dict = meta.obj_to_munch(fake_volume)
fake_volume2 = fakes.FakeVolume(
'volume2', 'available', 'Volume 2 Display Name'
)
fake_volume2_dict = meta.obj_to_munch(fake_volume2)
self.register_uris(
[
self.get_cinder_discovery_mock_dict(),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', 'detail']
),
json={'volumes': [fake_volume_dict]},
),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', 'detail']
),
json={'volumes': [fake_volume_dict, fake_volume2_dict]},
),
]
)
for a, b in zip([fake_volume_dict], self.cloud.list_volumes()):
self._compare_volumes(a, b)
# this call should hit the cache
for a, b in zip([fake_volume_dict], self.cloud.list_volumes()):
self._compare_volumes(a, b)
self.cloud.list_volumes.invalidate(self.cloud)
for a, b in zip(
[fake_volume_dict, fake_volume2_dict], self.cloud.list_volumes()
):
self._compare_volumes(a, b)
self.assert_calls()
def test_list_volumes_creating_invalidates(self):
fake_volume = fakes.FakeVolume(
'volume1', 'creating', 'Volume 1 Display Name'
)
fake_volume_dict = meta.obj_to_munch(fake_volume)
fake_volume2 = fakes.FakeVolume(
'volume2', 'available', 'Volume 2 Display Name'
)
fake_volume2_dict = meta.obj_to_munch(fake_volume2)
self.register_uris(
[
self.get_cinder_discovery_mock_dict(),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', 'detail']
),
json={'volumes': [fake_volume_dict]},
),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', 'detail']
),
json={'volumes': [fake_volume_dict, fake_volume2_dict]},
),
]
)
for a, b in zip([fake_volume_dict], self.cloud.list_volumes()):
self._compare_volumes(a, b)
for a, b in zip(
[fake_volume_dict, fake_volume2_dict], self.cloud.list_volumes()
):
self._compare_volumes(a, b)
self.assert_calls()
def test_create_volume_invalidates(self):
fake_volb4 = meta.obj_to_munch(
fakes.FakeVolume('volume1', 'available', '')
)
_id = '12345'
fake_vol_creating = meta.obj_to_munch(
fakes.FakeVolume(_id, 'creating', '')
)
fake_vol_avail = meta.obj_to_munch(
fakes.FakeVolume(_id, 'available', '')
)
def now_deleting(request, context):
fake_vol_avail['status'] = 'deleting'
self.register_uris(
[
self.get_cinder_discovery_mock_dict(),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', 'detail']
),
json={'volumes': [fake_volb4]},
),
dict(
method='POST',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes']
),
json={'volume': fake_vol_creating},
),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', _id]
),
json={'volume': fake_vol_creating},
),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', _id]
),
json={'volume': fake_vol_avail},
),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', 'detail']
),
json={'volumes': [fake_volb4, fake_vol_avail]},
),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', _id]
),
json={'volume': fake_vol_avail},
),
dict(
method='DELETE',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', _id]
),
json=now_deleting,
),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', _id]
),
status_code=404,
),
dict(
method='GET',
uri=self.get_mock_url(
'volumev3', 'public', append=['volumes', 'detail']
),
json={'volumes': [fake_volb4, fake_vol_avail]},
),
]
)
for a, b in zip([fake_volb4], self.cloud.list_volumes()):
self._compare_volumes(a, b)
volume = dict(
display_name='junk_vol',
size=1,
display_description='test junk volume',
)
self.cloud.create_volume(wait=True, timeout=2, **volume)
# If cache was not invalidated, we would not see our own volume here
# because the first volume was available and thus would already be
# cached.
for a, b in zip(
[fake_volb4, fake_vol_avail], self.cloud.list_volumes()
):
self._compare_volumes(a, b)
self.cloud.delete_volume(_id)
# And now delete and check same thing since list is cached as all
# available
for a, b in zip([fake_volb4], self.cloud.list_volumes()):
self._compare_volumes(a, b)
self.assert_calls()
def test_list_users(self):
user_data = self._get_user_data(email='test@example.com')
self.register_uris(
[
dict(
method='GET',
uri=self.get_mock_url(
service_type='identity',
resource='users',
base_url_append='v3',
),
status_code=200,
json={'users': [user_data.json_response['user']]},
)
]
)
users = self.cloud.list_users()
self.assertEqual(1, len(users))
self.assertEqual(user_data.user_id, users[0]['id'])
self.assertEqual(user_data.name, users[0]['name'])
self.assertEqual(user_data.email, users[0]['email'])
self.assert_calls()
def test_modify_user_invalidates_cache(self):
self.use_keystone_v2()
user_data = self._get_user_data(email='test@example.com')
new_resp = {'user': user_data.json_response['user'].copy()}
new_resp['user']['email'] = 'Nope@Nope.Nope'
new_req = {'user': {'email': new_resp['user']['email']}}
mock_users_url = self.get_mock_url(
service_type='identity', interface='admin', resource='users'
)
mock_user_resource_url = self.get_mock_url(
service_type='identity',
interface='admin',
resource='users',
append=[user_data.user_id],
)
empty_user_list_resp = {'users': []}
users_list_resp = {'users': [user_data.json_response['user']]}
updated_users_list_resp = {'users': [new_resp['user']]}
# Password is None in the original create below
del user_data.json_request['user']['password']
uris_to_mock = [
# Inital User List is Empty
dict(
method='GET',
uri=mock_users_url,
status_code=200,
json=empty_user_list_resp,
),
# POST to create the user
# GET to get the user data after POST
dict(
method='POST',
uri=mock_users_url,
status_code=200,
json=user_data.json_response,
validate=dict(json=user_data.json_request),
),
# List Users Call
dict(
method='GET',
uri=mock_users_url,
status_code=200,
json=users_list_resp,
),
# List users to get ID for update
# Get user using user_id from list
# Update user
# Get updated user
dict(
method='GET',
uri=mock_users_url,
status_code=200,
json=users_list_resp,
),
dict(
method='PUT',
uri=mock_user_resource_url,
status_code=200,
json=new_resp,
validate=dict(json=new_req),
),
# List Users Call
dict(
method='GET',
uri=mock_users_url,
status_code=200,
json=updated_users_list_resp,
),
# List User to get ID for delete
# delete user
dict(
method='GET',
uri=mock_users_url,
status_code=200,
json=updated_users_list_resp,
),
dict(method='DELETE', uri=mock_user_resource_url, status_code=204),
# List Users Call (empty post delete)
dict(
method='GET',
uri=mock_users_url,
status_code=200,
json=empty_user_list_resp,
),
]
self.register_uris(uris_to_mock)
# first cache an empty list
self.assertEqual([], self.cloud.list_users())
# now add one
created = self.cloud.create_user(
name=user_data.name, email=user_data.email
)
self.assertEqual(user_data.user_id, created['id'])
self.assertEqual(user_data.name, created['name'])
self.assertEqual(user_data.email, created['email'])
# Cache should have been invalidated
users = self.cloud.list_users()
self.assertEqual(user_data.user_id, users[0]['id'])
self.assertEqual(user_data.name, users[0]['name'])
self.assertEqual(user_data.email, users[0]['email'])
# Update and check to see if it is updated
updated = self.cloud.update_user(
user_data.user_id, email=new_resp['user']['email']
)
self.assertEqual(user_data.user_id, updated.id)
self.assertEqual(user_data.name, updated.name)
self.assertEqual(new_resp['user']['email'], updated.email)
users = self.cloud.list_users()
self.assertEqual(1, len(users))
self.assertEqual(user_data.user_id, users[0]['id'])
self.assertEqual(user_data.name, users[0]['name'])
self.assertEqual(new_resp['user']['email'], users[0]['email'])
# Now delete and ensure it disappears
self.cloud.delete_user(user_data.user_id)
self.assertEqual([], self.cloud.list_users())
self.assert_calls()
def test_list_flavors(self):
mock_uri = '{endpoint}/flavors/detail?is_public=None'.format(
endpoint=fakes.COMPUTE_ENDPOINT
)
uris_to_mock = [
dict(
method='GET',
uri=mock_uri,
validate=dict(
headers={'OpenStack-API-Version': 'compute 2.53'}
),
json={'flavors': []},
),
dict(
method='GET',
uri=mock_uri,
validate=dict(
headers={'OpenStack-API-Version': 'compute 2.53'}
),
json={'flavors': fakes.FAKE_FLAVOR_LIST},
),
]
self.use_compute_discovery()
self.register_uris(uris_to_mock)
self.assertEqual([], self.cloud.list_flavors())
self.assertEqual([], self.cloud.list_flavors())
self.cloud.list_flavors.invalidate(self.cloud)
self.assertResourceListEqual(
self.cloud.list_flavors(), fakes.FAKE_FLAVOR_LIST, _flavor.Flavor
)
self.assert_calls()
def test_list_images(self):
self.use_glance()
fake_image = fakes.make_fake_image(image_id='42')
self.register_uris(
[
dict(
method='GET',
uri=self.get_mock_url(
'image', 'public', append=['v2', 'images']
),
json={'images': []},
),
dict(
method='GET',
uri=self.get_mock_url(
'image', 'public', append=['v2', 'images']
),
json={'images': [fake_image]},
),
]
)
self.assertEqual([], self.cloud.list_images())
self.assertEqual([], self.cloud.list_images())
self.cloud.list_images.invalidate(self.cloud)
[
self._compare_images(a, b)
for a, b in zip([fake_image], self.cloud.list_images())
]
self.assert_calls()
def test_list_images_caches_deleted_status(self):
self.use_glance()
deleted_image_id = self.getUniqueString()
deleted_image = fakes.make_fake_image(
image_id=deleted_image_id, status='deleted'
)
active_image_id = self.getUniqueString()
active_image = fakes.make_fake_image(image_id=active_image_id)
list_return = {'images': [active_image, deleted_image]}
self.register_uris(
[
dict(
method='GET',
uri='https://image.example.com/v2/images',
json=list_return,
),
]
)
[
self._compare_images(a, b)
for a, b in zip([active_image], self.cloud.list_images())
]
[
self._compare_images(a, b)
for a, b in zip([active_image], self.cloud.list_images())
]
# We should only have one call
self.assert_calls()
def test_cache_no_cloud_name(self):
self.use_glance()
self.cloud.name = None
fi = fakes.make_fake_image(image_id=self.getUniqueString())
fi2 = fakes.make_fake_image(image_id=self.getUniqueString())
self.register_uris(
[
dict(
method='GET',
uri='https://image.example.com/v2/images',
json={'images': [fi]},
),
dict(
method='GET',
uri='https://image.example.com/v2/images',
json={'images': [fi, fi2]},
),
]
)
[
self._compare_images(a, b)
for a, b in zip([fi], self.cloud.list_images())
]
# Now test that the list was cached
[
self._compare_images(a, b)
for a, b in zip([fi], self.cloud.list_images())
]
# Invalidation too
self.cloud.list_images.invalidate(self.cloud)
[
self._compare_images(a, b)
for a, b in zip([fi, fi2], self.cloud.list_images())
]
def test_list_ports_filtered(self):
down_port = test_port.TestPort.mock_neutron_port_create_rep['port']
active_port = down_port.copy()
active_port['status'] = 'ACTIVE'
# We're testing to make sure a query string is passed when we're
# caching (cache by url), and that the results are still filtered.
self.register_uris(
[
dict(
method='GET',
uri=self.get_mock_url(
'network',
'public',
append=['v2.0', 'ports'],
qs_elements=['status=DOWN'],
),
json={
'ports': [
down_port,
active_port,
]
},
),
]
)
ports = self.cloud.list_ports(filters={'status': 'DOWN'})
for a, b in zip([down_port], ports):
self.assertDictEqual(
_port.Port(**a).to_dict(computed=False),
b.to_dict(computed=False),
)
self.assert_calls()
class TestCacheIgnoresQueuedStatus(base.TestCase):
scenarios = [
('queued', dict(status='queued')),
('saving', dict(status='saving')),
('pending_delete', dict(status='pending_delete')),
]
def setUp(self):
super(TestCacheIgnoresQueuedStatus, self).setUp(
cloud_config_fixture='clouds_cache.yaml'
)
self.use_glance()
active_image_id = self.getUniqueString()
self.active_image = fakes.make_fake_image(
image_id=active_image_id, status=self.status
)
self.active_list_return = {'images': [self.active_image]}
steady_image_id = self.getUniqueString()
self.steady_image = fakes.make_fake_image(image_id=steady_image_id)
self.steady_list_return = {
'images': [self.active_image, self.steady_image]
}
def _compare_images(self, exp, real):
self.assertDictEqual(
_image.Image(**exp).to_dict(computed=False),
real.to_dict(computed=False),
)
def test_list_images_ignores_pending_status(self):
self.register_uris(
[
dict(
method='GET',
uri='https://image.example.com/v2/images',
json=self.active_list_return,
),
dict(
method='GET',
uri='https://image.example.com/v2/images',
json=self.steady_list_return,
),
]
)
[
self._compare_images(a, b)
for a, b in zip([self.active_image], self.cloud.list_images())
]
# Should expect steady_image to appear if active wasn't cached
[
self._compare_images(a, b)
for a, b in zip(
[self.active_image, self.steady_image],
self.cloud.list_images(),
)
]
class TestCacheSteadyStatus(base.TestCase):
scenarios = [
('active', dict(status='active')),
('killed', dict(status='killed')),
]
def setUp(self):
super(TestCacheSteadyStatus, self).setUp(
cloud_config_fixture='clouds_cache.yaml'
)
self.use_glance()
active_image_id = self.getUniqueString()
self.active_image = fakes.make_fake_image(
image_id=active_image_id, status=self.status
)
self.active_list_return = {'images': [self.active_image]}
def _compare_images(self, exp, real):
self.assertDictEqual(
_image.Image(**exp).to_dict(computed=False),
real.to_dict(computed=False),
)
def test_list_images_caches_steady_status(self):
self.register_uris(
[
dict(
method='GET',
uri='https://image.example.com/v2/images',
json=self.active_list_return,
),
]
)
[
self._compare_images(a, b)
for a, b in zip([self.active_image], self.cloud.list_images())
]
[
self._compare_images(a, b)
for a, b in zip([self.active_image], self.cloud.list_images())
]
# We should only have one call
self.assert_calls()
class TestBogusAuth(base.TestCase):
def setUp(self):
super(TestBogusAuth, self).setUp(
cloud_config_fixture='clouds_cache.yaml'
)
def test_get_auth_bogus(self):
with testtools.ExpectedException(exceptions.ConfigException):
openstack.connect(cloud='_bogus_test_', config=self.config)