UnisKB/deploy_artifacts/tests/test_upgrade.py

254 lines
15 KiB
Python

import copy
import importlib.util
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import Mock, patch
SPEC = importlib.util.spec_from_file_location('upgrade', Path(__file__).resolve().parents[1] / 'upgrade.py')
upgrade = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(upgrade)
CHECK_SPEC = importlib.util.spec_from_file_location('internal_check', Path(__file__).resolve().parents[1] / 'check_internal_services.py')
internal_check = importlib.util.module_from_spec(CHECK_SPEC)
CHECK_SPEC.loader.exec_module(internal_check)
def containers():
result = {}
for name in upgrade.LEGACY:
result[name] = {'Id': 'old-' + name, 'Image': 'sha256:old', 'Name': '/' + name,
'State': {'Status': 'exited', 'Running': False},
'Config': {'Env': [], 'Labels': {}}, 'Mounts': [],
'HostConfig': {'PortBindings': {}}}
result['airag']['Config']['Env'] = [
'MAXKB_DB_HOST=127.0.0.1', 'MAXKB_DB_PASSWORD=literal$secret=${VALUE}',
'MAXKB_REDIS_HOST=127.0.0.1', 'MAXKB_VERSION=old', 'PATH=/old/python',
'PGDATA=/opt/maxkb/data/postgresql/pgdata', 'POSTGRES_PASSWORD=literal$pg',
]
result['mineru-api']['Config']['Env'] = ['CUDA_VISIBLE_DEVICES=1', 'MINERU_GPU_MEMORY_UTILIZATION=0.3', 'PYTHONPATH=/old/vendor', 'MINDMAP_LLM_API_KEY=legacy-private']
for name, port in (('unis-gateway', 8989), ('mindmap-web', 9898)):
result[name]['HostConfig']['PortBindings'][str(port) + '/tcp'] = [{'HostIp': '127.0.0.1', 'HostPort': str(port + 100)}]
return result
class UpgradeTests(unittest.TestCase):
def config(self, flavor='cuda'):
return upgrade.make_compose(containers(), {'kb': 'sha256:kb', 'parser': 'sha256:parser', 'gateway': 'sha256:gateway'}, Path('/srv/unis/clone'), Path('/srv/unis/models'), flavor, 'release-test')
def test_cuda_and_corex_have_separate_device_configuration(self):
cuda = self.config()['services']['mineru-api']
corex = self.config('corex')['services']['mineru-api']
self.assertEqual(cuda['deploy']['resources']['reservations']['devices'][0]['driver'], 'nvidia')
self.assertNotIn('privileged', cuda)
self.assertNotIn('runtime', corex)
self.assertNotIn('deploy', corex)
self.assertTrue(corex['privileged'])
self.assertIn('corex-4.4.0', corex['environment']['PYTHONPATH'])
self.assertEqual(cuda['environment']['CUDA_VISIBLE_DEVICES'], '1')
self.assertEqual(cuda['environment']['MINERU_GPU_MEMORY_UTILIZATION'], '0.3')
def test_worker_shares_local_db_and_durable_assets(self):
config = self.config()
kb = config['services']['airag']
worker = config['services']['hub-worker']
self.assertEqual(worker['network_mode'], 'service:airag')
self.assertEqual(worker['environment'], kb['environment'])
self.assertEqual(worker['volumes'], kb['volumes'])
self.assertEqual(worker['image'], kb['image'])
self.assertEqual(worker['environment']['AI_HUB_ASSET_DIR'], '/opt/maxkb/data/ai-hub-assets')
self.assertNotIn('--recover', worker['command'])
self.assertNotIn('MAXKB_VERSION', kb['environment'])
self.assertNotIn('PATH', kb['environment'])
self.assertEqual(kb['command'], ['/usr/bin/start-all.sh'])
self.assertEqual(kb['environment']['MAXKB_DB_PORT'], '5432')
self.assertEqual(kb['environment']['MAXKB_REDIS_PORT'], '6379')
self.assertNotIn('postgres', config['services'])
self.assertNotIn('redis', config['services'])
def test_external_and_sentinel_settings_cannot_enter_internal_deployment(self):
for field, value in (('MAXKB_DB_HOST', 'ares-pg'), ('MAXKB_REDIS_HOST', 'ares-redis'),
('MAXKB_DB_PORT', '5433'), ('MAXKB_REDIS_PORT', '6380'),
('MAXKB_REDIS_SENTINEL_SENTINELS', 'sentinel:26379')):
rows = containers()
env = upgrade.environment(rows['airag'])
env[field] = value
with self.subTest(field=field), self.assertRaises(ValueError):
upgrade.require_internal_services(env)
def service_clients(self):
db = Mock()
db.settings_dict = {'HOST': '127.0.0.1', 'PORT': '5432'}
cursor = db.cursor.return_value.__enter__ = Mock(return_value=Mock())
db.cursor.return_value.__exit__ = Mock(return_value=False)
cursor.return_value.fetchone.return_value = ('/opt/maxkb/data/postgresql/pgdata', '127.0.0.1', 5432, '17.6')
cache = Mock()
cache.connection_pool = SimpleNamespace(connection_kwargs={'host': '127.0.0.1', 'port': 6379})
cache.ping.return_value = True
cache.config_get.return_value = {'dir': '/opt/maxkb/data/redis'}
cache.info.return_value = {'redis_version': '7.0.15'}
return db, cache
def test_runtime_check_uses_actual_db_and_cache_configuration(self):
db, cache = self.service_clients()
report = internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata')
self.assertEqual(report['postgresql']['host'], '127.0.0.1')
self.assertTrue(report['redis']['ping'])
query = db.cursor.return_value.__enter__.return_value.execute.call_args.args[0]
self.assertIn('data_directory', query)
self.assertNotIn('UPDATE', query)
def test_runtime_check_rejects_effective_external_connection(self):
db, cache = self.service_clients()
db.settings_dict['HOST'] = 'ares-pg'
with self.assertRaisesRegex(ValueError, 'internal PostgreSQL'):
internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata')
db.settings_dict['HOST'] = '127.0.0.1'
cache.connection_pool.connection_kwargs['host'] = 'ares-redis'
with self.assertRaisesRegex(ValueError, 'internal Redis'):
internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata')
def test_runtime_check_rejects_wrong_server_data_directory(self):
db, cache = self.service_clients()
cache.config_get.return_value = {'dir': '/tmp/redis'}
with self.assertRaisesRegex(ValueError, 'persistent MaxKB mount'):
internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata')
cache.config_get.return_value = {'dir': '/opt/maxkb/data/redis'}
with self.assertRaisesRegex(ValueError, 'server path'):
internal_check.verify(db, cache, '/var/lib/postgresql/data')
def test_models_readonly_and_original_data_never_used_by_candidate(self):
config = self.config()
for service in config['services'].values():
for volume in service.get('volumes', []):
self.assertNotEqual(volume['source'], '/srv/unis/original')
if volume['target'] == '/models':
self.assertTrue(volume['read_only'])
self.assertFalse(volume['bind']['create_host_path'])
self.assertEqual(config['networks']['default']['name'], 'release-test')
self.assertNotIn('mindmap-web', config['services'])
def test_credentials_are_escaped_for_compose_and_not_sent_to_parser(self):
config = self.config()
escaped = upgrade.escape_compose(config)
value = escaped['services']['airag']['environment']['MAXKB_DB_PASSWORD']
self.assertEqual(value, 'literal$$secret=$${VALUE}')
self.assertNotIn('MINDMAP_LLM_API_KEY', config['services']['mineru-api']['environment'])
self.assertEqual(config['services']['airag']['environment']['MAXKB_DB_PASSWORD'], 'literal$secret=${VALUE}')
def test_existing_public_bindings_preserved(self):
ports = self.config()['services']['gateway']['ports']
self.assertEqual(ports[0]['published'], '9089')
self.assertEqual(ports[1]['published'], '9998')
self.assertTrue(all(p['host_ip'] == '127.0.0.1' for p in ports))
def test_running_or_paused_legacy_container_refused(self):
for status in ('running', 'paused', 'restarting', 'dead'):
with self.subTest(status=status):
rows = containers()
rows['airag']['State']['Status'] = status
with self.assertRaises(ValueError):
upgrade.ensure_stopped(rows)
def test_prepare_rejects_running_container_before_creating_files(self):
rows = containers()
rows['airag']['State']['Status'] = 'running'
with patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), Path('/srv/unis/models'))), patch.object(upgrade, 'run') as runner:
with self.assertRaises(ValueError):
upgrade.prepare(None)
runner.assert_not_called()
def test_rollback_refuses_unrelated_container(self):
state = {'phase': 'failed', 'project': 'this-release', 'old': containers(), 'renamed': list(upgrade.LEGACY)}
stranger = {'Config': {'Labels': {'com.docker.compose.project': 'some-other-release'}}, 'Id': 'unrelated'}
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inspect', return_value=stranger), patch.object(upgrade, 'run', return_value='unis-hub-worker\n') as runner:
with self.assertRaises(ValueError):
upgrade.rollback(Path('/srv/unis/release'))
self.assertEqual(runner.call_count, 1)
def test_rollback_keeps_new_data_and_restores_original_stopped_state(self):
rows = containers()
state = {'phase': 'failed', 'project': 'this-release', 'old': rows, 'renamed': list(upgrade.LEGACY)}
def read(name):
if name.startswith('old-'):
row = copy.deepcopy(rows[name[4:]])
row['Name'] += '-before-this-release'
return row
return {'Config': {'Labels': {'com.docker.compose.project': 'this-release'}}, 'Id': 'new-' + name}
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inspect', side_effect=read), patch.object(upgrade, 'save'), patch.object(upgrade, 'run', return_value='\n'.join(upgrade.NEW)) as runner:
upgrade.rollback(Path('/srv/unis/release'))
commands = [call.args for call in runner.call_args_list]
self.assertFalse(any('start' in command or 'rm' in command for command in commands))
stops = [command[-1] for command in commands if command[:2] == ('docker', 'stop')]
self.assertEqual(stops[-1], 'airag')
self.assertEqual(state['phase'], 'rolled-back')
def test_backup_and_compose_tampering_refused(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
(root / 'compose.json').write_text('{}')
(root / 'data.tar').write_bytes(b'cold-backup')
state = {'compose_sha256': upgrade.digest(root / 'compose.json'), 'backup_sha256': upgrade.digest(root / 'data.tar')}
(root / 'state.json').write_text(json.dumps(state))
self.assertEqual(upgrade.load_release(root), state)
(root / 'data.tar').write_bytes(b'changed')
with self.assertRaisesRegex(ValueError, 'Backup checksum'):
upgrade.load_release(root)
(root / 'compose.json').write_text('{"services":{}}')
with self.assertRaisesRegex(ValueError, 'Compose configuration'):
upgrade.load_release(root)
def test_wrong_migration_phase_refused(self):
with patch.object(upgrade, 'load_release', return_value={'phase': 'started-awaiting-acceptance'}), patch.object(upgrade, 'inventory') as inv:
with self.assertRaises(ValueError):
upgrade.apply(Path('/srv/unis/release'))
inv.assert_not_called()
def test_data_changed_since_preparation_is_rejected_before_gpu_or_rename(self):
rows = containers()
state = {'phase': 'prepared', 'data': '/srv/unis/clone', 'old': rows,
'original_signature': 'before', 'clone_signature': 'before'}
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), None)), patch.object(upgrade, 'ensure_exclusive'), patch.object(upgrade, 'safe_path', return_value=Path('/srv/unis/clone')), patch.object(upgrade, 'tree_signature', return_value='changed'), patch.object(upgrade, 'compose') as compose, patch.object(upgrade, 'run') as runner:
with self.assertRaisesRegex(ValueError, 'data changed'):
upgrade.apply(Path('/srv/unis/release'))
runner.assert_not_called()
compose.assert_not_called()
def test_migration_failure_freezes_clone_and_retains_original_containers(self):
rows = containers()
state = {'phase': 'prepared', 'project': 'this-release', 'data': '/srv/unis/clone', 'old': rows,
'original_signature': 'same', 'clone_signature': 'same', 'renamed': []}
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), None)), patch.object(upgrade, 'ensure_exclusive'), patch.object(upgrade, 'safe_path', return_value=Path('/srv/unis/clone')), patch.object(upgrade, 'tree_signature', return_value='same'), patch.object(upgrade.socket, 'socket'), patch.object(upgrade, 'save'), patch.object(upgrade, 'compose', side_effect=['gpu-ok', RuntimeError('migration failed'), 'stopped']) as compose, patch.object(upgrade, 'run') as runner:
with self.assertRaisesRegex(RuntimeError, 'migration failed'):
upgrade.apply(Path('/srv/unis/release'))
self.assertEqual(state['phase'], 'failed')
self.assertEqual(state['renamed'], list(upgrade.LEGACY))
self.assertTrue(all(call.args[:2] == ('docker', 'rename') for call in runner.call_args_list))
self.assertEqual(compose.call_args.args[1], 'stop')
def test_online_backup_container_must_mount_exact_legacy_pgdata(self):
row = {'Id': 'shared-pg', 'State': {'Running': True}, 'Config': {'Env': ['PG_MAJOR=17', 'PGDATA=/var/lib/postgresql/data']},
'Mounts': [{'Type': 'bind', 'Source': '/srv/unis/old/data/postgresql/pgdata', 'Destination': '/var/lib/postgresql/data'}]}
with patch.object(upgrade, 'inspect', return_value=row), patch.object(upgrade, 'safe_path', side_effect=Path):
value = upgrade.online_database('shared-pg', Path('/srv/unis/old'), containers())
self.assertEqual(value['id'], 'shared-pg')
row['Mounts'][0]['Source'] = '/srv/unis/another-db'
with self.assertRaisesRegex(ValueError, 'exactly'):
upgrade.online_database('shared-pg', Path('/srv/unis/old'), containers())
def test_online_backup_does_not_put_password_in_command_arguments(self):
row = {'Config': {'Env': ['POSTGRES_USER=postgres', 'POSTGRES_PASSWORD=private$replication']}}
with tempfile.TemporaryDirectory() as directory, patch.object(upgrade, 'inspect', return_value=row), patch.object(upgrade.subprocess, 'run', return_value=Mock(returncode=0)) as runner:
upgrade.backup_online({'id': 'shared-pg'}, Path(directory) / 'database.tar', containers())
args = runner.call_args.args[0]
self.assertNotIn('private$replication', ' '.join(args))
self.assertEqual(runner.call_args.kwargs['env']['PGPASSWORD'], 'private$replication')
self.assertIn('pg_basebackup', args)
self.assertIn('fetch', args)
if __name__ == '__main__':
unittest.main()