271 lines
17 KiB
Python
271 lines
17 KiB
Python
import copy
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
SPEC = importlib.util.spec_from_file_location('upgrade', Path(__file__).resolve().parents[1] / 'upgrade.py')
|
|
upgrade = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(upgrade)
|
|
CHECK_SPEC = importlib.util.spec_from_file_location('internal_check', Path(__file__).resolve().parents[1] / 'check_internal_services.py')
|
|
internal_check = importlib.util.module_from_spec(CHECK_SPEC)
|
|
CHECK_SPEC.loader.exec_module(internal_check)
|
|
|
|
|
|
def containers():
|
|
result = {}
|
|
for name in upgrade.LEGACY:
|
|
result[name] = {'Id': 'old-' + name, 'Image': 'sha256:old', 'Name': '/' + name,
|
|
'State': {'Status': 'exited', 'Running': False},
|
|
'Config': {'Env': [], 'Labels': {}}, 'Mounts': [],
|
|
'HostConfig': {'PortBindings': {}}}
|
|
result['airag']['Config']['Env'] = [
|
|
'MAXKB_DB_HOST=127.0.0.1', 'MAXKB_DB_PASSWORD=literal$secret=${VALUE}',
|
|
'MAXKB_REDIS_HOST=127.0.0.1', 'MAXKB_VERSION=old', 'PATH=/old/python',
|
|
'PGDATA=/opt/maxkb/data/postgresql/pgdata', 'POSTGRES_PASSWORD=literal$pg',
|
|
]
|
|
result['mineru-api']['Config']['Env'] = ['CUDA_VISIBLE_DEVICES=1', 'MINERU_GPU_MEMORY_UTILIZATION=0.3', 'PYTHONPATH=/old/vendor', 'MINDMAP_LLM_API_KEY=legacy-private']
|
|
for name, port in (('unis-gateway', 8989), ('mindmap-web', 9898)):
|
|
result[name]['HostConfig']['PortBindings'][str(port) + '/tcp'] = [{'HostIp': '127.0.0.1', 'HostPort': str(port + 100)}]
|
|
return result
|
|
|
|
|
|
class UpgradeTests(unittest.TestCase):
|
|
def config(self, flavor='cuda'):
|
|
return upgrade.make_compose(containers(), {'kb': 'sha256:kb', 'parser': 'sha256:parser', 'gateway': 'sha256:gateway'}, Path('/srv/unis/clone'), Path('/srv/unis/models'), flavor, 'release-test')
|
|
|
|
def test_cuda_and_corex_have_separate_device_configuration(self):
|
|
cuda = self.config()['services']['mineru-api']
|
|
corex = self.config('corex')['services']['mineru-api']
|
|
self.assertEqual(cuda['deploy']['resources']['reservations']['devices'][0]['driver'], 'nvidia')
|
|
self.assertEqual(cuda['runtime'], 'nvidia')
|
|
self.assertNotIn('privileged', cuda)
|
|
self.assertNotIn('runtime', corex)
|
|
self.assertNotIn('deploy', corex)
|
|
self.assertTrue(corex['privileged'])
|
|
self.assertIn('corex-4.4.0', corex['environment']['PYTHONPATH'])
|
|
self.assertEqual(cuda['environment']['CUDA_VISIBLE_DEVICES'], '1')
|
|
self.assertEqual(cuda['environment']['MINERU_GPU_MEMORY_UTILIZATION'], '0.3')
|
|
|
|
def test_worker_shares_local_db_and_durable_assets(self):
|
|
config = self.config()
|
|
kb = config['services']['airag']
|
|
worker = config['services']['hub-worker']
|
|
self.assertEqual(worker['network_mode'], 'service:airag')
|
|
self.assertEqual(worker['environment'], kb['environment'])
|
|
self.assertEqual(worker['volumes'], kb['volumes'])
|
|
self.assertEqual(worker['image'], kb['image'])
|
|
self.assertEqual(worker['environment']['AI_HUB_ASSET_DIR'], '/opt/maxkb/data/ai-hub-assets')
|
|
self.assertNotIn('--recover', worker['command'])
|
|
self.assertNotIn('MAXKB_VERSION', kb['environment'])
|
|
self.assertNotIn('PATH', kb['environment'])
|
|
self.assertEqual(kb['command'], ['/usr/bin/start-all.sh'])
|
|
self.assertEqual(kb['environment']['MAXKB_DB_PORT'], '5432')
|
|
self.assertEqual(kb['environment']['MAXKB_REDIS_PORT'], '6379')
|
|
self.assertNotIn('postgres', config['services'])
|
|
self.assertNotIn('redis', config['services'])
|
|
|
|
def test_external_and_sentinel_settings_cannot_enter_internal_deployment(self):
|
|
for field, value in (('MAXKB_DB_HOST', 'ares-pg'), ('MAXKB_REDIS_HOST', 'ares-redis'),
|
|
('MAXKB_DB_PORT', '5433'), ('MAXKB_REDIS_PORT', '6380'),
|
|
('MAXKB_REDIS_SENTINEL_SENTINELS', 'sentinel:26379')):
|
|
rows = containers()
|
|
env = upgrade.environment(rows['airag'])
|
|
env[field] = value
|
|
with self.subTest(field=field), self.assertRaises(ValueError):
|
|
upgrade.require_internal_services(env)
|
|
|
|
def service_clients(self):
|
|
db = Mock()
|
|
db.settings_dict = {'HOST': '127.0.0.1', 'PORT': '5432'}
|
|
cursor = db.cursor.return_value.__enter__ = Mock(return_value=Mock())
|
|
db.cursor.return_value.__exit__ = Mock(return_value=False)
|
|
cursor.return_value.fetchone.return_value = ('/opt/maxkb/data/postgresql/pgdata', '127.0.0.1', 5432, '17.6')
|
|
cache = Mock()
|
|
cache.connection_pool = SimpleNamespace(connection_kwargs={'host': '127.0.0.1', 'port': 6379})
|
|
cache.ping.return_value = True
|
|
cache.config_get.return_value = {'dir': '/opt/maxkb/data/redis'}
|
|
cache.info.return_value = {'redis_version': '7.0.15'}
|
|
return db, cache
|
|
|
|
def test_runtime_check_uses_actual_db_and_cache_configuration(self):
|
|
db, cache = self.service_clients()
|
|
report = internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata')
|
|
self.assertEqual(report['postgresql']['host'], '127.0.0.1')
|
|
self.assertTrue(report['redis']['ping'])
|
|
query = db.cursor.return_value.__enter__.return_value.execute.call_args.args[0]
|
|
self.assertIn('data_directory', query)
|
|
self.assertNotIn('UPDATE', query)
|
|
|
|
def test_runtime_check_rejects_effective_external_connection(self):
|
|
db, cache = self.service_clients()
|
|
db.settings_dict['HOST'] = 'ares-pg'
|
|
with self.assertRaisesRegex(ValueError, 'internal PostgreSQL'):
|
|
internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata')
|
|
db.settings_dict['HOST'] = '127.0.0.1'
|
|
cache.connection_pool.connection_kwargs['host'] = 'ares-redis'
|
|
with self.assertRaisesRegex(ValueError, 'internal Redis'):
|
|
internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata')
|
|
|
|
def test_runtime_check_rejects_wrong_server_data_directory(self):
|
|
db, cache = self.service_clients()
|
|
cache.config_get.return_value = {'dir': '/tmp/redis'}
|
|
with self.assertRaisesRegex(ValueError, 'persistent MaxKB mount'):
|
|
internal_check.verify(db, cache, '/opt/maxkb/data/postgresql/pgdata')
|
|
cache.config_get.return_value = {'dir': '/opt/maxkb/data/redis'}
|
|
with self.assertRaisesRegex(ValueError, 'server path'):
|
|
internal_check.verify(db, cache, '/var/lib/postgresql/data')
|
|
|
|
def test_models_readonly_and_original_data_never_used_by_candidate(self):
|
|
config = self.config()
|
|
for service in config['services'].values():
|
|
for volume in service.get('volumes', []):
|
|
self.assertNotEqual(volume['source'], '/srv/unis/original')
|
|
if volume['target'] == '/models':
|
|
self.assertTrue(volume['read_only'])
|
|
self.assertFalse(volume['bind']['create_host_path'])
|
|
self.assertEqual(config['networks']['default']['name'], 'release-test')
|
|
self.assertNotIn('mindmap-web', config['services'])
|
|
|
|
def test_credentials_are_escaped_for_compose_and_not_sent_to_parser(self):
|
|
config = self.config()
|
|
escaped = upgrade.escape_compose(config)
|
|
value = escaped['services']['airag']['environment']['MAXKB_DB_PASSWORD']
|
|
self.assertEqual(value, 'literal$$secret=$${VALUE}')
|
|
self.assertNotIn('MINDMAP_LLM_API_KEY', config['services']['mineru-api']['environment'])
|
|
self.assertEqual(config['services']['airag']['environment']['MAXKB_DB_PASSWORD'], 'literal$secret=${VALUE}')
|
|
|
|
def test_existing_public_bindings_preserved(self):
|
|
ports = self.config()['services']['gateway']['ports']
|
|
self.assertEqual(ports[0]['published'], '9089')
|
|
self.assertEqual(ports[1]['published'], '9998')
|
|
self.assertTrue(all(p['host_ip'] == '127.0.0.1' for p in ports))
|
|
|
|
def test_running_or_paused_legacy_container_refused(self):
|
|
for status in ('running', 'paused', 'restarting', 'dead'):
|
|
with self.subTest(status=status):
|
|
rows = containers()
|
|
rows['airag']['State']['Status'] = status
|
|
with self.assertRaises(ValueError):
|
|
upgrade.ensure_stopped(rows)
|
|
|
|
def test_prepare_rejects_running_container_before_creating_files(self):
|
|
rows = containers()
|
|
rows['airag']['State']['Status'] = 'running'
|
|
with patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), Path('/srv/unis/models'))), patch.object(upgrade, 'run') as runner:
|
|
with self.assertRaises(ValueError):
|
|
upgrade.prepare(None)
|
|
runner.assert_not_called()
|
|
|
|
def test_rollback_refuses_unrelated_container(self):
|
|
state = {'phase': 'failed', 'project': 'this-release', 'old': containers(), 'renamed': list(upgrade.LEGACY)}
|
|
stranger = {'Config': {'Labels': {'com.docker.compose.project': 'some-other-release'}}, 'Id': 'unrelated'}
|
|
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inspect', return_value=stranger), patch.object(upgrade, 'run', return_value='unis-hub-worker\n') as runner:
|
|
with self.assertRaises(ValueError):
|
|
upgrade.rollback(Path('/srv/unis/release'))
|
|
self.assertEqual(runner.call_count, 1)
|
|
|
|
def test_rollback_keeps_new_data_and_restores_original_stopped_state(self):
|
|
rows = containers()
|
|
state = {'phase': 'failed', 'project': 'this-release', 'old': rows, 'renamed': list(upgrade.LEGACY)}
|
|
def read(name):
|
|
if name.startswith('old-'):
|
|
row = copy.deepcopy(rows[name[4:]])
|
|
row['Name'] += '-before-this-release'
|
|
return row
|
|
return {'Config': {'Labels': {'com.docker.compose.project': 'this-release'}}, 'Id': 'new-' + name}
|
|
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inspect', side_effect=read), patch.object(upgrade, 'save'), patch.object(upgrade, 'run', return_value='\n'.join(upgrade.NEW)) as runner:
|
|
upgrade.rollback(Path('/srv/unis/release'))
|
|
commands = [call.args for call in runner.call_args_list]
|
|
self.assertFalse(any('start' in command or 'rm' in command for command in commands))
|
|
stops = [command[-1] for command in commands if command[:2] == ('docker', 'stop')]
|
|
self.assertEqual(stops[-1], 'airag')
|
|
self.assertEqual(state['phase'], 'rolled-back')
|
|
|
|
def test_backup_and_compose_tampering_refused(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
root = Path(directory)
|
|
(root / 'compose.json').write_text('{}')
|
|
(root / 'data.tar').write_bytes(b'cold-backup')
|
|
state = {'compose_sha256': upgrade.digest(root / 'compose.json'), 'backup_sha256': upgrade.digest(root / 'data.tar')}
|
|
(root / 'state.json').write_text(json.dumps(state))
|
|
self.assertEqual(upgrade.load_release(root), state)
|
|
(root / 'data.tar').write_bytes(b'changed')
|
|
with self.assertRaisesRegex(ValueError, 'Backup checksum'):
|
|
upgrade.load_release(root)
|
|
(root / 'compose.json').write_text('{"services":{}}')
|
|
with self.assertRaisesRegex(ValueError, 'Compose configuration'):
|
|
upgrade.load_release(root)
|
|
|
|
def test_wrong_migration_phase_refused(self):
|
|
with patch.object(upgrade, 'load_release', return_value={'phase': 'started-awaiting-acceptance'}), patch.object(upgrade, 'inventory') as inv:
|
|
with self.assertRaises(ValueError):
|
|
upgrade.apply(Path('/srv/unis/release'))
|
|
inv.assert_not_called()
|
|
|
|
def test_data_changed_since_preparation_is_rejected_before_gpu_or_rename(self):
|
|
rows = containers()
|
|
state = {'phase': 'prepared', 'data': '/srv/unis/clone', 'old': rows,
|
|
'original_signature': 'before', 'clone_signature': 'before'}
|
|
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), None)), patch.object(upgrade, 'ensure_exclusive'), patch.object(upgrade, 'safe_path', return_value=Path('/srv/unis/clone')), patch.object(upgrade, 'tree_signature', return_value='changed'), patch.object(upgrade, 'compose') as compose, patch.object(upgrade, 'run') as runner:
|
|
with self.assertRaisesRegex(ValueError, 'data changed'):
|
|
upgrade.apply(Path('/srv/unis/release'))
|
|
runner.assert_not_called()
|
|
compose.assert_not_called()
|
|
|
|
def test_migration_failure_freezes_clone_and_retains_original_containers(self):
|
|
rows = containers()
|
|
state = {'phase': 'prepared', 'project': 'this-release', 'data': '/srv/unis/clone', 'old': rows,
|
|
'original_signature': 'same', 'clone_signature': 'same', 'renamed': []}
|
|
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), None)), patch.object(upgrade, 'ensure_exclusive'), patch.object(upgrade, 'safe_path', return_value=Path('/srv/unis/clone')), patch.object(upgrade, 'tree_signature', return_value='same'), patch.object(upgrade.socket, 'socket'), patch.object(upgrade, 'save'), patch.object(upgrade, 'compose', side_effect=['gpu-ok', RuntimeError('migration failed'), 'stopped']) as compose, patch.object(upgrade, 'run') as runner:
|
|
with self.assertRaisesRegex(RuntimeError, 'migration failed'):
|
|
upgrade.apply(Path('/srv/unis/release'))
|
|
self.assertEqual(state['phase'], 'failed')
|
|
self.assertEqual(state['renamed'], list(upgrade.LEGACY))
|
|
self.assertTrue(all(call.args[:2] == ('docker', 'rename') for call in runner.call_args_list))
|
|
self.assertEqual(compose.call_args.args[1], 'stop')
|
|
|
|
def test_apply_initializes_unified_roles_after_migrations(self):
|
|
rows = containers()
|
|
state = {'phase': 'prepared', 'project': 'this-release', 'data': '/srv/unis/clone', 'old': rows,
|
|
'original_signature': 'same', 'clone_signature': 'same', 'renamed': [], 'new_ids': {}}
|
|
def inspect_new(name):
|
|
return {'Id': 'new-' + name, 'Config': {'Labels': {'com.docker.compose.project': 'this-release'}}}
|
|
def run_command(*args):
|
|
return '{}' if args[-1] == '/usr/local/bin/unis-check-internal-services.py' else ''
|
|
with patch.object(upgrade, 'load_release', return_value=state), patch.object(upgrade, 'inventory', return_value=(rows, Path('/srv/unis/old'), None)), patch.object(upgrade, 'ensure_exclusive'), patch.object(upgrade, 'safe_path', return_value=Path('/srv/unis/clone')), patch.object(upgrade, 'tree_signature', return_value='same'), patch.object(upgrade.socket, 'socket'), patch.object(upgrade, 'save'), patch.object(upgrade, 'compose'), patch.object(upgrade, 'inspect', side_effect=inspect_new), patch.object(upgrade, 'run', side_effect=run_command) as runner:
|
|
upgrade.apply(Path('/srv/unis/release'))
|
|
commands = [call.args for call in runner.call_args_list]
|
|
migration = ('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'migrate', '--check')
|
|
roles = ('docker', 'exec', 'airag', 'python', 'apps/manage.py', 'unify_platform_roles')
|
|
self.assertLess(commands.index(migration), commands.index(roles))
|
|
self.assertEqual(state['phase'], 'started-awaiting-acceptance')
|
|
|
|
|
|
def test_online_backup_container_must_mount_exact_legacy_pgdata(self):
|
|
row = {'Id': 'shared-pg', 'State': {'Running': True}, 'Config': {'Env': ['PG_MAJOR=17', 'PGDATA=/var/lib/postgresql/data']},
|
|
'Mounts': [{'Type': 'bind', 'Source': '/srv/unis/old/data/postgresql/pgdata', 'Destination': '/var/lib/postgresql/data'}]}
|
|
with patch.object(upgrade, 'inspect', return_value=row), patch.object(upgrade, 'safe_path', side_effect=Path):
|
|
value = upgrade.online_database('shared-pg', Path('/srv/unis/old'), containers())
|
|
self.assertEqual(value['id'], 'shared-pg')
|
|
row['Mounts'][0]['Source'] = '/srv/unis/another-db'
|
|
with self.assertRaisesRegex(ValueError, 'exactly'):
|
|
upgrade.online_database('shared-pg', Path('/srv/unis/old'), containers())
|
|
|
|
def test_online_backup_does_not_put_password_in_command_arguments(self):
|
|
row = {'Config': {'Env': ['POSTGRES_USER=postgres', 'POSTGRES_PASSWORD=private$replication']}}
|
|
with tempfile.TemporaryDirectory() as directory, patch.object(upgrade, 'inspect', return_value=row), patch.object(upgrade.subprocess, 'run', return_value=Mock(returncode=0)) as runner:
|
|
upgrade.backup_online({'id': 'shared-pg'}, Path(directory) / 'database.tar', containers())
|
|
args = runner.call_args.args[0]
|
|
self.assertNotIn('private$replication', ' '.join(args))
|
|
self.assertEqual(runner.call_args.kwargs['env']['PGPASSWORD'], 'private$replication')
|
|
self.assertIn('pg_basebackup', args)
|
|
self.assertIn('fetch', args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|