UnisKB/apps/models_provider/test_model_runtime.py

200 lines
8.8 KiB
Python
Raw Normal View History

import contextlib
import json
import threading
import weakref
import unittest
from concurrent.futures import ThreadPoolExecutor
from types import SimpleNamespace
from unittest.mock import patch
from common.model_runtime import ModelRuntime, loading_policy
from common.model_runtime_client import signed_request, verify_request
class FakeAdapter:
def __init__(self):
self.loaded, self.unloaded = [], []
self.native_ready = True
self.fail = False
self.block = None
def load(self, spec):
if self.block:
self.block.wait(3)
if self.fail:
raise ValueError('private error details')
self.loaded.append(spec['id'])
self.native_ready = True
return SimpleNamespace(), {'device': 'cpu'}
def unload(self, spec, value):
self.unloaded.append(spec['id'])
self.native_ready = False
def probe(self, spec):
return {'device': 'GPU', 'gpu_bytes': 100} if self.native_ready else None
def collect(self):
pass
def error(self, exc):
return 'load failed'
class RuntimeTests(unittest.TestCase):
def setUp(self):
self.now = 0
self.specs = {key: {'id': key, 'driver': 'local', 'fingerprint': 'v1',
'policy': {'strategy': 'ON_DEMAND', 'priority': priority, 'idle_seconds': 30}}
for key, priority in [('low', 10), ('high', 90)]}
self.adapter = FakeAdapter()
self.manager = ModelRuntime(lambda key: {**self.specs[key], 'policy': dict(self.specs[key]['policy'])},
self.adapter, lambda: self.now)
def test_no_load_until_first_inference_and_then_reuses_weights(self):
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
self.assertEqual(self.adapter.loaded, [])
with self.manager.inference('low') as first:
self.assertIsNotNone(first)
with self.manager.inference('low') as second:
self.assertIs(first, second)
self.assertEqual(self.adapter.loaded, ['low'])
def test_startup_preloads_highest_priority_first(self):
self.specs['low']['policy']['strategy'] = 'PRELOAD'
self.specs['high']['policy']['strategy'] = 'PRELOAD'
self.manager.preload(['low', 'high'])
for key in ['low', 'high']:
self.manager.entries[key]['future'].result(3)
self.assertEqual(self.adapter.loaded, ['high', 'low'])
self.assertLess(self.manager.status('high')['load_order'], self.manager.status('low')['load_order'])
def test_concurrent_first_calls_only_load_once(self):
self.adapter.block = threading.Event()
with ThreadPoolExecutor(max_workers=6) as pool:
futures = [pool.submit(self.manager.acquire, 'low') for _ in range(6)]
self.adapter.block.set()
tokens = [future.result(3) for future in futures]
self.assertEqual(len(set(tokens)), 6)
self.assertEqual(self.adapter.loaded, ['low'])
self.assertEqual(self.manager.status('low')['active_requests'], 6)
for token in tokens:
self.manager.release('low', token)
def test_in_use_models_cannot_be_unloaded(self):
token = self.manager.acquire('low')
with self.assertRaises(ValueError):
self.manager.unload('low')
self.manager.release('low', token)
self.manager.unload('low')
self.assertEqual(self.manager.status('low')['status'], 'UNLOADED')
def test_unload_releases_the_last_loader_reference(self):
class Weights:
pass
with patch.object(self.adapter, 'load', side_effect=lambda spec: (Weights(), {})):
token = self.manager.acquire('low')
pointer = weakref.ref(self.manager.entries['low']['value'])
self.manager.release('low', token)
self.manager.unload('low')
self.assertIsNone(pointer())
def test_idle_cleanup_respects_active_and_preloaded_models(self):
active = self.manager.acquire('low')
self.specs['high']['policy']['strategy'] = 'PRELOAD'
self.manager.preload(['high'])
self.manager.entries['high']['future'].result(3)
self.now = 100
self.manager.sweep()
self.assertEqual(self.adapter.unloaded, [])
self.manager.release('low', active)
self.now = 131
self.manager.sweep()
self.assertEqual(self.adapter.unloaded, ['low'])
self.assertEqual(self.manager.status('high')['status'], 'READY')
def test_failed_load_can_be_retried_and_private_errors_are_hidden(self):
self.adapter.fail = True
with self.assertRaisesRegex(ValueError, '^load failed$'):
self.manager.acquire('low')
self.assertEqual(self.manager.status('low')['status'], 'FAILED')
self.adapter.fail = False
with self.manager.inference('low'):
pass
self.assertEqual(self.manager.status('low')['status'], 'READY')
def test_changed_model_config_never_reuses_old_weights(self):
token = self.manager.acquire('low')
self.specs['low']['fingerprint'] = 'v2'
with self.assertRaises(ValueError):
self.manager.acquire('low')
self.manager.release('low', token)
with self.manager.inference('low'):
pass
self.assertEqual(self.adapter.loaded, ['low', 'low'])
self.assertEqual(self.adapter.unloaded, ['low'])
def test_native_expiration_requires_loading_again(self):
self.specs['low']['driver'] = 'ollama'
token = self.manager.acquire('low')
self.manager.release('low', token)
self.adapter.native_ready = False
token = self.manager.acquire('low')
self.manager.release('low', token)
self.assertEqual(self.adapter.loaded, ['low', 'low'])
def test_policy_validation(self):
for value in ([], False, {'priority': True}, {'priority': 101}, {'strategy': 'ANY'},
{'idle_seconds': 0}, {'engine_url': 'http://example.com'}):
with self.subTest(value=value), self.assertRaises(ValueError):
loading_policy(value)
def test_internal_rpc_rejects_forged_or_expired_payloads(self):
with patch('common.model_runtime_client._secret', return_value=b'test-only-key'):
body, signature = signed_request({'operation': 'load', 'model_id': 'test'})
self.assertEqual(verify_request(body, signature)['operation'], 'load')
with self.assertRaises(ValueError):
verify_request(body.replace(b'load', b'unload'), signature)
with self.assertRaises(ValueError):
verify_request(body, '')
with patch('common.model_runtime_client.time.time', return_value=1):
body, signature = signed_request({'operation': 'load'})
with self.assertRaises(ValueError):
verify_request(body, signature)
class ProviderHookTests(unittest.TestCase):
def test_embedding_inference_holds_a_lease_and_preserves_keep_alive(self):
from models_provider.impl.ollama_model_provider.model.embedding import OllamaEmbedding
events = []
@contextlib.contextmanager
def lease(model_id):
events.append(('acquire', model_id))
try:
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
finally:
events.append(('release', model_id))
model = OllamaEmbedding.new_instance('EMBEDDING', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
with patch('common.model_runtime_client.inference_lease', lease), patch.object(OllamaEmbedding, '_embed', return_value=[[1., 2.]]):
self.assertEqual(model.embed_query('text'), [1., 2.])
self.assertEqual(events, [('acquire', 'resource'), ('release', 'resource')])
self.assertEqual(model._default_params['keep_alive'], -1)
def test_llm_stream_releases_lease_when_generator_is_closed(self):
from models_provider.impl.ollama_model_provider.model.llm import OllamaChatModel
from langchain_ollama import ChatOllama
events = []
@contextlib.contextmanager
def lease(model_id):
events.append('acquire')
try:
yield {'strategy': 'PRELOAD', 'idle_seconds': 300}
finally:
events.append('release')
model = OllamaChatModel.new_instance('LLM', 'test', {'api_base': 'http://127.0.0.1:1'}, model_id='resource')
with patch('common.model_runtime_client.inference_lease', lease), patch.object(ChatOllama, '_create_chat_stream', return_value=iter([{'part': 1}, {'part': 2}])) as native:
stream = model._create_chat_stream([])
next(stream); stream.close()
self.assertEqual(native.call_args.kwargs['keep_alive'], -1)
self.assertEqual(events, ['acquire', 'release'])