UnisKB/apps/common/model_runtime.py

203 lines
8.8 KiB
Python
Raw Normal View History

"""Single-owner, priority-ordered weight loading. No engine or framework dependency."""
import contextlib
import itertools
import queue
import threading
import time
import uuid
from concurrent.futures import Future
def loading_policy(value=None):
value = {} if value is None else value
if not isinstance(value, dict) or set(value) - {'strategy', 'priority', 'idle_seconds'}:
raise ValueError('加载策略无效')
result = {'strategy': value.get('strategy', 'ON_DEMAND'), 'priority': value.get('priority', 50),
'idle_seconds': value.get('idle_seconds', 300)}
if result['strategy'] not in ('ON_DEMAND', 'PRELOAD'):
raise ValueError('加载方式无效')
if type(result['priority']) is not int or not 0 <= result['priority'] <= 100:
raise ValueError('加载优先级须为0~100的整数')
if type(result['idle_seconds']) is not int or not 30 <= result['idle_seconds'] <= 86400:
raise ValueError('空闲时间须为30~86400秒')
return result
class ModelRuntime:
def __init__(self, specs, adapter, clock=time.monotonic):
self.specs, self.adapter, self.clock = specs, adapter, clock
self.entries = {}
self.lock = threading.RLock()
self.queue = queue.PriorityQueue()
self.sequence = itertools.count()
self.load_sequence = itertools.count(1)
self.worker = None
def _entry(self, model_id):
model_id = str(model_id)
spec = self.specs(model_id)
spec['policy'] = loading_policy(spec.get('policy'))
entry = self.entries.get(model_id)
if entry and entry['spec']['fingerprint'] != spec['fingerprint']:
if entry['leases'] or entry['state'] == 'LOADING':
raise ValueError('模型配置已变更,请等待当前请求结束')
self._discard(entry)
entry = None
if entry is None:
entry = {'id': model_id, 'spec': spec, 'state': 'UNLOADED', 'value': None,
'leases': {}, 'future': None, 'generation': 0, 'error': '', 'info': {},
'last_used': self.clock(), 'loaded_at': None, 'started_at': None,
'load_order': None}
self.entries[model_id] = entry
previous = entry['spec']['policy']
entry['spec'] = spec
if entry['state'] == 'QUEUED' and previous['priority'] != spec['policy']['priority']:
entry['generation'] += 1
self._put(entry)
return entry
def _put(self, entry):
self.queue.put((-entry['spec']['policy']['priority'], next(self.sequence), entry['id'], entry['generation']))
def start(self):
with self.lock:
if self.worker is None:
self.worker = threading.Thread(target=self._work, name='model-loading', daemon=True)
self.worker.start()
def submit(self, model_id, start=True):
with self.lock:
entry = self._entry(model_id)
if entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama' and not self.adapter.probe(entry['spec']):
entry.update(state='UNLOADED', value=None, info={})
if entry['state'] == 'READY':
return entry
if entry['state'] not in ('LOADING', 'QUEUED'):
entry.update(state='QUEUED', error='', future=Future())
entry['generation'] += 1
self._put(entry)
if start:
self.start()
return entry
def preload(self, model_ids):
# Enqueue the complete startup batch before choosing the highest-priority model.
with self.lock:
for model_id in model_ids:
self.submit(model_id, start=False)
self.start()
def _work(self):
while True:
_, order, model_id, generation = self.queue.get()
try:
with self.lock:
entry = self.entries.get(model_id)
if not entry or entry['generation'] != generation or entry['state'] != 'QUEUED':
continue
entry.update(state='LOADING', started_at=time.time(), load_order=next(self.load_sequence))
spec = entry['spec']
future = entry['future']
value, info = self.adapter.load(spec)
with self.lock:
entry.update(state='READY', value=value, info=info, loaded_at=time.time(), last_used=self.clock())
future.set_result(True)
del value # The loader thread must not keep the last model alive after eviction.
except Exception as exc:
with self.lock:
entry.update(state='FAILED', error=self.adapter.error(exc), value=None)
if not entry['future'].done():
entry['future'].set_exception(ValueError(entry['error']))
finally:
self.queue.task_done()
def acquire(self, model_id, remote=False, timeout=600):
with self.lock:
entry = self.submit(model_id)
token = uuid.uuid4().hex
entry['leases'][token] = self.clock() + 3600 if remote else float('inf')
future = entry['future'] if entry['state'] != 'READY' else None
try:
if future:
future.result(timeout=timeout)
return token
except Exception:
self.release(model_id, token)
raise
def release(self, model_id, token):
with self.lock:
entry = self.entries.get(str(model_id))
if entry:
entry['leases'].pop(token, None)
entry['last_used'] = self.clock()
@contextlib.contextmanager
def inference(self, model_id):
token = self.acquire(model_id)
try:
with self.lock:
value = self.entries[str(model_id)]['value']
yield value
finally:
self.release(model_id, token)
def _discard(self, entry):
if entry['value'] is not None:
value = entry['value']
self.adapter.unload(entry['spec'], value)
entry['value'] = None
del value
self.adapter.collect()
entry.update(state='UNLOADED', loaded_at=None, info={})
def unload(self, model_id):
with self.lock:
entry = self._entry(model_id)
self._expire_leases(entry)
if entry['leases'] or entry['state'] in ('LOADING', 'QUEUED'):
raise ValueError('模型正在使用或加载,请稍后再试')
if entry['spec']['driver'] == 'ollama' and self.adapter.probe(entry['spec']):
entry['value'] = True
self._discard(entry)
def configure(self, model_id):
with self.lock:
entry = self._entry(model_id)
warm_again = entry['state'] == 'READY' and entry['spec']['driver'] == 'ollama'
if warm_again:
entry.update(state='UNLOADED', value=None)
if warm_again or entry['spec']['policy']['strategy'] == 'PRELOAD':
self.submit(model_id)
def _expire_leases(self, entry):
now = self.clock()
entry['leases'] = {key: expiry for key, expiry in entry['leases'].items() if expiry > now}
def sweep(self):
with self.lock:
for entry in list(self.entries.values()):
self._expire_leases(entry)
policy = entry['spec']['policy']
# Ollama owns its idle timer; do not unload models used by other clients.
if (entry['state'] == 'READY' and entry['spec']['driver'] == 'local' and
policy['strategy'] == 'ON_DEMAND' and not entry['leases'] and
self.clock() - entry['last_used'] >= policy['idle_seconds']):
self._discard(entry)
def status(self, model_id):
with self.lock:
entry = self._entry(model_id)
self._expire_leases(entry)
# Reconcile native residency, including expiration outside this process.
if entry['spec']['driver'] == 'ollama' and entry['state'] not in ('LOADING', 'QUEUED'):
info = self.adapter.probe(entry['spec'])
if info:
entry.update(state='READY', info=info, value=True)
elif entry['state'] == 'READY':
entry.update(state='UNLOADED', info={}, value=None, loaded_at=None)
return {'model_id': entry['id'], **entry['spec']['policy'], 'status': entry['state'],
'active_requests': len(entry['leases']), 'error': entry['error'],
'loaded_at': entry['loaded_at'], 'started_at': entry['started_at'],
'load_order': entry['load_order'], **entry['info']}