Refactor system architecture (#82)

Author: WoosukKwon

cacheflow/master/block_manager.py renamed to cacheflow/core/block_manager.py

@@ -4,8 +4,8 @@
 import time
 from typing import Any, Dict, List, Optional, Tuple
 
-from cacheflow.master.block_manager import BlockSpaceManager
-from cacheflow.master.policy import PolicyFactory
+from cacheflow.core.block_manager import BlockSpaceManager
+from cacheflow.core.policy import PolicyFactory
 from cacheflow.sampling_params import SamplingParams
 from cacheflow.sequence import Sequence
 from cacheflow.sequence import SequenceGroup

cacheflow/master/policy.py renamed to cacheflow/core/policy.py

@@ -8,20 +8,21 @@
 except ImportError:
     ray = None
 
+from cacheflow.core.scheduler import Scheduler
+from cacheflow.frontend.simple_frontend import SimpleFrontend
 from cacheflow.logger import init_logger
-from cacheflow.master.scheduler import Scheduler
-from cacheflow.master.simple_frontend import SimpleFrontend
-from cacheflow.models import get_memory_analyzer
-from cacheflow.worker.controller import Controller, DeviceID
+from cacheflow.model_executor import get_memory_analyzer
 from cacheflow.sequence import SequenceGroup
 from cacheflow.sampling_params import SamplingParams
 from cacheflow.utils import get_gpu_memory, get_cpu_memory
+from cacheflow.worker.controller import Controller, DeviceID
 
 
 logger = init_logger(__name__)
 
 
 class Server:
+
     def __init__(
         self,
         model: str,

cacheflow/master/scheduler.py renamed to cacheflow/core/scheduler.py

@@ -1,22 +1,22 @@
 import argparse
 import asyncio
+import json
 import time
 from typing import List, Dict, Optional
-import json
 
-import ray
-from transformers import AutoTokenizer
 from fastapi import FastAPI, Request
 from fastapi.responses import StreamingResponse
+import ray
+from transformers import AutoTokenizer
 import uvicorn
 
+from cacheflow.core.server import (Server, add_server_arguments,
+                                   process_server_arguments,
+                                   initialize_cluster)
 from cacheflow.sampling_params import SamplingParams
 from cacheflow.sequence import Sequence, SequenceGroup
-from cacheflow.master.server import (Server, add_server_arguments,
-                                     process_server_arguments,
-                                     initialize_cluster)
-from cacheflow.worker.controller import DeviceID
 from cacheflow.utils import Counter, get_gpu_memory, get_cpu_memory
+from cacheflow.worker.controller import DeviceID
 
 TIMEOUT_TO_PREVENT_DEADLOCK = 1 # seconds
 app = FastAPI()

cacheflow/master/server.py renamed to cacheflow/core/server.py

@@ -0,0 +1,11 @@
+from cacheflow.model_executor.input_metadata import InputMetadata
+from cacheflow.model_executor.model_loader import get_model, get_memory_analyzer
+from cacheflow.model_executor.utils import set_random_seed
+
+
+__all__ = [
+    "InputMetadata",
+    "get_model",
+    "get_memory_analyzer",
+    "set_random_seed",
+]

cacheflow/http_frontend/fastapi_frontend.py renamed to cacheflow/frontend/fastapi_frontend.py

@@ -7,7 +7,7 @@
 from cacheflow import attention_ops
 from cacheflow import cache_ops
 from cacheflow import pos_encoding_ops
-from cacheflow.models import InputMetadata
+from cacheflow.model_executor.input_metadata import InputMetadata
 
 
 class GPTCacheFlowAttention(nn.Module):