Python API Reference

This section documents the Python API for vector-inference.

Client Interface

vec_inf.client.api.VecInfClient

Client for interacting with Vector Inference programmatically.

This class provides methods for launching models, checking their status, retrieving metrics, and shutting down models using the Vector Inference infrastructure.

Methods:

Name	Description
list_models	List all available models
get_model_config	Get configuration for a specific model
launch_model	Launch a model on the cluster
get_status	Get status of a running model
get_metrics	Get performance metrics of a running model
shutdown_model	Shutdown a running model
wait_until_ready	Wait for a model to become ready
cleanup_logs	Remove logs from the log directory.

Examples:

>>> from vec_inf.api import VecInfClient
>>> client = VecInfClient()
>>> response = client.launch_model("Meta-Llama-3.1-8B-Instruct")
>>> job_id = response.slurm_job_id
>>> status = client.get_status(job_id)
>>> if status.status == ModelStatus.READY:
...     print(f"Model is ready at {status.base_url}")
>>> client.shutdown_model(job_id)

Source code in vec_inf/client/api.py

class VecInfClient:
    """Client for interacting with Vector Inference programmatically.

    This class provides methods for launching models, checking their status,
    retrieving metrics, and shutting down models using the Vector Inference
    infrastructure.

    Methods
    -------
    list_models()
        List all available models
    get_model_config(model_name)
        Get configuration for a specific model
    launch_model(model_name, options)
        Launch a model on the cluster
    get_status(slurm_job_id, log_dir)
        Get status of a running model
    get_metrics(slurm_job_id, log_dir)
        Get performance metrics of a running model
    shutdown_model(slurm_job_id)
        Shutdown a running model
    wait_until_ready(slurm_job_id, timeout_seconds, poll_interval_seconds, log_dir)
        Wait for a model to become ready

    cleanup_logs(log_dir, model_name, model_family, job_id, dry_run)
        Remove logs from the log directory.

    Examples
    --------
    >>> from vec_inf.api import VecInfClient
    >>> client = VecInfClient()
    >>> response = client.launch_model("Meta-Llama-3.1-8B-Instruct")
    >>> job_id = response.slurm_job_id
    >>> status = client.get_status(job_id)
    >>> if status.status == ModelStatus.READY:
    ...     print(f"Model is ready at {status.base_url}")
    >>> client.shutdown_model(job_id)
    """

    def __init__(self) -> None:
        """Initialize the Vector Inference client."""
        pass

    def list_models(self) -> list[ModelInfo]:
        """List all available models.

        Returns
        -------
        list[ModelInfo]
            List of ModelInfo objects containing information about available models,
            including their configurations and specifications.
        """
        model_registry = ModelRegistry()
        return model_registry.get_all_models()

    def get_model_config(self, model_name: str) -> ModelConfig:
        """Get the configuration for a specific model.

        Parameters
        ----------
        model_name : str
            Name of the model to get configuration for

        Returns
        -------
        ModelConfig
            Complete configuration for the specified model

        Raises
        ------
        ModelNotFoundError
            If the specified model is not found in the configuration
        """
        model_registry = ModelRegistry()
        return model_registry.get_single_model_config(model_name)

    def launch_model(
        self, model_name: str, options: Optional[LaunchOptions] = None
    ) -> LaunchResponse:
        """Launch a model on the cluster.

        Parameters
        ----------
        model_name : str
            Name of the model to launch
        options : LaunchOptions, optional
            Launch options to override default configuration

        Returns
        -------
        LaunchResponse
            Response containing launch details including:
            - SLURM job ID
            - Model configuration
            - Launch status

        Raises
        ------
        ModelConfigurationError
            If the model configuration is invalid
        SlurmJobError
            If there's an error launching the SLURM job
        """
        # Convert LaunchOptions to dictionary if provided
        options_dict: dict[str, Any] = {}
        if options:
            options_dict = {k: v for k, v in vars(options).items() if v is not None}

        # Create and use the API Launch Helper
        model_launcher = ModelLauncher(model_name, options_dict)
        return model_launcher.launch()

    def batch_launch_models(
        self,
        model_names: list[str],
        batch_config: Optional[str] = None,
        account: Optional[str] = None,
        work_dir: Optional[str] = None,
    ) -> BatchLaunchResponse:
        """Launch multiple models on the cluster.

        Parameters
        ----------
        model_names : list[str]
            List of model names to launch

        Returns
        -------
        BatchLaunchResponse
            Response containing launch details for each model

        Raises
        ------
        ModelConfigurationError
            If the model configuration is invalid
        """
        model_launcher = BatchModelLauncher(
            model_names, batch_config, account, work_dir
        )
        return model_launcher.launch()

    def get_status(self, slurm_job_id: str) -> StatusResponse:
        """Get the status of a running model.

        Parameters
        ----------
        slurm_job_id : str
            The SLURM job ID to check

        Returns
        -------
        StatusResponse
            Status information including:
            - Model name
            - Server status
            - Job state
            - Base URL (if ready)
            - Error information (if failed)
        """
        model_status_monitor = ModelStatusMonitor(slurm_job_id)
        return model_status_monitor.process_model_status()

    def get_metrics(self, slurm_job_id: str) -> MetricsResponse:
        """Get the performance metrics of a running model.

        Parameters
        ----------
        slurm_job_id : str
            The SLURM job ID to get metrics for

        Returns
        -------
        MetricsResponse
            Response containing:
            - Model name
            - Performance metrics or error message
            - Timestamp of collection
        """
        performance_metrics_collector = PerformanceMetricsCollector(slurm_job_id)

        metrics: Union[dict[str, float], str]
        if not performance_metrics_collector.metrics_url.startswith("http"):
            metrics = performance_metrics_collector.metrics_url
        else:
            metrics = performance_metrics_collector.fetch_metrics()

        return MetricsResponse(
            model_name=performance_metrics_collector.status_info.model_name,
            metrics=metrics,
            timestamp=time.time(),
        )

    def shutdown_model(self, slurm_job_id: str) -> bool:
        """Shutdown a running model.

        Parameters
        ----------
        slurm_job_id : str
            The SLURM job ID to shut down

        Returns
        -------
        bool
            True if the model was successfully shutdown

        Raises
        ------
        SlurmJobError
            If there was an error shutting down the model
        """
        shutdown_cmd = f"scancel {slurm_job_id}"
        _, stderr = run_bash_command(shutdown_cmd)
        if stderr:
            raise SlurmJobError(f"Failed to shutdown model: {stderr}")
        return True

    def wait_until_ready(
        self,
        slurm_job_id: str,
        timeout_seconds: int = 1800,
        poll_interval_seconds: int = 10,
    ) -> StatusResponse:
        """Wait until a model is ready or fails.

        Parameters
        ----------
        slurm_job_id : str
            The SLURM job ID to wait for
        timeout_seconds : int, optional
            Maximum time to wait in seconds, by default 1800 (30 mins)
        poll_interval_seconds : int, optional
            How often to check status in seconds, by default 10

        Returns
        -------
        StatusResponse
            Status information when the model becomes ready

        Raises
        ------
        SlurmJobError
            If the specified job is not found or there's an error with the job
        ServerError
            If the server fails to start within the timeout period
        APIError
            If there was an error checking the status

        Notes
        -----
        The timeout is reset if the model is still in PENDING state after the
        initial timeout period. This allows for longer queue times in the SLURM
        scheduler.
        """
        start_time = time.time()

        while True:
            status_info = self.get_status(slurm_job_id)

            if status_info.server_status == ModelStatus.READY:
                return status_info

            if status_info.server_status == ModelStatus.FAILED:
                error_message = status_info.failed_reason or "Unknown error"
                raise ServerError(f"Model failed to start: {error_message}")

            if status_info.server_status == ModelStatus.SHUTDOWN:
                raise ServerError("Model was shutdown before it became ready")

            # Check timeout
            if time.time() - start_time > timeout_seconds:
                if status_info.server_status == ModelStatus.PENDING:
                    warnings.warn(
                        f"Model is still pending after {timeout_seconds} seconds, resetting timer...",
                        UserWarning,
                        stacklevel=2,
                    )
                    start_time = time.time()
                raise ServerError(
                    f"Timed out waiting for model to become ready after {timeout_seconds} seconds"
                )

            # Wait before checking again
            time.sleep(poll_interval_seconds)

    def cleanup_logs(
        self,
        log_dir: Optional[Union[str, Path]] = None,
        model_family: Optional[str] = None,
        model_name: Optional[str] = None,
        job_id: Optional[int] = None,
        before_job_id: Optional[int] = None,
        dry_run: bool = False,
    ) -> list[Path]:
        """Remove logs from the log directory.

        Parameters
        ----------
        log_dir : str or Path, optional
            Root directory containing log files. Defaults to ~/.vec-inf-logs.
        model_family : str, optional
            Only delete logs for this model family.
        model_name : str, optional
            Only delete logs for this model name.
        job_id : int, optional
            If provided, only match directories with this exact SLURM job ID.
        before_job_id : int, optional
            If provided, only delete logs with job ID less than this value.
        dry_run : bool
            If True, return matching files without deleting them.

        Returns
        -------
        list[Path]
            List of deleted (or matched if dry_run) log file paths.
        """
        log_root = Path(log_dir) if log_dir else Path.home() / ".vec-inf-logs"
        matched = find_matching_dirs(
            log_dir=log_root,
            model_family=model_family,
            model_name=model_name,
            job_id=job_id,
            before_job_id=before_job_id,
        )

        if dry_run:
            return matched

        for path in matched:
            shutil.rmtree(path)

        return matched

__init__

__init__()

Initialize the Vector Inference client.

Source code in vec_inf/client/api.py

def __init__(self) -> None:
    """Initialize the Vector Inference client."""
    pass

list_models

list_models()

List all available models.

Returns:

Type	Description
list[ModelInfo]	List of ModelInfo objects containing information about available models, including their configurations and specifications.

Source code in vec_inf/client/api.py

def list_models(self) -> list[ModelInfo]:
    """List all available models.

    Returns
    -------
    list[ModelInfo]
        List of ModelInfo objects containing information about available models,
        including their configurations and specifications.
    """
    model_registry = ModelRegistry()
    return model_registry.get_all_models()

get_model_config

get_model_config(model_name)

Get the configuration for a specific model.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model to get configuration for	required

Returns:

Type	Description
ModelConfig	Complete configuration for the specified model

Raises:

Type	Description
ModelNotFoundError	If the specified model is not found in the configuration

Source code in vec_inf/client/api.py

def get_model_config(self, model_name: str) -> ModelConfig:
    """Get the configuration for a specific model.

    Parameters
    ----------
    model_name : str
        Name of the model to get configuration for

    Returns
    -------
    ModelConfig
        Complete configuration for the specified model

    Raises
    ------
    ModelNotFoundError
        If the specified model is not found in the configuration
    """
    model_registry = ModelRegistry()
    return model_registry.get_single_model_config(model_name)

launch_model

launch_model(model_name, options=None)

Launch a model on the cluster.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model to launch	required
options	LaunchOptions	Launch options to override default configuration	None

Returns:

Type	Description
LaunchResponse	Response containing launch details including: - SLURM job ID - Model configuration - Launch status

Raises:

Type	Description
ModelConfigurationError	If the model configuration is invalid
SlurmJobError	If there's an error launching the SLURM job

Source code in vec_inf/client/api.py

def launch_model(
    self, model_name: str, options: Optional[LaunchOptions] = None
) -> LaunchResponse:
    """Launch a model on the cluster.

    Parameters
    ----------
    model_name : str
        Name of the model to launch
    options : LaunchOptions, optional
        Launch options to override default configuration

    Returns
    -------
    LaunchResponse
        Response containing launch details including:
        - SLURM job ID
        - Model configuration
        - Launch status

    Raises
    ------
    ModelConfigurationError
        If the model configuration is invalid
    SlurmJobError
        If there's an error launching the SLURM job
    """
    # Convert LaunchOptions to dictionary if provided
    options_dict: dict[str, Any] = {}
    if options:
        options_dict = {k: v for k, v in vars(options).items() if v is not None}

    # Create and use the API Launch Helper
    model_launcher = ModelLauncher(model_name, options_dict)
    return model_launcher.launch()

batch_launch_models

batch_launch_models(
    model_names,
    batch_config=None,
    account=None,
    work_dir=None,
)

Launch multiple models on the cluster.

Parameters:

Name	Type	Description	Default
model_names	list[str]	List of model names to launch	required

Returns:

Type	Description
BatchLaunchResponse	Response containing launch details for each model

Raises:

Type	Description
ModelConfigurationError	If the model configuration is invalid

Source code in vec_inf/client/api.py

def batch_launch_models(
    self,
    model_names: list[str],
    batch_config: Optional[str] = None,
    account: Optional[str] = None,
    work_dir: Optional[str] = None,
) -> BatchLaunchResponse:
    """Launch multiple models on the cluster.

    Parameters
    ----------
    model_names : list[str]
        List of model names to launch

    Returns
    -------
    BatchLaunchResponse
        Response containing launch details for each model

    Raises
    ------
    ModelConfigurationError
        If the model configuration is invalid
    """
    model_launcher = BatchModelLauncher(
        model_names, batch_config, account, work_dir
    )
    return model_launcher.launch()

get_status

get_status(slurm_job_id)

Get the status of a running model.

Parameters:

Name	Type	Description	Default
slurm_job_id	str	The SLURM job ID to check	required

Returns:

Type	Description
StatusResponse	Status information including: - Model name - Server status - Job state - Base URL (if ready) - Error information (if failed)

Source code in vec_inf/client/api.py

def get_status(self, slurm_job_id: str) -> StatusResponse:
    """Get the status of a running model.

    Parameters
    ----------
    slurm_job_id : str
        The SLURM job ID to check

    Returns
    -------
    StatusResponse
        Status information including:
        - Model name
        - Server status
        - Job state
        - Base URL (if ready)
        - Error information (if failed)
    """
    model_status_monitor = ModelStatusMonitor(slurm_job_id)
    return model_status_monitor.process_model_status()

get_metrics

get_metrics(slurm_job_id)

Get the performance metrics of a running model.

Parameters:

Name	Type	Description	Default
slurm_job_id	str	The SLURM job ID to get metrics for	required

Returns:

Type	Description
MetricsResponse	Response containing: - Model name - Performance metrics or error message - Timestamp of collection

Source code in vec_inf/client/api.py

def get_metrics(self, slurm_job_id: str) -> MetricsResponse:
    """Get the performance metrics of a running model.

    Parameters
    ----------
    slurm_job_id : str
        The SLURM job ID to get metrics for

    Returns
    -------
    MetricsResponse
        Response containing:
        - Model name
        - Performance metrics or error message
        - Timestamp of collection
    """
    performance_metrics_collector = PerformanceMetricsCollector(slurm_job_id)

    metrics: Union[dict[str, float], str]
    if not performance_metrics_collector.metrics_url.startswith("http"):
        metrics = performance_metrics_collector.metrics_url
    else:
        metrics = performance_metrics_collector.fetch_metrics()

    return MetricsResponse(
        model_name=performance_metrics_collector.status_info.model_name,
        metrics=metrics,
        timestamp=time.time(),
    )

shutdown_model

shutdown_model(slurm_job_id)

Shutdown a running model.

Parameters:

Name	Type	Description	Default
slurm_job_id	str	The SLURM job ID to shut down	required

Returns:

Type	Description
bool	True if the model was successfully shutdown

Raises:

Type	Description
SlurmJobError	If there was an error shutting down the model

Source code in vec_inf/client/api.py

def shutdown_model(self, slurm_job_id: str) -> bool:
    """Shutdown a running model.

    Parameters
    ----------
    slurm_job_id : str
        The SLURM job ID to shut down

    Returns
    -------
    bool
        True if the model was successfully shutdown

    Raises
    ------
    SlurmJobError
        If there was an error shutting down the model
    """
    shutdown_cmd = f"scancel {slurm_job_id}"
    _, stderr = run_bash_command(shutdown_cmd)
    if stderr:
        raise SlurmJobError(f"Failed to shutdown model: {stderr}")
    return True

wait_until_ready

wait_until_ready(
    slurm_job_id,
    timeout_seconds=1800,
    poll_interval_seconds=10,
)

Wait until a model is ready or fails.

Parameters:

Name	Type	Description	Default
slurm_job_id	str	The SLURM job ID to wait for	required
timeout_seconds	int	Maximum time to wait in seconds, by default 1800 (30 mins)	1800
poll_interval_seconds	int	How often to check status in seconds, by default 10	10

Returns:

Type	Description
StatusResponse	Status information when the model becomes ready

Raises:

Type	Description
SlurmJobError	If the specified job is not found or there's an error with the job
ServerError	If the server fails to start within the timeout period
APIError	If there was an error checking the status

Notes

The timeout is reset if the model is still in PENDING state after the initial timeout period. This allows for longer queue times in the SLURM scheduler.

Source code in vec_inf/client/api.py

def wait_until_ready(
    self,
    slurm_job_id: str,
    timeout_seconds: int = 1800,
    poll_interval_seconds: int = 10,
) -> StatusResponse:
    """Wait until a model is ready or fails.

    Parameters
    ----------
    slurm_job_id : str
        The SLURM job ID to wait for
    timeout_seconds : int, optional
        Maximum time to wait in seconds, by default 1800 (30 mins)
    poll_interval_seconds : int, optional
        How often to check status in seconds, by default 10

    Returns
    -------
    StatusResponse
        Status information when the model becomes ready

    Raises
    ------
    SlurmJobError
        If the specified job is not found or there's an error with the job
    ServerError
        If the server fails to start within the timeout period
    APIError
        If there was an error checking the status

    Notes
    -----
    The timeout is reset if the model is still in PENDING state after the
    initial timeout period. This allows for longer queue times in the SLURM
    scheduler.
    """
    start_time = time.time()

    while True:
        status_info = self.get_status(slurm_job_id)

        if status_info.server_status == ModelStatus.READY:
            return status_info

        if status_info.server_status == ModelStatus.FAILED:
            error_message = status_info.failed_reason or "Unknown error"
            raise ServerError(f"Model failed to start: {error_message}")

        if status_info.server_status == ModelStatus.SHUTDOWN:
            raise ServerError("Model was shutdown before it became ready")

        # Check timeout
        if time.time() - start_time > timeout_seconds:
            if status_info.server_status == ModelStatus.PENDING:
                warnings.warn(
                    f"Model is still pending after {timeout_seconds} seconds, resetting timer...",
                    UserWarning,
                    stacklevel=2,
                )
                start_time = time.time()
            raise ServerError(
                f"Timed out waiting for model to become ready after {timeout_seconds} seconds"
            )

        # Wait before checking again
        time.sleep(poll_interval_seconds)

cleanup_logs

cleanup_logs(
    log_dir=None,
    model_family=None,
    model_name=None,
    job_id=None,
    before_job_id=None,
    dry_run=False,
)

Remove logs from the log directory.

Parameters:

Name	Type	Description	Default
log_dir	str or Path	Root directory containing log files. Defaults to ~/.vec-inf-logs.	None
model_family	str	Only delete logs for this model family.	None
model_name	str	Only delete logs for this model name.	None
job_id	int	If provided, only match directories with this exact SLURM job ID.	None
before_job_id	int	If provided, only delete logs with job ID less than this value.	None
dry_run	bool	If True, return matching files without deleting them.	False

Returns:

Type	Description
list[Path]	List of deleted (or matched if dry_run) log file paths.

Source code in vec_inf/client/api.py

def cleanup_logs(
    self,
    log_dir: Optional[Union[str, Path]] = None,
    model_family: Optional[str] = None,
    model_name: Optional[str] = None,
    job_id: Optional[int] = None,
    before_job_id: Optional[int] = None,
    dry_run: bool = False,
) -> list[Path]:
    """Remove logs from the log directory.

    Parameters
    ----------
    log_dir : str or Path, optional
        Root directory containing log files. Defaults to ~/.vec-inf-logs.
    model_family : str, optional
        Only delete logs for this model family.
    model_name : str, optional
        Only delete logs for this model name.
    job_id : int, optional
        If provided, only match directories with this exact SLURM job ID.
    before_job_id : int, optional
        If provided, only delete logs with job ID less than this value.
    dry_run : bool
        If True, return matching files without deleting them.

    Returns
    -------
    list[Path]
        List of deleted (or matched if dry_run) log file paths.
    """
    log_root = Path(log_dir) if log_dir else Path.home() / ".vec-inf-logs"
    matched = find_matching_dirs(
        log_dir=log_root,
        model_family=model_family,
        model_name=model_name,
        job_id=job_id,
        before_job_id=before_job_id,
    )

    if dry_run:
        return matched

    for path in matched:
        shutil.rmtree(path)

    return matched

Model Config

vec_inf.client.config.ModelConfig

Bases: BaseModel

Pydantic model for validating and managing model deployment configurations.

A configuration class that handles validation and management of model deployment settings, including model specifications, hardware requirements, and runtime parameters.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model, must be alphanumeric with allowed characters: '-', '_', '.'	required
model_family	str	Family/architecture of the model	required
model_variant	str	Specific variant or version of the model family	required
model_type	(LLM, VLM, Text_Embedding, Reward_Modeling)	Type of model architecture	'LLM'
gpus_per_node	int	Number of GPUs to use per node (1-MAX_GPUS_PER_NODE)	required
num_nodes	int	Number of nodes to use for deployment (1-MAX_NUM_NODES)	required
cpus_per_task	int	Number of CPU cores per task (1-MAX_CPUS_PER_TASK)	required
mem_per_node	str	Memory allocation per node in GB format (e.g., '32G')	required
vocab_size	int	Size of the model's vocabulary (1-1,000,000)	required
account	str	Charge resources used by this job to specified account.	required
work_dir	str	Set working directory for the batch job	required
qos	Union[QOS, str]	Quality of Service tier for job scheduling	required
time	str	Time limit for the job in HH:MM:SS format	required
partition	Union[PARTITION, str]	Slurm partition for job scheduling	required
resource_type	Union[RESOURCE_TYPE, str]	Type of resource to request for the job	required
venv	str	Virtual environment or container system to use	required
log_dir	Path	Directory path for storing logs	required
model_weights_parent_dir	Path	Base directory containing model weights	required
vllm_args	dict[str, Any]	Additional arguments for vLLM engine configuration	required

Notes

All fields are validated using Pydantic's validation system. The model is configured to be immutable (frozen) and forbids extra fields.

Source code in vec_inf/client/config.py

class ModelConfig(BaseModel):
    """Pydantic model for validating and managing model deployment configurations.

    A configuration class that handles validation and management of model deployment
    settings, including model specifications, hardware requirements, and runtime
    parameters.

    Parameters
    ----------
    model_name : str
        Name of the model, must be alphanumeric with allowed characters: '-', '_', '.'
    model_family : str
        Family/architecture of the model
    model_variant : str, optional
        Specific variant or version of the model family
    model_type : {'LLM', 'VLM', 'Text_Embedding', 'Reward_Modeling'}
        Type of model architecture
    gpus_per_node : int
        Number of GPUs to use per node (1-MAX_GPUS_PER_NODE)
    num_nodes : int
        Number of nodes to use for deployment (1-MAX_NUM_NODES)
    cpus_per_task : int, optional
        Number of CPU cores per task (1-MAX_CPUS_PER_TASK)
    mem_per_node : str, optional
        Memory allocation per node in GB format (e.g., '32G')
    vocab_size : int
        Size of the model's vocabulary (1-1,000,000)
    account : str, optional
        Charge resources used by this job to specified account.
    work_dir : str, optional
        Set working directory for the batch job
    qos : Union[QOS, str], optional
        Quality of Service tier for job scheduling
    time : str, optional
        Time limit for the job in HH:MM:SS format
    partition : Union[PARTITION, str], optional
        Slurm partition for job scheduling
    resource_type : Union[RESOURCE_TYPE, str], optional
        Type of resource to request for the job
    venv : str, optional
        Virtual environment or container system to use
    log_dir : Path, optional
        Directory path for storing logs
    model_weights_parent_dir : Path, optional
        Base directory containing model weights
    vllm_args : dict[str, Any], optional
        Additional arguments for vLLM engine configuration

    Notes
    -----
    All fields are validated using Pydantic's validation system. The model is
    configured to be immutable (frozen) and forbids extra fields.
    """

    model_name: str = Field(..., min_length=3, pattern=r"^[a-zA-Z0-9\-_\.]+$")
    model_family: str = Field(..., min_length=2)
    model_variant: Optional[str] = Field(
        default=None, description="Specific variant/version of the model family"
    )
    model_type: Literal["LLM", "VLM", "Text_Embedding", "Reward_Modeling"] = Field(
        ..., description="Type of model architecture"
    )
    gpus_per_node: int = Field(
        ..., gt=0, le=MAX_GPUS_PER_NODE, description="GPUs per node"
    )
    num_nodes: int = Field(..., gt=0, le=MAX_NUM_NODES, description="Number of nodes")
    cpus_per_task: int = Field(
        default=int(DEFAULT_ARGS["cpus_per_task"]),
        gt=0,
        le=MAX_CPUS_PER_TASK,
        description="CPUs per task",
    )
    mem_per_node: str = Field(
        default=DEFAULT_ARGS["mem_per_node"],
        pattern=r"^\d{1,4}G$",
        description="Memory per node",
    )
    vocab_size: int = Field(..., gt=0, le=1_000_000)
    account: Optional[str] = Field(
        default=None, description="Account name for job scheduling"
    )
    work_dir: Optional[str] = Field(
        default=None, description="Working directory for the job"
    )
    qos: Optional[Union[QOS, str]] = Field(
        default=DEFAULT_ARGS["qos"] if DEFAULT_ARGS["qos"] != "" else None,
        description="Quality of Service tier",
    )
    time: str = Field(
        default=DEFAULT_ARGS["time"],
        pattern=r"^\d{2}:\d{2}:\d{2}$",
        description="HH:MM:SS time limit",
    )
    partition: Optional[Union[PARTITION, str]] = Field(
        default=DEFAULT_ARGS["partition"] if DEFAULT_ARGS["partition"] != "" else None,
        description="GPU partition type",
    )
    resource_type: Optional[Union[RESOURCE_TYPE, str]] = Field(
        default=DEFAULT_ARGS["resource_type"]
        if DEFAULT_ARGS["resource_type"] != ""
        else None,
        description="Resource type",
    )
    exclude: Optional[str] = Field(
        default=DEFAULT_ARGS["exclude"],
        description="Exclude certain nodes from the resources granted to the job",
    )
    nodelist: Optional[str] = Field(
        default=DEFAULT_ARGS["nodelist"],
        description="Request a specific list of nodes for deployment",
    )
    bind: Optional[str] = Field(
        default=DEFAULT_ARGS["bind"],
        description="Additional binds for the container",
    )
    venv: str = Field(
        default=DEFAULT_ARGS["venv"],
        description="Virtual environment/container system",
    )
    log_dir: Path = Field(
        default=Path(DEFAULT_ARGS["log_dir"]),
        description="Log directory path",
    )
    model_weights_parent_dir: Path = Field(
        default=Path(DEFAULT_ARGS["model_weights_parent_dir"]),
        description="Base directory for model weights",
    )
    vllm_args: Optional[dict[str, Any]] = Field(
        default={}, description="vLLM engine arguments"
    )
    env: Optional[dict[str, Any]] = Field(
        default={}, description="Environment variables to be set"
    )
    model_config = ConfigDict(
        extra="forbid", str_strip_whitespace=True, validate_default=True, frozen=True
    )

model_name class-attribute instance-attribute

model_name = Field(
    ..., min_length=3, pattern="^[a-zA-Z0-9\\-_\\.]+$"
)

model_family class-attribute instance-attribute

model_family = Field(..., min_length=2)

model_variant class-attribute instance-attribute

model_variant = Field(
    default=None,
    description="Specific variant/version of the model family",
)

model_type class-attribute instance-attribute

model_type = Field(
    ..., description="Type of model architecture"
)

gpus_per_node class-attribute instance-attribute

gpus_per_node = Field(
    ...,
    gt=0,
    le=MAX_GPUS_PER_NODE,
    description="GPUs per node",
)

num_nodes class-attribute instance-attribute

num_nodes = Field(
    ...,
    gt=0,
    le=MAX_NUM_NODES,
    description="Number of nodes",
)

cpus_per_task class-attribute instance-attribute

cpus_per_task = Field(
    default=int(DEFAULT_ARGS["cpus_per_task"]),
    gt=0,
    le=MAX_CPUS_PER_TASK,
    description="CPUs per task",
)

mem_per_node class-attribute instance-attribute

mem_per_node = Field(
    default=DEFAULT_ARGS["mem_per_node"],
    pattern="^\\d{1,4}G$",
    description="Memory per node",
)

vocab_size class-attribute instance-attribute

vocab_size = Field(..., gt=0, le=1000000)

account class-attribute instance-attribute

account = Field(
    default=None,
    description="Account name for job scheduling",
)

work_dir class-attribute instance-attribute

work_dir = Field(
    default=None,
    description="Working directory for the job",
)

qos class-attribute instance-attribute

qos = Field(
    default=DEFAULT_ARGS["qos"]
    if DEFAULT_ARGS["qos"] != ""
    else None,
    description="Quality of Service tier",
)

time class-attribute instance-attribute

time = Field(
    default=DEFAULT_ARGS["time"],
    pattern="^\\d{2}:\\d{2}:\\d{2}$",
    description="HH:MM:SS time limit",
)

partition class-attribute instance-attribute

partition = Field(
    default=DEFAULT_ARGS["partition"]
    if DEFAULT_ARGS["partition"] != ""
    else None,
    description="GPU partition type",
)

resource_type class-attribute instance-attribute

resource_type = Field(
    default=DEFAULT_ARGS["resource_type"]
    if DEFAULT_ARGS["resource_type"] != ""
    else None,
    description="Resource type",
)

exclude class-attribute instance-attribute

exclude = Field(
    default=DEFAULT_ARGS["exclude"],
    description="Exclude certain nodes from the resources granted to the job",
)

nodelist class-attribute instance-attribute

nodelist = Field(
    default=DEFAULT_ARGS["nodelist"],
    description="Request a specific list of nodes for deployment",
)

bind class-attribute instance-attribute

bind = Field(
    default=DEFAULT_ARGS["bind"],
    description="Additional binds for the container",
)

venv class-attribute instance-attribute

venv = Field(
    default=DEFAULT_ARGS["venv"],
    description="Virtual environment/container system",
)

log_dir class-attribute instance-attribute

log_dir = Field(
    default=Path(DEFAULT_ARGS["log_dir"]),
    description="Log directory path",
)

model_weights_parent_dir class-attribute instance-attribute

model_weights_parent_dir = Field(
    default=Path(DEFAULT_ARGS["model_weights_parent_dir"]),
    description="Base directory for model weights",
)

vllm_args class-attribute instance-attribute

vllm_args = Field(
    default={}, description="vLLM engine arguments"
)

env class-attribute instance-attribute

env = Field(
    default={},
    description="Environment variables to be set",
)

model_config class-attribute instance-attribute

model_config = ConfigDict(
    extra="forbid",
    str_strip_whitespace=True,
    validate_default=True,
    frozen=True,
)

Data Models

vec_inf.client.models

Data models for Vector Inference API.

This module contains the data model classes used by the Vector Inference API for both request parameters and response objects.

Classes:

Name	Description
ModelStatus : Enum	Status states of a model
ModelType : Enum	Types of supported models
LaunchResponse : dataclass	Response from model launch operation
StatusResponse : dataclass	Response from model status check
MetricsResponse : dataclass	Response from metrics collection
LaunchOptions : dataclass	Options for model launch
LaunchOptionsDict : TypedDict	Dictionary representation of launch options
ModelInfo : datacitten	Information about available models

ModelStatus

Bases: str, Enum

Enum representing the possible status states of a model.

Attributes:

Name	Type	Description
PENDING	str	Model is waiting for Slurm to allocate resources
LAUNCHING	str	Model is in the process of starting
READY	str	Model is running and ready to serve requests
FAILED	str	Model failed to start or encountered an error
SHUTDOWN	str	Model was intentionally stopped
UNAVAILABLE	str	Model status cannot be determined

Source code in vec_inf/client/models.py

class ModelStatus(str, Enum):
    """Enum representing the possible status states of a model.

    Attributes
    ----------
    PENDING : str
        Model is waiting for Slurm to allocate resources
    LAUNCHING : str
        Model is in the process of starting
    READY : str
        Model is running and ready to serve requests
    FAILED : str
        Model failed to start or encountered an error
    SHUTDOWN : str
        Model was intentionally stopped
    UNAVAILABLE : str
        Model status cannot be determined
    """

    PENDING = "PENDING"
    LAUNCHING = "LAUNCHING"
    READY = "READY"
    FAILED = "FAILED"
    SHUTDOWN = "SHUTDOWN"
    UNAVAILABLE = "UNAVAILABLE"

ModelType

Bases: str, Enum

Enum representing the possible model types.

Attributes:

Name	Type	Description
LLM	str	Large Language Model
VLM	str	Vision Language Model
TEXT_EMBEDDING	str	Text Embedding Model
REWARD_MODELING	str	Reward Modeling Model

Source code in vec_inf/client/models.py

class ModelType(str, Enum):
    """Enum representing the possible model types.

    Attributes
    ----------
    LLM : str
        Large Language Model
    VLM : str
        Vision Language Model
    TEXT_EMBEDDING : str
        Text Embedding Model
    REWARD_MODELING : str
        Reward Modeling Model
    """

    LLM = "LLM"
    VLM = "VLM"
    TEXT_EMBEDDING = "Text_Embedding"
    REWARD_MODELING = "Reward_Modeling"

LaunchResponse dataclass

Response from launching a model.

Parameters:

Name	Type	Description	Default
slurm_job_id	str	ID of the launched SLURM job	required
model_name	str	Name of the launched model	required
config	dict[str, Any]	Configuration used for the launch	required
raw_output	str	Raw output from the launch command (hidden from repr)	required

Source code in vec_inf/client/models.py

@dataclass
class LaunchResponse:
    """Response from launching a model.

    Parameters
    ----------
    slurm_job_id : str
        ID of the launched SLURM job
    model_name : str
        Name of the launched model
    config : dict[str, Any]
        Configuration used for the launch
    raw_output : str
        Raw output from the launch command (hidden from repr)
    """

    slurm_job_id: str
    model_name: str
    config: dict[str, Any]
    raw_output: str = field(repr=False)

BatchLaunchResponse dataclass

Response from launching multiple models in batch mode.

Parameters:

Name	Type	Description	Default
slurm_job_id	str	ID of the launched SLURM job	required
slurm_job_name	str	Name of the launched SLURM job	required
model_names	list[str]	Names of the launched models	required
config	dict[str, Any]	Configuration used for the launch	required
raw_output	str	Raw output from the launch command (hidden from repr)	required

Source code in vec_inf/client/models.py

@dataclass
class BatchLaunchResponse:
    """Response from launching multiple models in batch mode.

    Parameters
    ----------
    slurm_job_id : str
        ID of the launched SLURM job
    slurm_job_name : str
        Name of the launched SLURM job
    model_names : list[str]
        Names of the launched models
    config : dict[str, Any]
        Configuration used for the launch
    raw_output : str
        Raw output from the launch command (hidden from repr)
    """

    slurm_job_id: str
    slurm_job_name: str
    model_names: list[str]
    config: dict[str, Any]
    raw_output: str = field(repr=False)

StatusResponse dataclass

Response from checking a model's status.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model	required
log_dir	str	Path to the SLURM log directory	required
server_status	ModelStatus	Current status of the server	required
job_state	Union[str, ModelStatus]	Current state of the SLURM job	required
raw_output	str	Raw output from status check (hidden from repr)	required
base_url	str	Base URL of the model server if ready	None
pending_reason	str	Reason for pending state if applicable	None
failed_reason	str	Reason for failure if applicable	None

Source code in vec_inf/client/models.py

@dataclass
class StatusResponse:
    """Response from checking a model's status.

    Parameters
    ----------
    model_name : str
        Name of the model
    log_dir : str
        Path to the SLURM log directory
    server_status : ModelStatus
        Current status of the server
    job_state : Union[str, ModelStatus]
        Current state of the SLURM job
    raw_output : str
        Raw output from status check (hidden from repr)
    base_url : str, optional
        Base URL of the model server if ready
    pending_reason : str, optional
        Reason for pending state if applicable
    failed_reason : str, optional
        Reason for failure if applicable
    """

    model_name: str
    log_dir: str
    server_status: ModelStatus
    job_state: Union[str, ModelStatus]
    raw_output: str = field(repr=False)
    base_url: Optional[str] = None
    pending_reason: Optional[str] = None
    failed_reason: Optional[str] = None

MetricsResponse dataclass

Response from retrieving model metrics.

Parameters:

Name	Type	Description	Default
model_name	str	Name of the model	required
metrics	Union[dict[str, float], str]	Either a dictionary of metrics or an error message	required
timestamp	float	Unix timestamp of when metrics were collected	required

Source code in vec_inf/client/models.py

@dataclass
class MetricsResponse:
    """Response from retrieving model metrics.

    Parameters
    ----------
    model_name : str
        Name of the model
    metrics : Union[dict[str, float], str]
        Either a dictionary of metrics or an error message
    timestamp : float
        Unix timestamp of when metrics were collected
    """

    model_name: str
    metrics: Union[dict[str, float], str]
    timestamp: float

LaunchOptions dataclass

Options for launching a model.

Parameters:

Name	Type	Description	Default
model_family	str	Family/architecture of the model	None
model_variant	str	Specific variant/version of the model	None
partition	str	SLURM partition to use	None
resource_type	str	Type of resource to request for the job	None
num_nodes	int	Number of nodes to allocate	None
gpus_per_node	int	Number of GPUs per node	None
account	str	Account name for job scheduling	None
work_dir	str	Set working directory for the batch job	None
qos	str	Quality of Service level	None
time	str	Time limit for the job	None
exclude	str	Exclude certain nodes from the resources granted to the job	None
node_list	str	Request a specific list of nodes for deployment	required
bind	str	Additional binds for the container as a comma separated list of bind paths	None
vocab_size	int	Size of model vocabulary	None
data_type	str	Data type for model weights	None
venv	str	Virtual environment to use	None
log_dir	str	Directory for logs	None
model_weights_parent_dir	str	Parent directory containing model weights	None
vllm_args	str	Additional arguments for vLLM	None
env	str	Environment variables to be set	None
config	str	Path to custom model config yaml	None

Source code in vec_inf/client/models.py

@dataclass
class LaunchOptions:
    """Options for launching a model.

    Parameters
    ----------
    model_family : str, optional
        Family/architecture of the model
    model_variant : str, optional
        Specific variant/version of the model
    partition : str, optional
        SLURM partition to use
    resource_type : str, optional
        Type of resource to request for the job
    num_nodes : int, optional
        Number of nodes to allocate
    gpus_per_node : int, optional
        Number of GPUs per node
    account : str, optional
        Account name for job scheduling
    work_dir : str, optional
        Set working directory for the batch job
    qos : str, optional
        Quality of Service level
    time : str, optional
        Time limit for the job
    exclude : str, optional
        Exclude certain nodes from the resources granted to the job
    node_list : str, optional
        Request a specific list of nodes for deployment
    bind : str, optional
        Additional binds for the container as a comma separated list of bind paths
    vocab_size : int, optional
        Size of model vocabulary
    data_type : str, optional
        Data type for model weights
    venv : str, optional
        Virtual environment to use
    log_dir : str, optional
        Directory for logs
    model_weights_parent_dir : str, optional
        Parent directory containing model weights
    vllm_args : str, optional
        Additional arguments for vLLM
    env : str, optional
        Environment variables to be set
    config : str, optional
        Path to custom model config yaml
    """

    model_family: Optional[str] = None
    model_variant: Optional[str] = None
    partition: Optional[str] = None
    resource_type: Optional[str] = None
    num_nodes: Optional[int] = None
    gpus_per_node: Optional[int] = None
    account: Optional[str] = None
    work_dir: Optional[str] = None
    qos: Optional[str] = None
    exclude: Optional[str] = None
    nodelist: Optional[str] = None
    bind: Optional[str] = None
    time: Optional[str] = None
    vocab_size: Optional[int] = None
    data_type: Optional[str] = None
    venv: Optional[str] = None
    log_dir: Optional[str] = None
    model_weights_parent_dir: Optional[str] = None
    vllm_args: Optional[str] = None
    env: Optional[str] = None
    config: Optional[str] = None

ModelInfo dataclass

Information about an available model.

Parameters:

Name	Type	Description	Default
name	str	Name of the model	required
family	str	Family/architecture of the model	required
variant	str	Specific variant/version of the model	required
model_type	ModelType	Type of the model	required
config	dict[str, Any]	Additional configuration parameters	required

Source code in vec_inf/client/models.py

@dataclass
class ModelInfo:
    """Information about an available model.

    Parameters
    ----------
    name : str
        Name of the model
    family : str
        Family/architecture of the model
    variant : str, optional
        Specific variant/version of the model
    model_type : ModelType
        Type of the model
    config : dict[str, Any]
        Additional configuration parameters
    """

    name: str
    family: str
    variant: Optional[str]
    model_type: ModelType
    config: dict[str, Any]