Skip to content

graflo.object_storage

S3-compatible object storage helpers (boto3): config, client, bucket ensure.

This package is separate from graph :class:~graflo.db.ConnectionManager targets. Use :class:~graflo.object_storage.config.S3EndpointConfig; its :meth:~graflo.object_storage.config.S3EndpointConfig.from_docker_env reads the bundled S3 service's settings, as Neo4jConfig.from_docker_env does.

Modules:

Name Description
bucket

Ensure S3 buckets exist (idempotent).

config

S3-compatible endpoint configuration (SeaweedFS, AWS S3, MinIO, etc.).

s3_client

Shared boto3 S3 client construction.

Attributes

__all__ = ['S3EndpointConfig', 'boto3_s3_client', 'boto3_s3_client_from_endpoint', 'ensure_bucket_exists', 'ensure_staging_bucket_for_config'] module-attribute

Classes

S3EndpointConfig

Bases: BaseModel

Credentials and endpoint for any S3-compatible API (boto3).

Source code in graflo/object_storage/config.py
class S3EndpointConfig(BaseModel):
    """Credentials and endpoint for any S3-compatible API (boto3)."""

    model_config = ConfigDict(extra="forbid")

    endpoint_url: str = Field(
        description="Base URL for the S3 API, e.g. http://127.0.0.1:8333",
    )
    access_key: str = Field(description="S3 access key id.")
    secret_key: str = Field(description="S3 secret access key.")
    bucket: str = Field(
        default="graflo-staging",
        description="Default bucket.",
    )
    region: str = Field(
        default="us-east-1",
        description="AWS region string passed to boto3 (us-east-1 for a local S3 service).",
    )

    def to_files_conn_config(self) -> FilesConnConfig:
        """The files connection of :attr:`bucket`, as a connection provider stores it."""
        from graflo.connections.sources import FilesConnConfig

        return FilesConnConfig(
            root=f"s3://{self.bucket}",
            region=self.region,
            endpoint_url=self.endpoint_url,
            access_key_id=self.access_key,
            secret_access_key=self.secret_key,
        )

    @classmethod
    def from_docker_env(cls, docker_dir: str | Path | None = None) -> S3EndpointConfig:
        """Load the S3 service settings of ``docker/seaweedfs``.

        Reads ``<docker_dir>/.env.example`` when it exists; every setting has a default
        matching the bundled compose file, whose S3 API accepts any key pair, so
        the file is optional.

        Settings: ``S3_ENDPOINT`` (or ``S3_PROTOCOL``, ``S3_HOSTNAME``,
        ``S3_API_PORT``), ``S3_ACCESS_KEY``, ``S3_SECRET_KEY``, ``S3_BUCKET``
        and ``AWS_REGION``.
        """
        from graflo.connections.env import docker_env

        env_vars = docker_env("seaweedfs", docker_dir, required=False)
        data: dict[str, Any] = {}

        if raw_endpoint := env_vars.get("S3_ENDPOINT"):
            data["endpoint_url"] = raw_endpoint
        else:
            host = env_vars.get("S3_HOSTNAME", "127.0.0.1")
            port = env_vars.get("S3_API_PORT", "8333")
            protocol = env_vars.get("S3_PROTOCOL", "http")
            data["endpoint_url"] = f"{protocol}://{host}:{port}"

        data["access_key"] = env_vars.get("S3_ACCESS_KEY", "graflo")
        data["secret_key"] = env_vars.get("S3_SECRET_KEY", "graflo-secret")

        if bucket := env_vars.get("S3_BUCKET"):
            data["bucket"] = bucket
        if region := env_vars.get("AWS_REGION"):
            data["region"] = region

        return cls(**data)

Attributes

access_key = Field(description='S3 access key id.') class-attribute instance-attribute
bucket = Field(default='graflo-staging', description='Default bucket.') class-attribute instance-attribute
endpoint_url = Field(description='Base URL for the S3 API, e.g. http://127.0.0.1:8333') class-attribute instance-attribute
model_config = ConfigDict(extra='forbid') class-attribute instance-attribute
region = Field(default='us-east-1', description='AWS region string passed to boto3 (us-east-1 for a local S3 service).') class-attribute instance-attribute
secret_key = Field(description='S3 secret access key.') class-attribute instance-attribute

Methods:

from_docker_env(docker_dir=None) classmethod

Load the S3 service settings of docker/seaweedfs.

Reads <docker_dir>/.env.example when it exists; every setting has a default matching the bundled compose file, whose S3 API accepts any key pair, so the file is optional.

Settings: S3_ENDPOINT (or S3_PROTOCOL, S3_HOSTNAME, S3_API_PORT), S3_ACCESS_KEY, S3_SECRET_KEY, S3_BUCKET and AWS_REGION.

Source code in graflo/object_storage/config.py
@classmethod
def from_docker_env(cls, docker_dir: str | Path | None = None) -> S3EndpointConfig:
    """Load the S3 service settings of ``docker/seaweedfs``.

    Reads ``<docker_dir>/.env.example`` when it exists; every setting has a default
    matching the bundled compose file, whose S3 API accepts any key pair, so
    the file is optional.

    Settings: ``S3_ENDPOINT`` (or ``S3_PROTOCOL``, ``S3_HOSTNAME``,
    ``S3_API_PORT``), ``S3_ACCESS_KEY``, ``S3_SECRET_KEY``, ``S3_BUCKET``
    and ``AWS_REGION``.
    """
    from graflo.connections.env import docker_env

    env_vars = docker_env("seaweedfs", docker_dir, required=False)
    data: dict[str, Any] = {}

    if raw_endpoint := env_vars.get("S3_ENDPOINT"):
        data["endpoint_url"] = raw_endpoint
    else:
        host = env_vars.get("S3_HOSTNAME", "127.0.0.1")
        port = env_vars.get("S3_API_PORT", "8333")
        protocol = env_vars.get("S3_PROTOCOL", "http")
        data["endpoint_url"] = f"{protocol}://{host}:{port}"

    data["access_key"] = env_vars.get("S3_ACCESS_KEY", "graflo")
    data["secret_key"] = env_vars.get("S3_SECRET_KEY", "graflo-secret")

    if bucket := env_vars.get("S3_BUCKET"):
        data["bucket"] = bucket
    if region := env_vars.get("AWS_REGION"):
        data["region"] = region

    return cls(**data)
to_files_conn_config()

The files connection of :attr:bucket, as a connection provider stores it.

Source code in graflo/object_storage/config.py
def to_files_conn_config(self) -> FilesConnConfig:
    """The files connection of :attr:`bucket`, as a connection provider stores it."""
    from graflo.connections.sources import FilesConnConfig

    return FilesConnConfig(
        root=f"s3://{self.bucket}",
        region=self.region,
        endpoint_url=self.endpoint_url,
        access_key_id=self.access_key,
        secret_access_key=self.secret_key,
    )

Functions:

boto3_s3_client(cfg)

Build a boto3 S3 client from :class:~graflo.connections.sources.S3ConnConfig.

Source code in graflo/object_storage/s3_client.py
def boto3_s3_client(cfg: S3ConnConfig) -> Any:
    """Build a boto3 S3 client from :class:`~graflo.connections.sources.S3ConnConfig`."""
    import boto3

    return boto3.client(
        "s3",
        region_name=cfg.region or None,
        endpoint_url=cfg.endpoint_url or None,
        aws_access_key_id=cfg.aws_access_key_id or None,
        aws_secret_access_key=cfg.aws_secret_access_key or None,
    )

boto3_s3_client_from_endpoint(cfg)

Build a boto3 S3 client from :class:~graflo.object_storage.config.S3EndpointConfig.

Source code in graflo/object_storage/s3_client.py
def boto3_s3_client_from_endpoint(cfg: S3EndpointConfig) -> Any:
    """Build a boto3 S3 client from :class:`~graflo.object_storage.config.S3EndpointConfig`."""
    import boto3

    return boto3.client(
        "s3",
        region_name=cfg.region or None,
        endpoint_url=cfg.endpoint_url or None,
        aws_access_key_id=cfg.access_key,
        aws_secret_access_key=cfg.secret_key,
    )

ensure_bucket_exists(client, bucket)

Create bucket if it does not exist. Return bucket.

Safe to call repeatedly. Raises :exc:botocore.exceptions.ClientError for non-recoverable API errors.

Source code in graflo/object_storage/bucket.py
def ensure_bucket_exists(client: Any, bucket: str) -> str:
    """Create *bucket* if it does not exist. Return *bucket*.

    Safe to call repeatedly. Raises :exc:`botocore.exceptions.ClientError`
    for non-recoverable API errors.
    """
    try:
        client.head_bucket(Bucket=bucket)
        logger.info("S3 bucket %r already exists", bucket)
        return bucket
    except ClientError as e:
        code = str(e.response.get("Error", {}).get("Code", ""))
        http = e.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
        missing = code in ("404", "NoSuchBucket", "NotFound") or http == 404
        if not missing:
            raise

    try:
        client.create_bucket(Bucket=bucket)
        logger.info("Created S3 bucket %r", bucket)
    except ClientError as e:
        code = str(e.response.get("Error", {}).get("Code", ""))
        if code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
            logger.info(
                "S3 bucket %r already exists (race or concurrent create)", bucket
            )
        else:
            raise

    return bucket

ensure_staging_bucket_for_config(cfg=None)

Ensure :attr:~graflo.object_storage.config.S3EndpointConfig.bucket exists.

If cfg is None, loads :meth:~graflo.object_storage.config.S3EndpointConfig.from_docker_env.

Source code in graflo/object_storage/bucket.py
def ensure_staging_bucket_for_config(cfg: S3EndpointConfig | None = None) -> str:
    """Ensure :attr:`~graflo.object_storage.config.S3EndpointConfig.bucket` exists.

    If *cfg* is ``None``, loads :meth:`~graflo.object_storage.config.S3EndpointConfig.from_docker_env`.
    """
    from graflo.object_storage.config import S3EndpointConfig
    from graflo.object_storage.s3_client import boto3_s3_client_from_endpoint

    resolved = cfg or S3EndpointConfig.from_docker_env()
    client = boto3_s3_client_from_endpoint(resolved)
    try:
        return ensure_bucket_exists(client, resolved.bucket)
    except EndpointConnectionError as e:
        raise RuntimeError(
            f"Cannot connect to {resolved.endpoint_url!r}. "
            "Start the S3 service (e.g. `docker compose --profile graflo.seaweedfs up -d` "
            "in `docker/seaweedfs`) and ensure `S3_API_PORT` / `S3_ENDPOINT` matches the "
            "published S3 API port (default host 8333)."
        ) from e