ci.load

def load(
    name: str | list[str],
    *,
    source: Literal["local", "platform"] = "local",
    match: str | dict | None = None,
    ext: list[str] | None = None,
    columns: list[str] | None = None,
    map: Callable | None = None,
    where: str | None = None,
    as_: Literal["pandas", "polars", "iter", "tensor", "hf"] = "pandas",
    lazy: bool = False,
    batch_size: int = 10_000,
    confirm_large: bool = False,
) -> Any

# Glob string: simplest form
ci.load("s3://bucket/", match="year=2025/month=*/*.parquet")

# Structured dict: separate path glob, filename regex, column pushdown
ci.load("s3://bucket/", match={
    "path":     "year=2025/month=*/",
    "filename": r"events_.*\.parquet",
    "columns":  ["user_id", "ts", "event_type"],
})

# Row-wise: plain callable, receives one row at a time
ci.load("./raw/", map=lambda row: {"text": row["raw"].lower()})

# Batch-wise: decorate with @ci.map, receives the full frame
@ci.map
def to_features(frame):
    frame["text"] = frame["raw"].str.lower()
    return frame

ci.load("./raw/", map=to_features)

# Local
df = ci.load("./data/events.parquet")
df = ci.load("training-data")                 # probes ./training-data/, ./data/training-data/

# Multi-source: parallel load, concatenate
df = ci.load(["./a/", "./b/"])

# Cloud
df = ci.load("s3://ml-data/events/", match="year=2025/month=*/*.parquet")
df = ci.load("gs://bucket/events/", ext=["parquet"], columns=["user_id", "ts"])

# SQL
df = ci.load("postgres://prod/events", where="created_at > '2025-01-01'")
df = ci.load("snowflake://wh/db/schema/table", where="region = 'EMEA' LIMIT 100000")

# Platform-registered
df = ci.load("bucket1", source="platform")

# Polars + lazy
handle = ci.load("./events.parquet", as_="polars", lazy=True)
out = handle.collect().filter(pl.col("label") == 1)