polars-redis

The Problem

# Typical pattern: Fetch everything, process in app
keys = redis.scan_iter("user:*")                      # Iterate all keys
users = []
for key in keys:
    data = redis.hgetall(key)                         # N+1 queries
    users.append(data)

df = pd.DataFrame(users)                              # Manual conversion
df["age"] = df["age"].astype(int)                     # Manual type coercion
df["score"] = df["score"].astype(float)
active = df[df["status"] == "active"]                 # Filter in Python
result = active.groupby("department")["score"].mean() # Aggregate in Python

• N+1 queries for N keys
• Manual type conversion
• All data transferred, then filtered client-side
• No native DataFrame integration

The Solution

import polars as pl
import polars_redis as pr

# One call, proper types, batched operations
df = pr.scan_hashes(
    "redis://localhost:6379",
    "user:*",
    schema={"name": "utf8", "age": "int64", "score": "float64", "department": "utf8"}
).collect()

# With RediSearch: filter in Redis, not Python
df = pr.search_hashes(
    "redis://localhost:6379",
    index="users_idx",
    query="@status:{active}",
    schema={"name": "utf8", "department": "utf8", "score": "float64"}
)

• Batched async operations (configurable batch size)
• Automatic schema inference or explicit types
• Server-side filtering with RediSearch
• Native Polars LazyFrame integration

What is Polars?

pola.rs - A lightning-fast DataFrame library for Python and Rust

import polars as pl

# Read, transform, write
df = pl.read_csv("data.csv")
result = (
    df.filter(pl.col("age") > 25)
      .group_by("department")
      .agg(pl.col("salary").mean())
      .sort("salary", descending=True)
)
result.write_parquet("output.parquet")

Learn more: docs.pola.rs - Polars User Guide

What is polars-redis?

Scanning Hashes

# Basic scan with explicit schema
df = pr.scan_hashes(
    "redis://localhost:6379",
    "product:*",
    schema={
        "name": "utf8",
        "price": "float64",
        "quantity": "int64",
        "in_stock": "bool"
    }
).collect()

# With schema inference (samples keys to detect types)
df = pr.scan_hashes(
    "redis://localhost:6379",
    "product:*",
    infer_schema=True,
    infer_schema_length=100  # Sample 100 keys
).collect()

# Include the Redis key as a column
df = pr.scan_hashes(
    "redis://localhost:6379",
    "product:*",
    schema={"name": "utf8", "price": "float64"},
    with_key=True,
    key_column="_key"
).collect()

Schema Inference

# Let polars-redis figure out the types
df = pr.scan_hashes(
    "redis://localhost:6379",
    "event:*",
    infer_schema=True,
    infer_schema_length=50  # Sample size
).collect()

Int64    "42", "-100", "0"
Float64  "3.14", "-0.5", "1e10"
Bool     "true", "false", "1", "0"
Utf8     everything else

schema = {
    "id": "int64",
    "name": "utf8",
    "score": "float64",
    "active": "bool"
}

Tip: Use explicit schema in production for consistency

Writing Data

# Create a DataFrame
df = pl.DataFrame({
    "id": [1, 2, 3],
    "name": ["alice", "bob", "charlie"],
    "score": [95.5, 87.3, 92.1],
    "active": [True, True, False]
})

# Write as Redis hashes
pr.write_hashes(
    "redis://localhost:6379",
    df,
    key_column="id",       # Use 'id' column for key generation
    key_prefix="user:"     # Creates user:1, user:2, user:3
)

# Verify
redis-cli HGETALL user:1
# 1) "name"
# 2) "alice"
# 3) "score"
# 4) "95.5"
# 5) "active"
# 6) "true"

RediSearch: Server-Side Filtering

# First, create an index (one-time setup)
FT.CREATE users_idx ON HASH PREFIX 1 user: SCHEMA \
    name TEXT SORTABLE \
    age NUMERIC SORTABLE \
    department TAG \
    salary NUMERIC

# Query with predicate pushdown - filtering happens in Redis
df = pr.search_hashes(
    "redis://localhost:6379",
    index="users_idx",
    query="@age:[25 50] @department:{engineering|product}",
    schema={"name": "utf8", "age": "int64", "department": "utf8", "salary": "float64"}
)

# Only matching documents are transferred!
# Compare to scanning ALL user:* keys and filtering in Python

• Numeric ranges: @age:[25 50]
• Tag filters: @department:{engineering}
• Full-text: @name:alice
• Negation: -@status:{inactive}

Query Builder: Polars-like Syntax

from polars_redis import col, search_hashes

# Build queries with familiar Polars-like syntax
# Instead of learning RediSearch query language:
query = (col("age") > 30) & (col("status") == "active")

# Use directly in search_hashes
df = search_hashes(
    "redis://localhost:6379",
    index="users_idx",
    query=query,  # Automatically converts to "@age:[(30 +inf] @status:{active}"
    schema={"name": pl.Utf8, "age": pl.Int64, "status": pl.Utf8}
)

# Complex queries are easy to build
query = (
    (col("age").is_between(25, 50)) &
    (col("department").has_any_tag(["engineering", "product"])) &
    ~(col("status") == "inactive")  # NOT
)

Result: Automatic translation to RediSearch syntax with predicate pushdown

Query Builder: Full API

# Numeric comparisons
col("age") > 30      # @age:[(30 +inf]
col("age") >= 30     # @age:[30 +inf]
col("age") < 30      # @age:[-inf (30]
col("price").is_between(10, 100)

# Equality
col("status") == "active"
col("status") != "deleted"

# Membership
col("role").is_in(["admin", "user"])

# Text search
col("title").contains("python")
col("name").starts_with("jo")
col("name").ends_with("son")
col("name").fuzzy("john", 1)
col("title").phrase("hello", "world")

# Tags (exact match)
col("category").has_tag("science")
col("tags").has_any_tag(["a", "b"])

# Geo
col("loc").within_radius(
    -122.4, 37.7, 10, "km"
)

# Logical operators: & (AND), | (OR), ~ (NOT)
query = (col("age") > 30) & ((col("dept") == "eng") | (col("dept") == "prod")) & ~(col("status") == "deleted")

RediSearch: Server-Side Aggregation

# Aggregate in Redis, not Python
df = pr.aggregate_hashes(
    "redis://localhost:6379",
    index="users_idx",
    query="*",
    group_by=["department"],
    reduce=[
        ("COUNT", [], "employee_count"),
        ("AVG", ["@salary"], "avg_salary"),
        ("MAX", ["@salary"], "max_salary"),
        ("SUM", ["@salary"], "total_payroll")
    ],
    sort_by=[("avg_salary", False)],  # Descending
    limit=10
)

# Result
shape: (3, 5)
+--------------+----------------+------------+------------+---------------+
| department   | employee_count | avg_salary | max_salary | total_payroll |
+--------------+----------------+------------+------------+---------------+
| engineering  | 45             | 125000.0   | 180000.0   | 5625000.0     |
| product      | 20             | 115000.0   | 160000.0   | 2300000.0     |
| marketing    | 15             | 95000.0    | 140000.0   | 1425000.0     |
+--------------+----------------+------------+------------+---------------+

FT.AGGREGATE: Advanced Features

# Computed fields with APPLY
df = pr.aggregate_hashes(
    "redis://localhost:6379",
    index="sales_idx",
    query="@date:[2024010100 2024123123]",
    group_by=["product_category"],
    reduce=[
        ("COUNT", [], "orders"),
        ("SUM", ["@amount"], "revenue")
    ],
    apply=[
        ("@revenue / @orders", "avg_order_value")  # Computed field
    ],
    filter_expr="@orders > 100",  # Post-aggregation filter
    sort_by=[("revenue", False)]
)

# Global aggregation (no grouping)
df = pr.aggregate_hashes(
    "redis://localhost:6379",
    index="users_idx",
    query="@status:{active}",
    reduce=[
        ("COUNT", [], "total_users"),
        ("AVG", ["@age"], "avg_age")
    ]
)
# Returns single row with totals

RedisJSON Support

# Scan JSON documents
df = pr.scan_json(
    "redis://localhost:6379",
    "event:*",
    schema={
        "$.timestamp": "utf8",
        "$.user.id": "int64",
        "$.user.name": "utf8",
        "$.data.value": "float64",
        "$.tags": "utf8"  # Arrays become JSON strings
    },
    json_path="$"
).collect()

# Write DataFrame as JSON documents
pr.write_json(
    "redis://localhost:6379",
    df,
    key_column="id",
    key_prefix="event:"
)

Note: RediSearch also indexes JSON documents with ON JSON

Performance Features

# Configure batch size
df = pr.scan_hashes(
    "redis://localhost:6379",
    "user:*",
    schema={"name": "utf8"},
    batch_size=1000,   # Keys per batch
    scan_count=100     # SCAN COUNT hint
).collect()

# With SCAN: transfer all, filter client
df = pr.scan_hashes(url, "user:*", schema)
df = df.filter(pl.col("age") > 25)

# With RediSearch: filter in Redis
df = pr.search_hashes(
    url, index="users_idx",
    query="@age:[25 +inf]",
    schema=schema
)

Redis Cluster Support

# Connect to Redis Cluster (keys distributed across nodes)
from polars_redis import ClusterHashBatchIterator

# Provide any cluster node URLs - auto-discovers all nodes
nodes = ["redis://node1:7000", "redis://node2:7001", "redis://node3:7002"]

iterator = ClusterHashBatchIterator(
    nodes,
    schema=HashSchema(...),
    config=BatchConfig(pattern="user:*", batch_size=1000)
)

# Automatically scans ALL cluster nodes
# Keys are distributed by slot, iterator handles routing

Use Case: Ephemeral Data Workbench

# Redis as a fast, queryable staging area
# 1. Load data from any source
df = pl.read_csv("sales_data.csv")
pr.write_hashes("redis://localhost", df, key_column="id", key_prefix="sale:")

# 2. Create index for fast queries
# FT.CREATE sales_idx ON HASH PREFIX 1 sale: SCHEMA ...

# 3. Query and analyze interactively
results = pr.aggregate_hashes(
    "redis://localhost",
    index="sales_idx",
    query="@region:{west}",
    group_by=["product"],
    reduce=[("SUM", ["@amount"], "total")]
)

# 4. TTL for automatic cleanup (set in Redis)
# EXPIRE sale:* 3600

• Fast loading: Write DataFrames directly to Redis
• Interactive queries: RediSearch for instant filtering
• Auto-cleanup: TTL-based expiration
• Ideal for: ETL staging, data exploration, session analytics

Architecture

  +------------------+     +--------------------+     +------------------+
  |  Python/Polars   |     |    polars-redis    |     |      Redis       |
  |                  |     |    (Rust + PyO3)   |     |                  |
  +------------------+     +--------------------+     +------------------+
          |                         |                         |
          v                         v                         v
     LazyFrame API           Async batching              SCAN + HGETALL
     DataFrame ops           Schema inference            FT.SEARCH
     .collect()              Arrow conversion            FT.AGGREGATE
                             Pipeline commands           JSON.GET/SET

  +--------------------------------------------------------------------------+
  |                           Data Flow                                       |
  |  scan_hashes() -> SCAN keys -> batch HGETALL -> parse -> Arrow -> Polars |
  |  search_hashes() -> FT.SEARCH -> parse results -> Arrow -> Polars        |
  |  write_hashes() -> Polars -> rows -> batch HSET pipeline                 |
  +--------------------------------------------------------------------------+

Rust API

use polars_redis::{scan_hashes, HashSchema, BatchConfig, RedisType};

// Define schema
let schema = HashSchema::new()
    .with_field("name", RedisType::Utf8)
    .with_field("age", RedisType::Int64)
    .with_field("score", RedisType::Float64)
    .with_key(true)
    .with_key_column_name("_key");

// Configure batching
let config = BatchConfig::default()
    .with_batch_size(1000)
    .with_scan_count(100);

// Scan to DataFrame
let df = scan_hashes(
    "redis://localhost:6379",
    "user:*",
    schema,
    config
).await?;

println!("{}", df);

Full async/await support with tokio runtime

Installation

# From PyPI
pip install polars-redis

# With RediSearch support
pip install polars-redis[search]

# From source
git clone https://github.com/joshrotenberg/polars-redis
cd polars-redis
pip install maturin
maturin develop --features search

# Cargo.toml
[dependencies]
polars-redis = "0.1"

# With RediSearch support
polars-redis = {
    version = "0.1",
    features = ["search"]
}

Requirements

# Redis Stack with Docker
docker run -d \
  --name redis-stack \
  -p 6379:6379 \
  redis/redis-stack:latest

# Verify RediSearch
redis-cli MODULE LIST
# 1) "name" "search"
# 2) "name" "ReJSON"

API Summary

# Scanning
pr.scan_hashes(url, pattern, schema, ...)      # Scan hash keys to LazyFrame
pr.scan_json(url, pattern, schema, ...)        # Scan JSON keys to LazyFrame
pr.scan_strings(url, pattern, ...)             # Scan string keys to LazyFrame

# RediSearch with Query Builder
from polars_redis import col, search_hashes
query = (col("age") > 30) & (col("status") == "active")
pr.search_hashes(url, index, query, schema)    # Predicate pushdown!
pr.aggregate_hashes(url, index, query, ...)    # Server-side aggregation

# Writing
pr.write_hashes(df, url, key_column, ...)      # DataFrame to hashes
pr.write_json(df, url, key_column, ...)        # DataFrame to JSON

# Redis Cluster
ClusterHashBatchIterator(nodes, schema, config)  # Scan across cluster nodes
ClusterStringBatchIterator(nodes, schema, config)

Resources