Polars Study Guide

# pip install polars
import polars as pl
import numpy as np

# From dict
df = pl.DataFrame({
    'name': ['Alice', 'Bob', 'Carol', 'Dave'],
    'age': [25, 30, 35, 28],
    'score': [92.5, 88.0, 95.1, 76.3],
    'active': [True, False, True, True]
})
print(df)

# Schema info
print(df.schema)
print(df.dtypes)
print(df.shape)

import polars as pl

# Series
s = pl.Series('values', [10, 20, 30, 40, 50])
print(s)
print('dtype:', s.dtype)
print('mean:', s.mean())

# Explicit dtypes
df = pl.DataFrame({
    'id': pl.Series([1, 2, 3], dtype=pl.Int32),
    'price': pl.Series([9.99, 14.99, 4.99], dtype=pl.Float32),
    'label': pl.Series(['a', 'b', 'c'], dtype=pl.Categorical),
})
print(df)
print(df.schema)

# Quick summary
df2 = pl.DataFrame({'x': range(100), 'y': [i**2 for i in range(100)]})
print(df2.describe())

import polars as pl
import io

# From a CSV string (in-memory)
csv_data = 'name,age,score\nAlice,25,92.5\nBob,30,88.0\nCarol,35,95.1'
df_csv = pl.read_csv(io.StringIO(csv_data))
print('From CSV string:')
print(df_csv)

# From list of dicts
records = [
    {'product': 'Widget', 'price': 9.99, 'qty': 100},
    {'product': 'Gadget', 'price': 24.99, 'qty': 50},
    {'product': 'Doohickey', 'price': 4.99, 'qty': 200},
]
df_dicts = pl.DataFrame(records)
print('From list of dicts:')
print(df_dicts)
print('Schema:', df_dicts.schema)

import polars as pl
import numpy as np

# pl.from_records — list of tuples with explicit schema
records = [(1, 'Alice', 92.5), (2, 'Bob', 88.0), (3, 'Carol', 95.1)]
schema = {'id': pl.Int32, 'name': pl.Utf8, 'score': pl.Float32}
df_rec = pl.from_records(records, schema=schema)
print('from_records:')
print(df_rec)
print('dtypes:', df_rec.dtypes)

# pl.from_arrow — zero-copy from PyArrow table
try:
    import pyarrow as pa
    arrow_table = pa.table({'x': [10, 20, 30], 'y': [1.1, 2.2, 3.3]})
    df_arrow = pl.from_arrow(arrow_table)
    print('from_arrow:', df_arrow)
except ImportError:
    print('pyarrow not installed — skipping from_arrow demo')

# Schema enforcement with dtypes dict
np.random.seed(42)
df_typed = pl.DataFrame(
    {
        'user_id': np.random.randint(1, 1000, 5).tolist(),
        'amount': np.random.uniform(10, 500, 5).tolist(),
        'category': ['food', 'tech', 'food', 'travel', 'tech'],
    },
    schema={'user_id': pl.Int32, 'amount': pl.Float64, 'category': pl.Categorical}
)
print('Schema-enforced DataFrame:')
print(df_typed)
print('Schema:', df_typed.schema)

import polars as pl

df = pl.DataFrame({
    'name': ['Alice', 'Bob', 'Carol', 'Dave', 'Eve'],
    'salary': [60000, 75000, 90000, 55000, 110000],
    'dept': ['Eng', 'Sales', 'Eng', 'HR', 'Eng'],
    'years': [3, 5, 8, 2, 12]
})

# select: pick & transform columns
print(df.select(['name', 'salary']))

# with_columns: add new columns
df2 = df.with_columns([
    (pl.col('salary') * 1.1).alias('new_salary'),
    (pl.col('salary') / pl.col('years')).alias('salary_per_year'),
    pl.col('name').str.to_uppercase().alias('name_upper')
])
print(df2)

import polars as pl

df = pl.DataFrame({
    'x': [1, 2, 3, 4, 5],
    'y': [10, 20, 30, 40, 50],
    'label': ['a', 'b', 'a', 'b', 'a']
})

# Expression chaining
result = df.select([
    pl.col('x'),
    pl.col('y'),
    (pl.col('x') + pl.col('y')).alias('sum'),
    (pl.col('y') / pl.col('x')).alias('ratio').round(2),
    pl.col('x').pow(2).alias('x_squared'),
    pl.col('label').is_in(['a']).alias('is_a')
])
print(result)

# Horizontal operations
df2 = df.with_columns(
    pl.max_horizontal('x', 'y').alias('row_max'),
    pl.sum_horizontal('x', 'y').alias('row_sum')
)
print(df2)

import polars as pl

df = pl.DataFrame({
    'name': ['Alice', 'Bob', 'Carol', 'Dave', 'Eve'],
    'score': [92, 45, 78, 55, 88],
    'attempts': [1, 3, 2, 4, 1]
})

# pl.when().then().otherwise() — like np.where but composable
result = df.with_columns([
    pl.when(pl.col('score') >= 90)
      .then(pl.lit('A'))
      .when(pl.col('score') >= 70)
      .then(pl.lit('B'))
      .when(pl.col('score') >= 55)
      .then(pl.lit('C'))
      .otherwise(pl.lit('F'))
      .alias('grade'),
    pl.when(pl.col('attempts') == 1)
      .then(pl.lit('First try'))
      .otherwise(pl.lit('Retried'))
      .alias('attempt_label')
])
print(result)

import polars as pl

df = pl.DataFrame({
    'first': ['Alice', 'Bob', 'Carol'],
    'last': ['Smith', 'Jones', 'White'],
    'score_a': [85, 90, 78],
    'score_b': [92, 88, 95],
})

# pl.struct — pack multiple cols into a struct column
df2 = df.with_columns(
    pl.struct(['first', 'last']).alias('full_name_struct'),
    pl.struct(['score_a', 'score_b']).alias('scores_struct')
)
print('With struct columns:')
print(df2.select(['full_name_struct', 'scores_struct']))

# unnest — expand a struct back to individual columns
df3 = df2.select('scores_struct').unnest('scores_struct')
print('Unnested scores:')
print(df3)

# pl.concat_list — combine columns into a list column
df4 = df.with_columns(
    pl.concat_list(['score_a', 'score_b']).alias('all_scores')
)
print('concat_list result:')
print(df4.select(['first', 'all_scores']))

# pl.col('*') — select all columns at once
df5 = df.select(pl.col('*').exclude(['first', 'last']))
print('All numeric cols via col(*).exclude:')
print(df5)

import polars as pl

df = pl.DataFrame({
    'name': ['Alice', 'Bob', 'Carol', 'Dave', 'Eve', 'Frank'],
    'age': [25, 42, 35, 19, 55, 30],
    'city': ['NY', 'LA', 'NY', 'Chicago', 'LA', 'NY'],
    'salary': [70000, 95000, 80000, 45000, 120000, 65000]
})

# Single condition
print(df.filter(pl.col('age') > 30))

# Multiple conditions (AND)
print(df.filter(
    (pl.col('city') == 'NY') & (pl.col('salary') > 65000)
))

# OR condition
print(df.filter(
    (pl.col('city') == 'LA') | (pl.col('age') < 25)
))

# is_in
print(df.filter(pl.col('city').is_in(['NY', 'LA'])))

import polars as pl

df = pl.DataFrame({
    'product': ['A', 'B', 'C', 'D', 'E'],
    'sales': [300, 150, 450, 280, 100],
    'region': ['East', 'West', 'East', 'West', 'East'],
    'profit': [45.0, 30.0, 90.0, 55.0, 10.0]
})

# Sort single column
print(df.sort('sales', descending=True))

# Sort multiple columns
print(df.sort(['region', 'sales'], descending=[False, True]))

# top_k — faster than sort for large data
print(df.top_k(3, by='profit'))

# Null handling in sort
df2 = pl.DataFrame({'val': [3, None, 1, None, 2]})
print(df2.sort('val', nulls_last=True))

import polars as pl

df = pl.DataFrame({
    'product': ['Widget Pro', 'Gadget Mini', 'Widget Lite', 'Super Gadget', 'Widget Max'],
    'price': [29.99, 9.99, 14.99, 49.99, 39.99],
    'rating': [4.5, 3.8, 4.1, 4.9, 4.3],
    'in_stock': [True, False, True, True, False]
})

# Chain multiple filter conditions
result = df.filter(
    pl.col('product').str.contains('Widget') &
    pl.col('price').is_between(10.0, 45.0) &
    pl.col('in_stock') &
    (pl.col('rating') >= 4.0)
)
print('Filtered:', result)

# Negate a filter with ~
not_widget = df.filter(~pl.col('product').str.contains('Widget'))
print('Not Widget:', not_widget)

import polars as pl
import numpy as np

np.random.seed(42)
df = pl.DataFrame({
    'name': [f'Item_{i}' for i in range(10)],
    'price': [12.5, None, 45.0, 8.99, None, 99.0, 34.5, 22.0, None, 67.0],
    'rating': np.round(np.random.uniform(1.0, 5.0, 10), 1).tolist(),
    'region': ['East','West','East','North','South','West','East','North','South','West'],
})

# is_between on a column with nulls (nulls excluded automatically)
mid_range = df.filter(pl.col('price').is_between(10.0, 60.0))
print('Price between 10 and 60:')
print(mid_range)

# top_k — fast partial sort (no need to sort the whole frame)
print('Top 3 by rating (top_k):')
print(df.top_k(3, by='rating'))

# Sort by multiple cols with nulls_last=True
print('Sort by region asc, price desc (nulls last):')
print(df.sort(
    ['region', 'price'],
    descending=[False, True],
    nulls_last=True
))

import polars as pl

df = pl.DataFrame({
    'dept': ['Eng','Eng','Sales','Sales','HR','HR','Eng'],
    'name': ['Alice','Bob','Carol','Dave','Eve','Frank','Grace'],
    'salary': [90000, 85000, 70000, 65000, 55000, 58000, 95000],
    'years': [5, 3, 7, 2, 8, 4, 6]
})

# Basic aggregation
print(df.group_by('dept').agg([
    pl.len().alias('headcount'),
    pl.col('salary').mean().alias('avg_salary'),
    pl.col('salary').max().alias('max_salary'),
    pl.col('years').sum().alias('total_years')
]))

# Multiple groups
print(df.group_by(['dept']).agg(
    pl.col('salary').median().alias('median_salary')
).sort('median_salary', descending=True))

import polars as pl
import numpy as np

np.random.seed(42)
df = pl.DataFrame({
    'category': np.random.choice(['A','B','C'], 1000),
    'value': np.random.normal(100, 20, 1000),
    'qty': np.random.randint(1, 50, 1000)
})

stats = df.group_by('category').agg([
    pl.len().alias('count'),
    pl.col('value').mean().round(2).alias('mean'),
    pl.col('value').std().round(2).alias('std'),
    pl.col('value').quantile(0.25).round(2).alias('q25'),
    pl.col('value').quantile(0.75).round(2).alias('q75'),
    pl.col('qty').sum().alias('total_qty'),
    (pl.col('value') * pl.col('qty')).sum().round(2).alias('weighted_sum')
]).sort('category')
print(stats)

import polars as pl
import numpy as np

np.random.seed(0)
df = pl.DataFrame({
    'region': np.random.choice(['North','South','East','West'], 200),
    'product': np.random.choice(['A','B','C'], 200),
    'revenue': np.random.exponential(1000, 200).round(2),
    'units': np.random.randint(1, 50, 200)
})

# Multi-column groupby with many aggregations
summary = df.group_by(['region', 'product']).agg([
    pl.len().alias('count'),
    pl.col('revenue').sum().round(2).alias('total_revenue'),
    pl.col('revenue').mean().round(2).alias('avg_revenue'),
    pl.col('units').sum().alias('total_units'),
    (pl.col('revenue').sum() / pl.col('units').sum()).round(2).alias('rev_per_unit')
]).sort(['region', 'product'])
print(summary)

# Filter groups: only regions where total revenue > 20000
high_rev = df.group_by('region').agg(
    pl.col('revenue').sum().alias('total')
).filter(pl.col('total') > 20000).sort('total', descending=True)
print('High-revenue regions:', high_rev)

import polars as pl
import numpy as np
from datetime import datetime, timedelta

np.random.seed(42)
n = 500
base = datetime(2024, 1, 1)
# Build a time-series DataFrame with a proper Datetime column
df = pl.DataFrame({
    'ts': [base + timedelta(hours=int(h)) for h in np.random.uniform(0, 24*30, n)],
    'sales': np.random.exponential(200, n).round(2),
    'units': np.random.randint(1, 20, n),
}).sort('ts')

# group_by_dynamic — aggregate by calendar window (daily buckets)
daily = df.group_by_dynamic('ts', every='1d').agg([
    pl.col('sales').sum().round(2).alias('daily_revenue'),
    pl.col('units').sum().alias('daily_units'),
    pl.len().alias('transactions'),
])
print('Daily aggregation (first 7 days):')
print(daily.head(7))

# Rolling aggregation — 7-day rolling sum over the sorted frame
rolling = df.with_columns(
    pl.col('sales').rolling_sum(window_size=7, min_periods=1).alias('rolling_7d_sales')
)
print('Rolling 7-day sales sum (last 5 rows):')
print(rolling.select(['ts', 'sales', 'rolling_7d_sales']).tail(5))

import polars as pl

customers = pl.DataFrame({
    'id': [1, 2, 3, 4, 5],
    'name': ['Alice', 'Bob', 'Carol', 'Dave', 'Eve']
})
orders = pl.DataFrame({
    'order_id': [101, 102, 103, 104],
    'cust_id': [1, 2, 2, 6],  # 6 has no customer
    'amount': [250, 80, 120, 400]
})

# Inner join
print('INNER:')
print(customers.join(orders, left_on='id', right_on='cust_id', how='inner'))

# Left join
print('LEFT (all customers):')
print(customers.join(orders, left_on='id', right_on='cust_id', how='left'))

# Anti join — customers with NO orders
print('ANTI (no orders):')
print(customers.join(orders, left_on='id', right_on='cust_id', how='anti'))

import polars as pl

products = pl.DataFrame({
    'sku': ['A', 'B', 'C', 'D', 'E'],
    'price': [10.0, 25.0, 15.0, 8.0, 50.0]
})
sold = pl.DataFrame({'sku': ['A', 'C', 'E']})

# Semi join — only rows with a match (no extra columns)
print('SEMI (sold products):')
print(products.join(sold, on='sku', how='semi'))

# Vertical concat (stack rows)
df1 = pl.DataFrame({'a': [1, 2], 'b': ['x', 'y']})
df2 = pl.DataFrame({'a': [3, 4], 'b': ['z', 'w']})
print('CONCAT vertical:')
print(pl.concat([df1, df2]))

# Horizontal concat (side by side)
df3 = pl.DataFrame({'c': [10, 20], 'd': [30, 40]})
print('CONCAT horizontal:')
print(pl.concat([df1, df3], how='horizontal'))

import polars as pl

# Cross join — all combinations
sizes = pl.DataFrame({'size': ['S', 'M', 'L', 'XL']})
colors = pl.DataFrame({'color': ['Red', 'Blue', 'Green']})
variants = sizes.join(colors, how='cross')
print(f'Product variants ({len(variants)} rows):')
print(variants)

# Join with overlapping column names — use suffix
orders = pl.DataFrame({
    'order_id': [1, 2, 3],
    'amount': [100, 200, 300],
    'status': ['paid', 'pending', 'paid']
})
returns = pl.DataFrame({
    'order_id': [2, 3],
    'amount': [200, 150],  # refund amount
    'status': ['approved', 'partial']
})
joined = orders.join(returns, on='order_id', how='left', suffix='_refund')
print('With suffix disambiguation:')
print(joined)

import polars as pl

# --- Semi join: keep only left rows that have a match ---
inventory = pl.DataFrame({
    'sku': ['A001', 'B002', 'C003', 'D004', 'E005'],
    'stock': [100, 0, 50, 200, 0]
})
ordered_skus = pl.DataFrame({'sku': ['A001', 'C003', 'E005']})

in_stock_orders = inventory.join(ordered_skus, on='sku', how='semi')
print('Semi join — ordered items in inventory:')
print(in_stock_orders)

# --- join with on= (same column name on both sides) ---
employees = pl.DataFrame({
    'dept_id': [1, 2, 3, 1],
    'name': ['Alice', 'Bob', 'Carol', 'Dave']
})
departments = pl.DataFrame({
    'dept_id': [1, 2, 3],
    'dept_name': ['Engineering', 'Sales', 'HR']
})
joined = employees.join(departments, on='dept_id', how='inner')
print('join with on= (shared key name):')
print(joined)

# --- Asof join concept: join on nearest key ---
# Match each trade to the most recent quote before it
quotes = pl.DataFrame({
    'time': [1, 3, 5, 8, 12],
    'bid': [100.0, 100.5, 101.0, 101.5, 102.0]
})
trades = pl.DataFrame({
    'time': [2, 4, 9, 11],
    'qty': [10, 5, 20, 8]
})
asof = trades.join_asof(quotes, on='time', strategy='backward')
print('Asof join (each trade gets most recent bid):')
print(asof)

import polars as pl

df = pl.DataFrame({
    'email': ['alice@example.com', 'bob@test.org', 'carol@example.com'],
    'full_name': ['Alice Smith', 'Bob Jones', 'Carol White'],
    'code': ['  ABC-123  ', 'DEF-456', ' GHI-789 ']
})

result = df.with_columns([
    pl.col('email').str.split('@').list.get(1).alias('domain'),
    pl.col('email').str.contains('example').alias('is_example'),
    pl.col('full_name').str.split(' ').list.get(0).alias('first_name'),
    pl.col('full_name').str.to_lowercase().alias('name_lower'),
    pl.col('code').str.strip_chars().alias('code_clean'),
    pl.col('code').str.strip_chars().str.replace('-', '_').alias('code_underscore')
])
print(result)

import polars as pl
from datetime import date

df = pl.DataFrame({
    'date_str': ['2024-01-15', '2024-03-22', '2024-07-04', '2024-12-31'],
    'event': ['Q1 Review', 'Sprint End', 'Holiday', 'Year End']
})

df2 = df.with_columns(
    pl.col('date_str').str.to_date('%Y-%m-%d').alias('date')
)

result = df2.with_columns([
    pl.col('date').dt.year().alias('year'),
    pl.col('date').dt.month().alias('month'),
    pl.col('date').dt.day().alias('day'),
    pl.col('date').dt.weekday().alias('weekday'),  # Mon=1
    pl.col('date').dt.quarter().alias('quarter'),
    pl.col('date').dt.strftime('%B %d, %Y').alias('formatted')
])
print(result)

import polars as pl
from datetime import date

# String operations
df = pl.DataFrame({
    'url': ['https://example.com/product/123', 'https://example.com/cart', 'https://shop.io/item/456'],
    'tags': ['python,data,science', 'ml,ai,deep-learning', 'stats,python,r'],
})

result = df.with_columns([
    pl.col('url').str.contains('product').alias('is_product_page'),
    pl.col('url').str.split('/').list.last().alias('path_end'),
    pl.col('url').str.replace('https://', '').alias('url_no_scheme'),
    pl.col('tags').str.split(',').list.len().alias('tag_count'),
    pl.col('tags').str.contains('python').alias('has_python'),
])
print(result)

# Date arithmetic
df2 = pl.DataFrame({
    'start': [date(2024, 1, 1), date(2024, 3, 15), date(2024, 6, 30)],
    'end':   [date(2024, 4, 1), date(2024, 5, 20), date(2024, 12, 31)]
})
df3 = df2.with_columns([
    pl.col('start').dt.strftime('%B %Y').alias('start_label'),
    (pl.col('end') - pl.col('start')).dt.total_days().alias('duration_days'),
    pl.col('start').dt.month().alias('start_month'),
    pl.col('end').dt.year().alias('end_year'),
])
print(df3)

import polars as pl
from datetime import date

# String regex extraction
df = pl.DataFrame({'log': [
    '2024-01-15 ERROR db_pool Connection timeout after 30s',
    '2024-01-16 INFO  auth_svc User alice logged in',
    '2024-01-17 WARN  api_gw  Rate limit at 95%',
]})

result = df.with_columns([
    pl.col('log').str.extract(r'(\d{4}-\d{2}-\d{2})', 1).alias('date'),
    pl.col('log').str.extract(r'\d{4}-\d{2}-\d{2} (\w+)', 1).alias('level'),
    pl.col('log').str.extract(r'\w+ \w+\s+(\w+)\s', 1).alias('service'),
])
print('Regex extraction:')
print(result)

# str.split_exact — fixed number of parts
df2 = pl.DataFrame({'ts': ['2024-01-15', '2024-06-30', '2024-12-01']})
split = df2.with_columns(
    pl.col('ts').str.split_exact('-', 2).alias('parts')
).unnest('parts').rename({'field_0':'year','field_1':'month','field_2':'day'})
print('\nSplit date parts:')
print(split)

# dt.truncate — round dates to month/week
df3 = pl.DataFrame({'dt': pl.date_range(
    pl.date(2024, 1, 1), pl.date(2024, 3, 31), interval='11d', eager=True
)})
df3 = df3.with_columns([
    pl.col('dt').dt.truncate('1mo').alias('month_start'),
    pl.col('dt').dt.truncate('1w').alias('week_start'),
])
print('\nDate truncation:')
print(df3.head(6))

import polars as pl
import numpy as np

np.random.seed(42)
df = pl.DataFrame({
    'id': range(1_000_000),
    'value': np.random.randn(1_000_000),
    'group': np.random.choice(['A','B','C','D'], 1_000_000),
    'score': np.random.randint(0, 100, 1_000_000)
})

# Convert to LazyFrame
result = (
    df.lazy()                            # build query plan
    .filter(pl.col('score') > 50)        # predicate pushdown
    .with_columns(
        (pl.col('value') * 2).alias('value2')
    )
    .group_by('group').agg(
        pl.col('value2').mean().round(4).alias('mean_v2'),
        pl.len().alias('count')
    )
    .sort('group')
)

# See the query plan
print(result.explain())

# Execute
print(result.collect())

import polars as pl
import tempfile, os, numpy as np

# Write a sample CSV to scan
np.random.seed(42)
n = 100_000
tmp = pl.DataFrame({
    'a': np.random.randint(0, 100, n),
    'b': np.random.randn(n),
    'c': np.random.choice(['X','Y','Z'], n)
})
path = os.path.join(tempfile.gettempdir(), 'polars_demo.csv')
tmp.write_csv(path)

# scan_csv: reads lazily — only loads needed data
result = (
    pl.scan_csv(path)
    .filter(pl.col('c') == 'X')
    .filter(pl.col('a') > 90)
    .select(['a', 'b'])
    .collect()
)
print('Filtered result:', result.shape)
print(result.head())

import polars as pl
import numpy as np

np.random.seed(42)
# Build a large DataFrame then use LazyFrame
df = pl.DataFrame({
    'user_id': np.random.randint(1, 500, 50_000),
    'product': np.random.choice(['A','B','C','D','E'], 50_000),
    'amount': np.random.exponential(100, 50_000).round(2),
    'returned': np.random.choice([True, False], 50_000, p=[0.05, 0.95])
})

# Convert to lazy and build a multi-step pipeline
lazy_result = (
    df.lazy()
    .filter(~pl.col('returned'))                   # exclude returns
    .filter(pl.col('amount') > 10)                 # min order
    .with_columns(
        (pl.col('amount') * 1.1).alias('amount_with_tax')
    )
    .group_by(['user_id', 'product']).agg([
        pl.len().alias('orders'),
        pl.col('amount_with_tax').sum().round(2).alias('total_spent')
    ])
    .filter(pl.col('orders') >= 3)                 # frequent buyers
    .sort('total_spent', descending=True)
)

# Execute
result = lazy_result.collect()
print(f'Frequent buyers: {len(result):,}')
print(result.head(8))

import polars as pl
import numpy as np
import tempfile, os

np.random.seed(42)
n = 500_000
df = pl.DataFrame({
    'id':       range(n),
    'amount':   np.random.exponential(100, n).round(2),
    'category': np.random.choice(['A','B','C','D'], n),
    'flag':     np.random.choice([True, False], n),
})
pq_path = os.path.join(tempfile.gettempdir(), 'streaming_demo.parquet')
df.write_parquet(pq_path)

# LazyFrame schema inspection — no data loaded yet
lf = pl.scan_parquet(pq_path)
print('Schema (no data loaded):')
print(lf.schema)
print('Columns:', lf.columns)

# Build a lazy query
query = (
    lf.filter(pl.col('flag') & (pl.col('amount') > 50))
      .group_by('category')
      .agg([
          pl.col('amount').mean().round(2).alias('avg'),
          pl.len().alias('count')
      ])
      .sort('avg', descending=True)
)

# Show the optimized query plan
print('\nOptimized plan:')
query.explain(optimized=True)

# Execute
result = query.collect()
print('\nResult:')
print(result)

import polars as pl
import numpy as np
import tempfile, os

np.random.seed(42)
df = pl.DataFrame({
    'id': range(100),
    'name': [f'Item_{i}' for i in range(100)],
    'value': np.random.randn(100).round(4),
    'tag': np.random.choice(['A','B','C'], 100)
})

tmp = tempfile.gettempdir()

# CSV
csv_path = os.path.join(tmp, 'demo.csv')
df.write_csv(csv_path)
df_csv = pl.read_csv(csv_path)
print('CSV roundtrip:', df_csv.shape)

# JSON (newline-delimited)
json_path = os.path.join(tmp, 'demo.ndjson')
df.write_ndjson(json_path)
df_json = pl.read_ndjson(json_path)
print('NDJSON roundtrip:', df_json.shape)

import polars as pl
import numpy as np
import tempfile, os

np.random.seed(42)
n = 500_000
df = pl.DataFrame({
    'id': range(n),
    'amount': np.random.exponential(200, n).round(2),
    'category': np.random.choice(['A','B','C','D'], n),
    'flag': np.random.choice([True, False], n)
})

tmp = tempfile.gettempdir()
pq_path = os.path.join(tmp, 'demo.parquet')

# Write Parquet (compressed, columnar)
df.write_parquet(pq_path, compression='zstd')
size_mb = os.path.getsize(pq_path) / 1024 / 1024
print(f'Parquet size: {size_mb:.2f} MB (for {n:,} rows)')

# Read full
df2 = pl.read_parquet(pq_path)
print('Read back:', df2.shape)

# Scan (lazy) — column pruning + predicate pushdown
result = pl.scan_parquet(pq_path).filter(
    pl.col('category') == 'A'
).select(['id','amount']).collect()
print('Filtered parquet:', result.shape)

import polars as pl
import numpy as np
import tempfile, os, io

np.random.seed(42)
df = pl.DataFrame({
    'id': range(1000),
    'value': np.random.randn(1000).round(4),
    'category': np.random.choice(['A','B','C'], 1000)
})

tmp = tempfile.gettempdir()

# IPC (Arrow) — fastest local I/O format
ipc_path = os.path.join(tmp, 'demo.arrow')
df.write_ipc(ipc_path)
df_ipc = pl.read_ipc(ipc_path)
size_ipc = os.path.getsize(ipc_path) / 1024
print(f'IPC size: {size_ipc:.1f} KB, shape: {df_ipc.shape}')

# In-memory CSV read from string
csv_str = df.write_csv()
df_str = pl.read_csv(io.StringIO(csv_str))
print('From string CSV:', df_str.shape)

# Column pruning on Parquet read
pq_path = os.path.join(tmp, 'pruned_demo.parquet')
df.write_parquet(pq_path)
df_cols = pl.read_parquet(pq_path, columns=['id', 'category'])
print('Column-pruned parquet:', df_cols.shape, df_cols.columns)

import polars as pl
import io

# scan_csv with explicit schema override
csv_data = 'id,score,grade,active\n1,92.5,A,true\n2,78.0,B,false\n3,85.5,A,true\n'
schema_override = {
    'id':     pl.Int32,
    'score':  pl.Float32,
    'grade':  pl.Categorical,
    'active': pl.Boolean,
}
df = pl.read_csv(io.StringIO(csv_data), schema_overrides=schema_override)
print('With schema override:')
print(df)
print('Dtypes:', df.dtypes)

# In-memory Parquet (BytesIO — no file system needed)
import numpy as np
np.random.seed(0)
df2 = pl.DataFrame({
    'x': np.random.randn(1000).astype('float32'),
    'y': np.random.randint(0, 10, 1000),
})
buf = io.BytesIO()
df2.write_parquet(buf, compression='zstd')
buf.seek(0)
size_kb = buf.getbuffer().nbytes / 1024
df3 = pl.read_parquet(buf)
print(f'\nIn-memory Parquet: {df2.shape} → {size_kb:.1f} KB')
print('Read back:', df3.shape)

import polars as pl
import numpy as np

np.random.seed(42)
df = pl.DataFrame({
    'day': range(1, 31),
    'sales': np.random.randint(50, 200, 30)
})

result = df.with_columns([
    pl.col('sales').cum_sum().alias('cumulative_sales'),
    pl.col('sales').rolling_mean(window_size=7).round(1).alias('7d_avg'),
    pl.col('sales').rolling_max(window_size=7).alias('7d_max'),
    pl.col('sales').rolling_std(window_size=7).round(2).alias('7d_std'),
    pl.col('sales').pct_change().round(4).alias('pct_change')
])
print(result.tail(10))

import polars as pl

df = pl.DataFrame({
    'dept': ['Eng','Eng','Eng','Sales','Sales','HR','HR'],
    'name': ['Alice','Bob','Carol','Dave','Eve','Frank','Grace'],
    'salary': [90000, 85000, 95000, 70000, 65000, 55000, 58000]
})

result = df.with_columns([
    pl.col('salary').rank('dense').over('dept').alias('rank_in_dept'),
    pl.col('salary').mean().over('dept').round(0).alias('dept_avg'),
    (pl.col('salary') - pl.col('salary').mean().over('dept'))
        .round(0).alias('vs_dept_avg'),
    pl.col('salary').max().over('dept').alias('dept_max')
])
print(result)

import polars as pl
import numpy as np

np.random.seed(42)
df = pl.DataFrame({
    'day': range(1, 21),
    'sales': np.random.randint(100, 500, 20)
})

result = df.with_columns([
    pl.col('sales').shift(1).alias('prev_day'),
    pl.col('sales').shift(-1).alias('next_day'),
    (pl.col('sales') - pl.col('sales').shift(1)).alias('day_delta'),
    pl.col('sales').rolling_mean(window_size=3, min_periods=1).round(1).alias('3d_avg'),
    pl.col('sales').cum_max().alias('running_max'),
    (pl.col('sales') / pl.col('sales').max() * 100).round(1).alias('pct_of_max')
])
print(result)

import polars as pl
import numpy as np

np.random.seed(42)
df = pl.DataFrame({
    'day':   range(1, 31),
    'price': np.random.uniform(95, 105, 30).round(2),
    'vol':   np.random.randint(1000, 5000, 30),
})

result = df.with_columns([
    # Exponential weighted mean (recent values get more weight)
    pl.col('price').ewm_mean(span=5).round(3).alias('ewm_5'),
    # Rolling 7-day median (robust to outliers)
    pl.col('price').rolling_median(window_size=7).round(3).alias('rolling_med_7'),
    # Rolling 75th percentile
    pl.col('vol').rolling_quantile(quantile=0.75, window_size=7).alias('vol_q75'),
])
print(result.head(10))

# map_elements for custom per-element logic
df2 = pl.DataFrame({'scores': [[85, 90, 78], [60, 70], [95, 88, 92, 80]]})
result2 = df2.with_columns(
    pl.col('scores').map_elements(
        lambda lst: round(sum(lst) / len(lst), 2),
        return_dtype=pl.Float64
    ).alias('avg_score')
)
print('\nCustom avg via map_elements:')
print(result2)

import polars as pl
import pandas as pd

# Creating a DataFrame
# Pandas:
pd_df = pd.DataFrame({'a': [1,2,3], 'b': [4,5,6]})

# Polars:
pl_df = pl.DataFrame({'a': [1,2,3], 'b': [4,5,6]})

# Filtering
# Pandas: df[df['a'] > 1]
# Polars:
print(pl_df.filter(pl.col('a') > 1))

# Adding column
# Pandas: df['c'] = df['a'] + df['b']
# Polars:
pl_df2 = pl_df.with_columns((pl.col('a') + pl.col('b')).alias('c'))
print(pl_df2)

# GroupBy
# Pandas: df.groupby('a')['b'].sum()
# Polars:
pl_df3 = pl.DataFrame({'a': ['x','x','y','y'], 'b': [1,2,3,4]})
print(pl_df3.group_by('a').agg(pl.col('b').sum()))

import polars as pl
import pandas as pd
import numpy as np

# Start with Pandas
pd_df = pd.DataFrame({
    'name': ['Alice', 'Bob', 'Carol'],
    'score': [92.5, 88.0, 95.1]
})

# Pandas -> Polars
pl_df = pl.from_pandas(pd_df)
print('From pandas:', pl_df)

# Polars -> Pandas
back_to_pd = pl_df.to_pandas()
print('Back to pandas:', back_to_pd.dtypes)

# Polars -> NumPy
arr = pl_df['score'].to_numpy()
print('NumPy array:', arr)

# Apply pandas where Polars lacks support
# e.g., complex visualization or sklearn pipelines
from sklearn.preprocessing import StandardScaler
X = StandardScaler().fit_transform(pl_df.select('score').to_pandas())
print('Scaled scores:', X.flatten().round(2))

import polars as pl
import pandas as pd
import numpy as np

np.random.seed(42)
n = 1000
data = {
    'dept': np.random.choice(['Eng', 'Sales', 'HR'], n),
    'score': np.random.randint(0, 100, n),
    'active': np.random.choice([True, False], n)
}

# Pandas: filter + conditional column + groupby
pd_df = pd.DataFrame(data)
pd_active = pd_df[pd_df['active']].copy()
pd_active['grade'] = pd.cut(pd_active['score'], bins=[0,60,80,100], labels=['C','B','A'])
pd_result = pd_active.groupby(['dept','grade'])['score'].mean().round(2).reset_index()
print('Pandas:')
print(pd_result.sort_values(['dept','grade']).head(6))

# Polars equivalent — more explicit, no chained assignment
pl_df = pl.DataFrame(data)
pl_result = (
    pl_df.filter(pl.col('active'))
    .with_columns(
        pl.when(pl.col('score') >= 80).then(pl.lit('A'))
          .when(pl.col('score') >= 60).then(pl.lit('B'))
          .otherwise(pl.lit('C')).alias('grade')
    )
    .group_by(['dept', 'grade'])
    .agg(pl.col('score').mean().round(2).alias('avg_score'))
    .sort(['dept', 'grade'])
)
print('\nPolars:')
print(pl_result.head(6))

import polars as pl
import pandas as pd
import numpy as np

# Categorical dtype in Polars (much more efficient than object)
np.random.seed(42)
n = 100_000
df_pl = pl.DataFrame({
    'product':  pl.Series(np.random.choice(['Widget','Gadget','Doohickey'], n)).cast(pl.Categorical),
    'region':   pl.Series(np.random.choice(['North','South','East','West'], n)).cast(pl.Categorical),
    'revenue':  np.random.exponential(200, n).round(2),
})
print('Categorical memory usage:')
print(f'  estimated size: {df_pl.estimated_size("mb"):.2f} MB')
print(f'  product unique values: {df_pl["product"].n_unique()}')

# Zero-copy to pandas via Arrow
df_pd = df_pl.to_pandas(use_pyarrow_extension_array=True)
print('\nArrow-backed pandas dtypes:')
print(df_pd.dtypes)

# from_pandas preserving categorical
df_pd2 = pd.DataFrame({
    'color': pd.Categorical(['red','blue','red','green','blue']),
    'val':   [1, 2, 3, 4, 5]
})
df_pl2 = pl.from_pandas(df_pd2)
print('\nFrom pandas Categorical:')
print(df_pl2)
print('dtype:', df_pl2['color'].dtype)

import polars as pl
import numpy as np

np.random.seed(42)
data = {
    'id':       range(100_000),
    'category': np.random.choice(['A','B','C','D'], 100_000),
    'value':    np.random.randn(100_000) * 100 + 500,
    'qty':      np.random.randint(1, 100, 100_000),
}

# Eager DataFrame
df = pl.DataFrame(data)

# Lazy: nothing executes yet
lf = pl.LazyFrame(data)
plan = (
    lf
    .filter(pl.col('value') > 500)
    .group_by('category')
    .agg(
        pl.col('value').mean().alias('avg_val'),
        pl.col('qty').sum().alias('total_qty'),
    )
    .sort('avg_val', descending=True)
)
print('Lazy plan (before execution):')
print(plan.explain())
result = plan.collect()  # executes query
print('\nResult after .collect():')
print(result)

import polars as pl
import numpy as np
import time

np.random.seed(0)
n = 500_000
df = pl.DataFrame({
    'user_id': np.random.randint(1, 10000, n),
    'product': np.random.choice(['laptop','phone','tablet','watch'], n),
    'revenue': np.random.exponential(100, n),
    'country': np.random.choice(['US','UK','DE','FR','JP'], n),
})

# Eager: no optimization
t0 = time.perf_counter()
eager = (
    df
    .filter(pl.col('country') == 'US')
    .filter(pl.col('revenue') > 50)
    .group_by('product')
    .agg(pl.col('revenue').sum())
)
t_eager = time.perf_counter() - t0

# Lazy: Polars merges filters and optimizes scan
t0 = time.perf_counter()
lazy = (
    df.lazy()
    .filter(pl.col('country') == 'US')
    .filter(pl.col('revenue') > 50)
    .group_by('product')
    .agg(pl.col('revenue').sum())
    .collect()
)
t_lazy = time.perf_counter() - t0
print(f'Eager: {t_eager*1000:.1f}ms | Lazy: {t_lazy*1000:.1f}ms')
print(lazy.sort('revenue', descending=True))

import polars as pl
import tempfile, os, numpy as np

# Create a sample CSV file
np.random.seed(42)
rows = 200_000
with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as f:
    fname = f.name
    f.write('id,category,value,qty\n')
    cats = np.random.choice(['A','B','C'], rows)
    vals = np.random.randn(rows) * 50 + 200
    qtys = np.random.randint(1, 50, rows)
    for i in range(rows):
        f.write(f'{i},{cats[i]},{vals[i]:.2f},{qtys[i]}\n')

# scan_csv = lazy: doesn't load file yet
lf = pl.scan_csv(fname)
print('Schema:', lf.schema)

result = (
    lf
    .filter(pl.col('value') > 200)
    .group_by('category')
    .agg(
        pl.col('value').mean().round(2).alias('avg_value'),
        pl.col('qty').sum().alias('total_qty'),
        pl.len().alias('count'),
    )
    .sort('avg_value', descending=True)
    .collect(streaming=True)  # streaming=True for large files
)
print(result)
os.unlink(fname)

import polars as pl
import numpy as np
import tempfile, os

np.random.seed(42)
n = 100_000
df = pl.DataFrame({
    'ts':       pl.date_range(
                    pl.date(2024, 1, 1), pl.date(2024, 12, 31),
                    interval='1h', eager=True
                )[:n],
    'sensor':   np.random.choice(['s1','s2','s3'], n),
    'reading':  np.random.normal(25, 5, n),
})

tmpdir = tempfile.mkdtemp()
out_path = os.path.join(tmpdir, 'out.parquet')

# Write to parquet
df.write_parquet(out_path)

# Read back with scan_parquet (lazy)
result = (
    pl.scan_parquet(out_path)
    .filter(pl.col('reading') > 25)
    .group_by('sensor')
    .agg(pl.col('reading').mean().alias('avg_reading'), pl.len().alias('n'))
    .collect()
)
print(result)
fsize = os.path.getsize(out_path)
print(f'Parquet file size: {fsize/1024:.1f} KB for {n:,} rows')
import shutil; shutil.rmtree(tmpdir)

import polars as pl
import numpy as np

np.random.seed(42)
dates = pl.date_range(pl.date(2024, 1, 1), pl.date(2024, 12, 31), interval='1d', eager=True)
df = pl.DataFrame({
    'date':  dates,
    'sales': np.random.normal(1000, 200, len(dates)).round(2),
})

result = df.with_columns([
    pl.col('sales').rolling_mean(window_size=7).alias('rolling_7d'),
    pl.col('sales').rolling_mean(window_size=30).alias('rolling_30d'),
    pl.col('sales').rolling_std(window_size=7).alias('rolling_std_7d'),
    pl.col('sales').cum_sum().alias('cumulative_sales'),
    pl.col('sales').rolling_max(window_size=7).alias('rolling_max_7d'),
])
print(result.head(35).tail(5))
print(f'\nTotal annual sales: {result["cumulative_sales"][-1]:,.0f}')
print(f'Peak 7-day avg: {result["rolling_7d"].max():,.1f}')

import polars as pl
import numpy as np

np.random.seed(0)
df = pl.DataFrame({
    'product':  np.random.choice(['A','B','C','D','E'], 50),
    'region':   np.random.choice(['North','South','East'], 50),
    'revenue':  np.random.exponential(1000, 50).round(2),
})

# Global rank
ranked = df.with_columns([
    pl.col('revenue').rank(method='dense', descending=True).alias('global_rank'),
])

# Rank within region
regional = df.with_columns([
    pl.col('revenue').rank(method='dense', descending=True)
      .over('region').alias('region_rank'),
    pl.col('revenue').rank(method='ordinal', descending=True)
      .over('region').alias('region_ordinal_rank'),
])

print('Top 5 by revenue with global rank:')
print(ranked.sort('global_rank').head())
print('\nTop 3 per region:')
print(regional.filter(pl.col('region_rank') <= 3).sort(['region','region_rank']))

import polars as pl
import numpy as np

np.random.seed(42)
n = 200
df = pl.DataFrame({
    'employee': [f'emp_{i}' for i in range(n)],
    'dept':     np.random.choice(['Engineering','Marketing','Sales','HR'], n),
    'salary':   np.random.normal(80000, 20000, n).round(0),
    'yrs_exp':  np.random.randint(1, 20, n),
})

result = df.with_columns([
    pl.col('salary').mean().over('dept').alias('dept_avg_salary'),
    pl.col('salary').rank(method='dense', descending=True).over('dept').alias('dept_rank'),
    (pl.col('salary') - pl.col('salary').mean().over('dept')).alias('salary_vs_dept_avg'),
    pl.len().over('dept').alias('dept_size'),
])

print('Employees earning above dept average:')
above_avg = result.filter(pl.col('salary_vs_dept_avg') > 0)
print(above_avg.sort('dept').select(['employee','dept','salary','dept_avg_salary','dept_rank']).head(8))

import polars as pl
import numpy as np

np.random.seed(7)
dates = pl.date_range(pl.date(2024, 1, 1), pl.date(2024, 6, 30), interval='1d', eager=True)
stocks = ['AAPL', 'GOOG']
all_dfs = []
for stock in stocks:
    prices = 150 + np.cumsum(np.random.randn(len(dates)) * 2)
    all_dfs.append(pl.DataFrame({'date': dates, 'stock': stock, 'close': prices.round(2)}))
df = pl.concat(all_dfs).sort(['stock', 'date'])

result = df.with_columns([
    pl.col('close').shift(1).over('stock').alias('prev_close'),
    pl.col('close').shift(7).over('stock').alias('close_7d_ago'),
    pl.col('close').pct_change().over('stock').alias('daily_return'),
    pl.col('close').rolling_mean(window_size=5).over('stock').alias('ma5'),
    pl.col('close').rolling_mean(window_size=20).over('stock').alias('ma20'),
])
print(result.filter(pl.col('stock') == 'AAPL').tail(5))

import polars as pl
import pandas as pd
import numpy as np
import time

np.random.seed(42)
n = 1_000_000
data = {
    'id':       np.random.randint(1, 10000, n),
    'category': np.random.choice(['A','B','C','D','E'], n),
    'value':    np.random.randn(n) * 100,
    'qty':      np.random.randint(1, 50, n),
}

df_pd = pd.DataFrame(data)
df_pl = pl.DataFrame(data)

def bench(fn, name, reps=3):
    times = []
    for _ in range(reps):
        t0 = time.perf_counter()
        fn()
        times.append(time.perf_counter() - t0)
    print(f'{name:<30} {min(times)*1000:.1f}ms (best of {reps})')

bench(lambda: df_pd.groupby('category')['value'].agg(['mean','sum','std']), 'Pandas groupby')
bench(lambda: df_pl.group_by('category').agg(pl.col('value').mean(), pl.col('value').sum(), pl.col('value').std()), 'Polars groupby')

bench(lambda: df_pd.sort_values('value'), 'Pandas sort')
bench(lambda: df_pl.sort('value'), 'Polars sort')

import polars as pl
import numpy as np

np.random.seed(42)
n = 500_000
# Default dtype inference
df_default = pl.DataFrame({
    'user_id':  np.random.randint(1, 10000, n),
    'country':  np.random.choice(['US','UK','DE','FR','JP'], n),
    'product':  np.random.choice([f'prod_{i}' for i in range(100)], n),
    'revenue':  np.random.exponential(50, n),
    'quantity': np.random.randint(1, 20, n),
})

# Optimized dtypes
df_opt = df_default.with_columns([
    pl.col('user_id').cast(pl.UInt32),
    pl.col('country').cast(pl.Categorical),
    pl.col('product').cast(pl.Categorical),
    pl.col('revenue').cast(pl.Float32),
    pl.col('quantity').cast(pl.UInt8),
])

def mem_mb(df): return df.estimated_size('mb')

print(f'Default schema: {mem_mb(df_default):.1f} MB')
print(f'Optimized schema: {mem_mb(df_opt):.1f} MB')
print(f'Reduction: {(1 - mem_mb(df_opt)/mem_mb(df_default))*100:.0f}%')
print('\nOptimized schema:')
print(df_opt.schema)

import polars as pl
import numpy as np
import time

print(f'Polars version: {pl.__version__}')
print(f'Available threads: {pl.thread_pool_size()}')

np.random.seed(0)
n = 2_000_000
df = pl.DataFrame({
    'g': np.random.randint(0, 100, n),
    'v': np.random.randn(n),
})

# Polars uses multiple threads automatically
# Demonstrate with a complex query
t0 = time.perf_counter()
result = (
    df.lazy()
    .group_by('g')
    .agg([
        pl.col('v').mean().alias('mean'),
        pl.col('v').std().alias('std'),
        pl.col('v').quantile(0.25).alias('q25'),
        pl.col('v').quantile(0.75).alias('q75'),
        pl.col('v').max() - pl.col('v').min(),
    ])
    .collect()
)
t = time.perf_counter() - t0
print(f'\n{n:,} rows, 100 groups, 5 aggs: {t*1000:.1f}ms')
print(result.head())

import polars as pl
import numpy as np

np.random.seed(42)
n = 1_000_000
df = pl.DataFrame({
    'id':       np.arange(n),
    'region':   np.random.choice(['NA','EU','APAC'], n),
    'product':  np.random.choice(['A','B','C','D'], n),
    'revenue':  np.random.exponential(200, n),
    'cost':     np.random.exponential(80, n),
    'quantity': np.random.randint(1, 100, n),
    'discount': np.random.uniform(0, 0.3, n),
})

# Without optimization hints (eager)
# Polars lazy plan shows predicate/column pushdown
lf = df.lazy()
plan = (
    lf
    .filter(pl.col('region') == 'EU')
    .select(['product', 'revenue', 'quantity'])
    .group_by('product')
    .agg(
        pl.col('revenue').sum().alias('total_rev'),
        pl.col('quantity').mean().alias('avg_qty'),
    )
)
print('Optimized query plan:')
print(plan.explain(optimized=True))
result = plan.collect()
print('\nResult:')
print(result.sort('total_rev', descending=True))

import polars as pl
import numpy as np
np.random.seed(0)
n = 1_000_000
# Build large lazy dataset
lf = (
    pl.LazyFrame({
        "id":       np.arange(n),
        "category": np.random.choice(["A","B","C","D"], n),
        "value":    np.random.normal(100, 20, n),
        "weight":   np.random.uniform(0.5, 2.0, n),
    })
    .filter(pl.col("value") > 60)
    .with_columns([
        (pl.col("value") * pl.col("weight")).alias("weighted_value"),
        pl.col("category").cast(pl.Categorical),
    ])
    .group_by("category")
    .agg([
        pl.col("weighted_value").mean().alias("mean_wv"),
        pl.col("value").std().alias("std_val"),
        pl.col("id").count().alias("n"),
    ])
    .sort("mean_wv", descending=True)
)
# Show the query plan
print("Query plan:")
print(lf.explain())
result = lf.collect()
print(result)

import polars as pl
import numpy as np
np.random.seed(1)
n = 500_000
# Streaming scan simulation: process in chunks
df = pl.DataFrame({
    "user_id":   np.random.randint(1, 1000, n),
    "revenue":   np.random.lognormal(3, 1, n),
    "product":   np.random.choice(["X","Y","Z"], n),
    "is_premium":np.random.randint(0, 2, n).astype(bool),
})
# Save and reload to test scan_parquet streaming
import tempfile, os
tmp = tempfile.mktemp(suffix=".parquet")
df.write_parquet(tmp)
# Streaming aggregation from parquet
result = (
    pl.scan_parquet(tmp)
    .filter(pl.col("is_premium"))
    .group_by("product")
    .agg([
        pl.col("revenue").sum().alias("total_rev"),
        pl.col("user_id").n_unique().alias("unique_users"),
        pl.col("revenue").mean().alias("avg_rev"),
    ])
    .sort("total_rev", descending=True)
    .collect(streaming=True)
)
os.unlink(tmp)
print(result)

import polars as pl
import numpy as np
import time
np.random.seed(42)
n = 200_000
df = pl.DataFrame({
    "id":    np.arange(n),
    "x":     np.random.randn(n),
    "group": np.random.choice([f"G{i}" for i in range(50)], n),
})
# Pattern 1: inefficient (multiple passes)
t0 = time.perf_counter()
for _ in range(3):
    _ = df.filter(pl.col("x") > 0).group_by("group").agg(pl.col("x").mean())
t1 = time.perf_counter()
# Pattern 2: single lazy pass with predicate pushdown
t2 = time.perf_counter()
for _ in range(3):
    _ = (pl.LazyFrame(df)
         .filter(pl.col("x") > 0)
         .group_by("group")
         .agg(pl.col("x").mean())
         .collect())
t3 = time.perf_counter()
print(f"Eager repeated passes: {(t1-t0)*1000:.1f} ms")
print(f"LazyFrame with pushdown: {(t3-t2)*1000:.1f} ms")

import polars as pl
import numpy as np
np.random.seed(0)
n = 1000
dates = pl.date_range(
    pl.date(2022, 1, 1), pl.date(2024, 9, 26), interval="1d", eager=True
)[:n]
df = pl.DataFrame({
    "date":  dates,
    "value": 100 + np.cumsum(np.random.normal(0, 1, n)),
    "volume":np.random.poisson(1000, n).astype(float),
})
result = (
    df.with_columns([
        pl.col("date").dt.year().alias("year"),
        pl.col("date").dt.month().alias("month"),
        pl.col("date").dt.weekday().alias("dow"),
        pl.col("value").rolling_mean(window_size=7).alias("ma7"),
        pl.col("value").rolling_mean(window_size=30).alias("ma30"),
        pl.col("value").pct_change().alias("return_1d"),
    ])
    .filter(pl.col("ma7").is_not_null())
)
print(result.select(["date","value","ma7","ma30","return_1d"]).head(5))
print(result.select(pl.col("return_1d").std()).item())

import polars as pl
import numpy as np
np.random.seed(1)
n = 500
dates = pl.date_range(pl.date(2023,1,1), pl.date(2024,5,14), interval="1d", eager=True)[:n]
df = pl.DataFrame({
    "date":  dates,
    "price": 50 + np.cumsum(np.random.normal(0, 0.5, n)),
})
# Resample to weekly and monthly
weekly = (
    df.group_by_dynamic("date", every="1w")
    .agg([
        pl.col("price").first().alias("open"),
        pl.col("price").max().alias("high"),
        pl.col("price").min().alias("low"),
        pl.col("price").last().alias("close"),
        pl.col("price").mean().alias("avg"),
    ])
)
monthly = (
    df.group_by_dynamic("date", every="1mo")
    .agg([
        pl.col("price").last().alias("month_close"),
        pl.col("price").std().alias("monthly_vol"),
    ])
)
print("Weekly OHLC:"); print(weekly.head(4))
print("Monthly volatility:"); print(monthly.head(4))

import polars as pl
import numpy as np
np.random.seed(2)
n = 300
dates = pl.date_range(pl.date(2023,1,1), pl.date(2023,10,28), interval="1d", eager=True)[:n]
df = pl.DataFrame({
    "date":  dates,
    "value": 100 + np.cumsum(np.random.normal(0, 1.5, n)),
})
result = (
    df.with_columns([
        pl.col("value").shift(1).alias("lag_1"),
        pl.col("value").shift(7).alias("lag_7"),
        pl.col("value").shift(30).alias("lag_30"),
        pl.col("value").rolling_std(window_size=14).alias("vol_14"),
        (pl.col("value") / pl.col("value").shift(1) - 1).alias("return_1d"),
        pl.col("value").rolling_max(window_size=52).alias("rolling_high_52"),
        pl.col("value").rolling_min(window_size=52).alias("rolling_low_52"),
    ])
    .with_columns([
        ((pl.col("value") - pl.col("rolling_low_52")) /
         (pl.col("rolling_high_52") - pl.col("rolling_low_52"))).alias("pct_rank_52w"),
    ])
    .drop_nulls()
)
print(result.select(["date","value","lag_1","vol_14","pct_rank_52w"]).tail(5))
print(f"Rows after dropna: {len(result)}")

import polars as pl
import pyarrow as pa
import numpy as np
np.random.seed(0)
n = 100_000
# Create Polars DataFrame and convert to Arrow
df = pl.DataFrame({
    "id":      np.arange(n),
    "score":   np.random.randn(n),
    "category":np.random.choice(["A","B","C"], n),
    "amount":  np.random.lognormal(4, 1, n),
})
# Polars -> Arrow -> Polars (zero-copy where possible)
arrow_table = df.to_arrow()
print(f"Arrow schema: {arrow_table.schema}")
print(f"Num chunks: {arrow_table.column('score').num_chunks}")
# Compute with pyarrow
import pyarrow.compute as pc
mean_score = pc.mean(arrow_table.column("score"))
print(f"Mean score via pyarrow: {mean_score.as_py():.6f}")
# Back to Polars
df2 = pl.from_arrow(arrow_table)
print(f"Roundtrip OK: {df.shape == df2.shape}")

import polars as pl
import numpy as np
import tempfile, os, time
np.random.seed(1)
n = 500_000
df = pl.DataFrame({
    "id":       np.arange(n),
    "group":    np.random.choice(["X","Y","Z","W"], n),
    "value":    np.random.randn(n),
    "amount":   np.random.lognormal(3, 1.5, n),
    "flag":     np.random.randint(0, 2, n).astype(bool),
})
tmp_parquet = tempfile.mktemp(suffix=".parquet")
tmp_csv     = tempfile.mktemp(suffix=".csv")
# Write and read Parquet vs CSV
t0 = time.perf_counter()
df.write_parquet(tmp_parquet, compression="snappy")
t1 = time.perf_counter()
df.write_csv(tmp_csv)
t2 = time.perf_counter()
df_pq = pl.read_parquet(tmp_parquet)
t3 = time.perf_counter()
df_csv = pl.read_csv(tmp_csv)
t4 = time.perf_counter()
pq_mb  = os.path.getsize(tmp_parquet) / 1e6
csv_mb = os.path.getsize(tmp_csv) / 1e6
print(f"Parquet: write={t1-t0:.3f}s read={t3-t2:.3f}s size={pq_mb:.1f}MB")
print(f"CSV:     write={t2-t1:.3f}s read={t4-t3:.3f}s size={csv_mb:.1f}MB")
for f in [tmp_parquet, tmp_csv]: os.unlink(f)

import polars as pl
import pyarrow as pa
import pyarrow.parquet as pq
import numpy as np
import tempfile, os
np.random.seed(2)
n = 200_000
df = pl.DataFrame({
    "year":   np.random.choice([2021,2022,2023], n),
    "region": np.random.choice(["NA","EU","APAC"], n),
    "product":np.random.choice(["A","B","C","D"], n),
    "revenue":np.random.lognormal(5, 1, n),
    "units":  np.random.poisson(50, n),
})
# Write partitioned parquet (by year+region) via pyarrow
arrow_table = df.to_arrow()
tmpdir = tempfile.mkdtemp()
pq.write_to_dataset(arrow_table, root_path=tmpdir,
                    partition_cols=["year","region"])
# Read only 2023 NA partition with Polars scan
result = (
    pl.scan_parquet(f"{tmpdir}/year=2023/region=NA/**/*.parquet")
    .group_by("product")
    .agg(pl.col("revenue").sum(), pl.col("units").sum())
    .collect()
)
print(f"2023 NA partition result:"); print(result)
import shutil; shutil.rmtree(tmpdir)

import polars as pl
import numpy as np

np.random.seed(42)
n = 10_000

# Create a DataFrame
df = pl.DataFrame({
    "id": range(n),
    "category": np.random.choice(["A", "B", "C", "D"], n),
    "value": np.random.randn(n) * 100,
    "quantity": np.random.randint(1, 100, n),
    "active": np.random.choice([True, False], n),
})

# Eager (immediate execution)
eager_result = (df
    .filter(pl.col("active"))
    .filter(pl.col("value") > 0)
    .select(["category", "value", "quantity"])
    .group_by("category")
    .agg(pl.col("value").mean().alias("avg_value"),
         pl.col("quantity").sum().alias("total_qty"))
)
print("Eager result:")
print(eager_result.sort("category"))

# Lazy (deferred execution)
lazy_result = (df.lazy()
    .filter(pl.col("active"))
    .filter(pl.col("value") > 0)
    .select(["category", "value", "quantity"])
    .group_by("category")
    .agg(pl.col("value").mean().alias("avg_value"),
         pl.col("quantity").sum().alias("total_qty"))
    .collect()  # execute here
)
print("\nLazy result (same output):")
print(lazy_result.sort("category"))
print(f"\nResults match: {eager_result.sort('category').equals(lazy_result.sort('category'))}")

import polars as pl
import numpy as np

np.random.seed(42)
n = 50_000
df = pl.DataFrame({
    "user_id": range(n),
    "age": np.random.randint(18, 70, n),
    "country": np.random.choice(["US", "UK", "DE", "FR", "JP"], n),
    "purchase_amount": np.random.exponential(50, n),
    "is_premium": np.random.choice([True, False], n),
})

# Build lazy query
q = (df.lazy()
     .filter(pl.col("is_premium") & (pl.col("age") >= 25))
     .filter(pl.col("purchase_amount") > 20)
     .select(["user_id", "country", "purchase_amount", "age"])
     .group_by("country")
     .agg([
         pl.col("purchase_amount").mean().alias("avg_purchase"),
         pl.col("purchase_amount").max().alias("max_purchase"),
         pl.len().alias("n_users"),
     ])
     .sort("avg_purchase", descending=True)
)

# Inspect the query plan
print("Optimized query plan:")
print(q.explain(optimized=True))

# Execute
result = q.collect()
print("\nResult:")
print(result)

import polars as pl
import numpy as np, tempfile, os

np.random.seed(42)

# Create a large CSV file for streaming demo
with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as f:
    csv_path = f.name
    f.write("id,category,value,quantity\n")
    for i in range(100_000):
        cat = np.random.choice(["X","Y","Z"])
        val = round(np.random.randn()*100, 2)
        qty = np.random.randint(1, 50)
        f.write(f"{i},{cat},{val},{qty}\n")

# Stream processing: process without loading all data into memory
result = (
    pl.scan_csv(csv_path)  # scan_csv returns LazyFrame
    .filter(pl.col("value") > 0)
    .group_by("category")
    .agg([
        pl.col("value").sum().alias("total_value"),
        pl.col("quantity").mean().alias("avg_qty"),
        pl.len().alias("count"),
    ])
    .sort("total_value", descending=True)
    .collect(streaming=True)  # stream=True for memory-efficient processing
)

print(f"File size: {os.path.getsize(csv_path)/1024:.0f} KB")
print("Streaming result:")
print(result)
os.unlink(csv_path)

import polars as pl
import numpy as np

np.random.seed(42)
n = 5000
df = pl.DataFrame({
    "date": pl.date_range(pl.date(2024, 1, 1), pl.date(2024, 12, 31), "1d", eager=True)[:n % 366].extend(
        pl.date_range(pl.date(2024, 1, 1), pl.date(2024, 12, 31), "1d", eager=True)
    )[:n],
    "product": np.random.choice(["Widget", "Gadget", "Doohickey", "Thingamajig"], n),
    "region": np.random.choice(["North", "South", "East", "West"], n),
    "sales": np.random.exponential(100, n).round(2),
    "units": np.random.randint(1, 50, n),
    "returned": np.random.choice([True, False], n, p=[0.1, 0.9]),
})

# Multi-column groupby with complex aggregations
result = df.group_by(["product", "region"]).agg([
    pl.col("sales").sum().alias("total_sales"),
    pl.col("sales").mean().round(2).alias("avg_sale"),
    pl.col("sales").std().round(2).alias("std_sale"),
    pl.col("units").sum().alias("total_units"),
    pl.col("returned").mean().round(4).alias("return_rate"),
    pl.len().alias("n_transactions"),
    pl.col("sales").quantile(0.9).round(2).alias("p90_sale"),
]).sort(["product", "total_sales"], descending=[False, True])

print("Product x Region aggregation:")
print(result.head(8))
print(f"\nTotal rows: {result.shape[0]}")

import polars as pl
import numpy as np

np.random.seed(42)
n = 200
dates = pl.date_range(pl.date(2024, 1, 1), period=f"{n}d", interval="1d", eager=True)
df = pl.DataFrame({
    "date": dates,
    "value": np.cumsum(np.random.randn(n)) + 50,
    "volume": np.random.randint(100, 1000, n),
})

# Rolling aggregations (7-day moving average)
df_rolling = df.with_columns([
    pl.col("value").rolling_mean(window_size=7).alias("ma7"),
    pl.col("value").rolling_mean(window_size=30).alias("ma30"),
    pl.col("value").rolling_std(window_size=7).alias("std7"),
    pl.col("volume").rolling_sum(window_size=7).alias("vol7"),
])
print("Rolling aggregations:")
print(df_rolling.tail(5))

# Dynamic groupby (group by month)
df_monthly = (df
    .with_columns(pl.col("date").dt.month().alias("month"))
    .group_by("month")
    .agg([
        pl.col("value").mean().round(2).alias("avg_val"),
        pl.col("value").max().alias("max_val"),
        pl.col("volume").sum().alias("total_vol"),
    ])
    .sort("month")
)
print("\nMonthly aggregation:")
print(df_monthly)

import polars as pl
import numpy as np

np.random.seed(42)
n = 1000
df = pl.DataFrame({
    "quarter": np.random.choice(["Q1","Q2","Q3","Q4"], n),
    "product": np.random.choice(["A","B","C"], n),
    "sales": np.random.exponential(100, n).round(2),
    "target": np.random.uniform(80, 120, n).round(2),
})

# Conditional aggregation: count by condition
result = df.group_by(["quarter", "product"]).agg([
    pl.col("sales").sum().alias("total_sales"),
    (pl.col("sales") > pl.col("target")).sum().alias("above_target"),
    pl.len().alias("n"),
    ((pl.col("sales") > pl.col("target")).sum() / pl.len()).round(3).alias("hit_rate"),
])
print("Conditional aggregation:")
print(result.sort(["quarter", "product"]).head(8))

# Pivot: reshape data (like pandas pivot_table)
pivot = (df.group_by(["quarter", "product"])
    .agg(pl.col("sales").sum().alias("total"))
    .pivot(index="quarter", on="product", values="total", aggregate_function="sum")
    .sort("quarter"))
print("\nPivot table (quarter x product sales):")
print(pivot)

import polars as pl
import numpy as np

np.random.seed(42)

# Customers table
customers = pl.DataFrame({
    "customer_id": range(1, 11),
    "name": [f"Customer_{i}" for i in range(1, 11)],
    "tier": np.random.choice(["Gold", "Silver", "Bronze"], 10),
})

# Orders table
orders = pl.DataFrame({
    "order_id": range(1, 21),
    "customer_id": np.random.choice(range(1, 14), 20),  # some customers missing
    "amount": np.random.exponential(100, 20).round(2),
    "status": np.random.choice(["completed", "pending", "cancelled"], 20),
})

# Inner join: only matching customers
inner = customers.join(orders, on="customer_id", how="inner")
print(f"Inner join: {len(inner)} rows (customers with orders)")

# Left join: all customers, NaN for missing orders
left = customers.join(orders, on="customer_id", how="left")
print(f"Left join:  {len(left)} rows (all customers)")
print(f"  Customers with no orders: {left.filter(pl.col('order_id').is_null()).shape[0]}")

# Full/outer join
outer = customers.join(orders, on="customer_id", how="full")
print(f"Full join:  {len(outer)} rows")

# Aggregation join: aggregate before joining
order_summary = (orders
    .group_by("customer_id")
    .agg([pl.col("amount").sum().alias("total_spend"),
          pl.len().alias("order_count")])
)
enriched = customers.join(order_summary, on="customer_id", how="left")
print("\nCustomer enriched data:")
print(enriched.head(5))

import polars as pl
import numpy as np

np.random.seed(42)

# Product table
products = pl.DataFrame({
    "product_id": range(1, 21),
    "name": [f"Product_{i}" for i in range(1, 21)],
    "price": np.random.uniform(10, 200, 20).round(2),
    "category": np.random.choice(["Electronics","Clothing","Books"], 20),
})

# Sold products (subset)
sold = pl.DataFrame({
    "product_id": np.random.choice(range(1, 21), 12, replace=False),
    "units_sold": np.random.randint(5, 100, 12),
})

# Semi-join: products that have been sold (filtering join)
has_sold = products.join(sold, on="product_id", how="semi")
print(f"Products sold: {len(has_sold)}")

# Anti-join: products that have NOT been sold
never_sold = products.join(sold, on="product_id", how="anti")
print(f"Products never sold: {len(never_sold)}")
print(never_sold.select(["product_id", "name", "price"]).head(5))

# Cross join (cartesian product)
sizes = pl.DataFrame({"size": ["S", "M", "L", "XL"]})
colors = pl.DataFrame({"color": ["Red", "Blue", "Green"]})
variants = sizes.join(colors, how="cross")
print(f"\nProduct variants (cross join): {len(variants)} combinations")
print(variants.head(6))

import polars as pl
import numpy as np, time

np.random.seed(42)
n = 100_000

# Large DataFrames for join performance demo
df_left = pl.DataFrame({
    "id": np.arange(n),
    "value_a": np.random.randn(n),
    "group": np.random.choice(list("ABCDE"), n),
})
df_right = pl.DataFrame({
    "id": np.random.randint(0, n, n),
    "value_b": np.random.randn(n),
    "score": np.random.uniform(0, 1, n),
})

# Standard join
t0 = time.time()
result = df_left.join(df_right, on="id", how="inner")
t1 = time.time()
print(f"Inner join ({len(result):,} rows): {(t1-t0)*1000:.1f}ms")

# Join with pre-filtering (reduce data first)
t0 = time.time()
left_filtered = df_left.filter(pl.col("group").is_in(["A", "B"]))
right_filtered = df_right.filter(pl.col("score") > 0.5)
result_filtered = left_filtered.join(right_filtered, on="id", how="inner")
t1 = time.time()
print(f"Pre-filtered join ({len(result_filtered):,} rows): {(t1-t0)*1000:.1f}ms")

# Multiple join keys
df_multi_left = pl.DataFrame({"key1": list("AABBC"), "key2": [1,2,1,2,1], "val": range(5)})
df_multi_right = pl.DataFrame({"key1": list("AABC"), "key2": [1,2,2,1], "label": list("WXYZ")})
multi_join = df_multi_left.join(df_multi_right, on=["key1", "key2"], how="left")
print("\nMulti-key join:")
print(multi_join)

import polars as pl
import numpy as np

# Sample text data
df = pl.DataFrame({
    "email": ["alice@example.com", "BOB.SMITH@Gmail.COM", "  charlie@test.org  ",
              "invalid-email", "diana.prince@hero.net"],
    "full_name": ["Alice Johnson", "Bob Smith", "Charlie Brown",
                  "Diana Prince", "Eve Williams"],
    "phone": ["(555) 123-4567", "555.234.5678", "1-555-345-6789",
              "5554567890", "555 456 7890"],
    "description": ["  Product is great! ", "NEEDS improvement",
                    "works as expected...", "  EXCELLENT quality  ", "good value"],
})

# String operations
result = df.with_columns([
    pl.col("email").str.to_lowercase().str.strip_chars().alias("email_clean"),
    pl.col("full_name").str.split(" ").list.first().alias("first_name"),
    pl.col("full_name").str.split(" ").list.last().alias("last_name"),
    pl.col("description").str.strip_chars().str.to_lowercase().alias("desc_clean"),
    pl.col("phone").str.replace_all(r"[^\d]", "").alias("phone_digits"),
])

print("String operations result:")
print(result.select(["email_clean", "first_name", "last_name", "phone_digits", "desc_clean"]))

# String contains/starts_with/ends_with
email_check = df.with_columns([
    pl.col("email").str.contains("@").alias("has_at"),
    pl.col("email").str.ends_with(".com").alias("is_dotcom"),
    pl.col("email").str.contains("@").alias("is_valid"),
])
print("\nEmail validation:")
print(email_check.select(["email", "has_at", "is_dotcom", "is_valid"]))

import polars as pl

# Text data with structured patterns
df = pl.DataFrame({
    "log_entry": [
        "2024-01-15 10:23:45 ERROR user_id=123 msg=Login failed",
        "2024-01-15 10:24:01 INFO user_id=456 msg=Login success",
        "2024-01-15 10:25:12 WARN user_id=789 msg=Rate limit exceeded",
        "2024-01-15 10:26:33 ERROR user_id=101 msg=DB connection timeout",
        "2024-01-15 10:27:55 INFO user_id=202 msg=Profile updated",
    ]
})

# Extract structured data from log entries
result = df.with_columns([
    # Extract date and time
    pl.col("log_entry").str.extract(r"(\d{4}-\d{2}-\d{2})", 1).alias("date"),
    pl.col("log_entry").str.extract(r"(\d{2}:\d{2}:\d{2})", 1).alias("time"),
    # Extract log level
    pl.col("log_entry").str.extract(r"(ERROR|WARN|INFO|DEBUG)", 1).alias("level"),
    # Extract user_id
    pl.col("log_entry").str.extract(r"user_id=(\d+)", 1).cast(pl.Int32).alias("user_id"),
    # Extract message
    pl.col("log_entry").str.extract(r"msg=(.+)$", 1).alias("message"),
])
print("Extracted log fields:")
print(result.drop("log_entry"))

# Count pattern matches
df2 = pl.DataFrame({
    "text": ["apple pie and apple juice", "banana split", "apple apple apple", "cherry"]
})
df2 = df2.with_columns(
    pl.col("text").str.count_matches("apple").alias("apple_count")
)
print("\nPattern count:")
print(df2)

import polars as pl

df = pl.DataFrame({
    "csv_row": ["Alice,30,Engineer,New York",
                "Bob,25,Designer,Los Angeles",
                "Charlie,35,Manager,Chicago"],
    "tags": ["python|pandas|polars", "javascript|react|node", "python|sklearn|pytorch"],
    "path": ["/home/user/data/2024/jan/sales.csv",
             "/home/user/data/2024/feb/orders.parquet",
             "/home/user/data/2024/mar/logs.json"],
})

# Split CSV string into columns
parsed = (df
    .with_columns(pl.col("csv_row").str.split(",").alias("parts"))
    .with_columns([
        pl.col("parts").list.get(0).alias("name"),
        pl.col("parts").list.get(1).cast(pl.Int32).alias("age"),
        pl.col("parts").list.get(2).alias("role"),
        pl.col("parts").list.get(3).alias("city"),
    ])
    .drop(["parts", "csv_row"])
)
print("Parsed CSV columns:")
print(parsed)

# Tags: split and explode
tags_exploded = (df
    .with_columns(pl.col("tags").str.split("|"))
    .explode("tags")
    .rename({"tags": "tag"})
    .select(["tag"])
    .group_by("tag")
    .agg(pl.len().alias("count"))
    .sort("count", descending=True)
)
print("\nTag frequency:")
print(tags_exploded)

# Extract filename from path
df_paths = df.with_columns(
    pl.col("path").str.split("/").list.last().alias("filename")
)
print("\nFilenames:", df_paths["filename"].to_list())

import polars as pl
import numpy as np
from datetime import date, timedelta

# Create datetime data
np.random.seed(42)
n = 200
start = date(2024, 1, 1)
dates = [start + timedelta(days=int(d)) for d in np.random.randint(0, 365, n)]

df = pl.DataFrame({
    "event_date": pl.Series(dates),
    "sales": np.random.exponential(100, n).round(2),
    "returns": np.random.randint(0, 5, n),
})

# Date component extraction
df_with_parts = df.with_columns([
    pl.col("event_date").dt.year().alias("year"),
    pl.col("event_date").dt.month().alias("month"),
    pl.col("event_date").dt.day().alias("day"),
    pl.col("event_date").dt.weekday().alias("weekday"),  # 0=Monday
    pl.col("event_date").dt.week().alias("week_num"),
    pl.col("event_date").dt.quarter().alias("quarter"),
])
print("Date components:")
print(df_with_parts.head(3))

# Date arithmetic
df_with_calc = df.with_columns([
    (pl.col("event_date") + pl.duration(days=30)).alias("due_date"),
    (pl.lit(date(2024, 12, 31)) - pl.col("event_date")).dt.total_days().alias("days_until_eoy"),
])
print("\nDate arithmetic:")
print(df_with_calc.select(["event_date", "due_date", "days_until_eoy"]).head(3))

# Groupby month
monthly = df.group_by(pl.col("event_date").dt.month().alias("month")).agg(
    pl.col("sales").sum().round(2).alias("total_sales"),
    pl.len().alias("n_transactions")
).sort("month")
print("\nMonthly summary:")
print(monthly)

import polars as pl
import numpy as np
from datetime import datetime, timedelta

np.random.seed(42)
n = 1000

# Minute-level timestamp data
start = datetime(2024, 1, 1, 0, 0, 0)
timestamps = [start + timedelta(minutes=i*5 + np.random.randint(0, 5)) for i in range(n)]

df = pl.DataFrame({
    "ts": pl.Series(timestamps).cast(pl.Datetime),
    "temp": 20 + np.cumsum(np.random.randn(n) * 0.5),
    "pressure": 1013 + np.cumsum(np.random.randn(n) * 0.2),
})

# Truncate to hourly resolution
df_hourly = df.with_columns(
    pl.col("ts").dt.truncate("1h").alias("hour")
).group_by("hour").agg([
    pl.col("temp").mean().round(2).alias("avg_temp"),
    pl.col("temp").min().alias("min_temp"),
    pl.col("temp").max().alias("max_temp"),
    pl.col("pressure").mean().round(2).alias("avg_pressure"),
    pl.len().alias("n_readings"),
]).sort("hour")

print("Hourly resampled data (first 5 hours):")
print(df_hourly.head(5))

# Daily summary
df_daily = df.with_columns(
    pl.col("ts").dt.date().alias("date")
).group_by("date").agg([
    pl.col("temp").mean().round(2).alias("daily_avg_temp"),
    pl.col("temp").std().round(3).alias("temp_std"),
]).sort("date")
print("\nDaily summary (first 3 days):")
print(df_daily.head(3))

import polars as pl
from datetime import datetime, timezone, timedelta

# Timezone-aware datetime operations
timestamps_utc = pl.Series([
    datetime(2024, 3, 15, 12, 0, 0),
    datetime(2024, 6, 15, 18, 30, 0),
    datetime(2024, 12, 1, 8, 45, 0),
]).cast(pl.Datetime).dt.replace_time_zone("UTC")

df = pl.DataFrame({
    "event_utc": timestamps_utc,
    "duration_sec": pl.Series([3600, 7200, 1800]),  # seconds
})

# Convert to different timezones
df_tz = df.with_columns([
    pl.col("event_utc").dt.convert_time_zone("America/New_York").alias("event_eastern"),
    pl.col("event_utc").dt.convert_time_zone("Europe/London").alias("event_london"),
    pl.col("event_utc").dt.convert_time_zone("Asia/Tokyo").alias("event_tokyo"),
])
print("Multi-timezone timestamps:")
print(df_tz.select(["event_utc", "event_eastern", "event_tokyo"]))

# Duration calculations
df_duration = df.with_columns([
    pl.duration(seconds=pl.col("duration_sec")).alias("duration"),
    (pl.col("event_utc") + pl.duration(hours=2)).alias("event_plus_2h"),
])
print("\nDuration operations:")
print(df_duration.select(["event_utc", "duration", "event_plus_2h"]))

# Business day calculation (using weekday filter)
n_workdays = (pl.date_range(pl.date(2024, 1, 1), pl.date(2024, 1, 31), "1d", eager=True)
    .filter(pl.Series([d.weekday() < 5 for d in pl.date_range(
        pl.date(2024, 1, 1), pl.date(2024, 1, 31), "1d", eager=True).to_list()]))
    .len())
print(f"\nWorkdays in January 2024: {n_workdays}")

import polars as pl
import numpy as np

np.random.seed(42)
df = pl.DataFrame({
    "date": pl.date_range(pl.date(2024, 1, 1), period="30d", interval="1d", eager=True),
    "product": np.tile(["A", "B", "C"], 10),
    "region": np.random.choice(["East", "West"], 30),
    "sales": np.random.exponential(100, 30).round(2),
})

# Ranking within groups (dense_rank)
df_ranked = df.with_columns([
    pl.col("sales").rank("dense", descending=True).over("product").alias("rank_in_product"),
    pl.col("sales").rank("ordinal", descending=True).over("region").alias("rank_in_region"),
])
print("Rankings:")
print(df_ranked.select(["product", "region", "sales", "rank_in_product", "rank_in_region"]).head(6))

# Cumulative sum and running stats
df_cumulative = df.sort("date").with_columns([
    pl.col("sales").cum_sum().over("product").alias("cumsum_by_product"),
    pl.col("sales").cum_mean().over("product").alias("cum_mean_by_product"),
])
print("\nCumulative stats by product:")
print(df_cumulative.select(["date", "product", "sales",
                             "cumsum_by_product", "cum_mean_by_product"]).head(6))

# Group-level stats without collapsing
df_group_stats = df.with_columns([
    pl.col("sales").mean().over("product").alias("product_avg"),
    pl.col("sales").sum().over("region").alias("region_total"),
    (pl.col("sales") / pl.col("sales").sum().over("product")).alias("pct_of_product"),
])
print("\nGroup stats (no collapse):")
print(df_group_stats.head(6))

import polars as pl
import numpy as np

np.random.seed(42)
n = 60
df = pl.DataFrame({
    "date": pl.date_range(pl.date(2024, 1, 1), period=f"{n}d", interval="1d", eager=True),
    "product": ["A"]*(n//2) + ["B"]*(n//2),
    "sales": np.random.exponential(100, n).round(2),
})

# Sort before lag/lead
df = df.sort(["product", "date"])

# Lag/lead features for time series
df_features = df.with_columns([
    pl.col("sales").shift(1).over("product").alias("lag_1"),
    pl.col("sales").shift(7).over("product").alias("lag_7"),
    pl.col("sales").shift(-1).over("product").alias("lead_1"),
    # Difference from previous period
    (pl.col("sales") - pl.col("sales").shift(1).over("product")).alias("delta"),
    # Percent change
    ((pl.col("sales") - pl.col("sales").shift(1).over("product")) /
     pl.col("sales").shift(1).over("product")).round(4).alias("pct_change"),
    # Rolling mean (window function style)
    pl.col("sales").rolling_mean(window_size=7).over("product").alias("rolling_7"),
])

print("Lag/lead features:")
print(df_features.filter(pl.col("product")=="A").head(8))

import polars as pl
import numpy as np

np.random.seed(42)
n = 500
df = pl.DataFrame({
    "customer_id": range(n),
    "age": np.random.randint(18, 75, n),
    "income": np.random.exponential(50000, n).round(0),
    "credit_score": np.random.randint(300, 850, n),
    "loan_amount": np.random.exponential(15000, n).round(0),
    "on_time_payments": np.random.randint(0, 60, n),
    "total_payments": np.random.randint(12, 60, n),
})

# Complex conditional expressions
df_scored = df.with_columns([
    # Age category
    pl.when(pl.col("age") < 25).then(pl.lit("Young"))
      .when(pl.col("age") < 45).then(pl.lit("Middle"))
      .otherwise(pl.lit("Senior")).alias("age_group"),

    # Credit tier
    pl.when(pl.col("credit_score") >= 750).then(pl.lit("Excellent"))
      .when(pl.col("credit_score") >= 650).then(pl.lit("Good"))
      .when(pl.col("credit_score") >= 550).then(pl.lit("Fair"))
      .otherwise(pl.lit("Poor")).alias("credit_tier"),

    # Payment ratio
    (pl.col("on_time_payments") / pl.col("total_payments")).round(3).alias("payment_ratio"),

    # Approval decision
    pl.when(
        (pl.col("credit_score") >= 650) &
        (pl.col("income") >= 30000) &
        (pl.col("loan_amount") <= pl.col("income") * 5)
    ).then(pl.lit("Approved")).otherwise(pl.lit("Denied")).alias("decision"),
])

print("Loan applications with features:")
print(df_scored.head(8))

# Summary by tier
print("\nApproval by credit tier:")
print(df_scored.group_by("credit_tier").agg([
    pl.len().alias("n"),
    (pl.col("decision")=="Approved").mean().round(3).alias("approval_rate"),
]).sort("credit_tier"))

import polars as pl
import pandas as pd
import numpy as np
import time

np.random.seed(42)
n = 500_000

# Create test data
data = {
    "id": range(n),
    "category": np.random.choice(list("ABCDEFGHIJ"), n),
    "sub_cat": np.random.choice(list("XYZ"), n),
    "value1": np.random.randn(n) * 100,
    "value2": np.random.exponential(50, n),
    "count": np.random.randint(1, 100, n),
}

df_pd = pd.DataFrame(data)
df_pl = pl.DataFrame(data)

# Benchmark: complex groupby + aggregation
def pandas_groupby():
    return (df_pd
        .groupby(["category", "sub_cat"])
        .agg({"value1": ["mean", "std", "sum"], "value2": ["mean", "max"], "count": "sum"})
    )

def polars_groupby():
    return (df_pl.lazy()
        .group_by(["category", "sub_cat"])
        .agg([
            pl.col("value1").mean().alias("v1_mean"),
            pl.col("value1").std().alias("v1_std"),
            pl.col("value1").sum().alias("v1_sum"),
            pl.col("value2").mean().alias("v2_mean"),
            pl.col("value2").max().alias("v2_max"),
            pl.col("count").sum().alias("count_sum"),
        ])
        .collect()
    )

# Time both
for name, fn in [("Pandas", pandas_groupby), ("Polars", polars_groupby)]:
    times = []
    for _ in range(3):
        t0 = time.time()
        result = fn()
        times.append((time.time()-t0)*1000)
    print(f"{name}: avg={sum(times)/len(times):.1f}ms, min={min(times):.1f}ms")

import polars as pl
import pandas as pd
import numpy as np
import time

np.random.seed(42)
n_main = 200_000
n_ref = 1_000

main_data = {
    "user_id": np.random.randint(0, n_ref, n_main),
    "product": np.random.choice(list("ABCDE"), n_main),
    "amount": np.random.exponential(50, n_main),
    "date": np.random.randint(0, 365, n_main),
}
ref_data = {
    "user_id": range(n_ref),
    "tier": np.random.choice(["Gold", "Silver", "Bronze"], n_ref),
    "discount": np.random.uniform(0, 0.3, n_ref).round(3),
}

df_pd_main = pd.DataFrame(main_data)
df_pd_ref = pd.DataFrame(ref_data)
df_pl_main = pl.DataFrame(main_data)
df_pl_ref = pl.DataFrame(ref_data)

def pandas_join_filter():
    merged = df_pd_main.merge(df_pd_ref, on="user_id", how="left")
    filtered = merged[(merged["tier"]=="Gold") & (merged["amount"] > 50)]
    return filtered.groupby("product")["amount"].agg(["sum","mean","count"])

def polars_join_filter():
    return (df_pl_main.lazy()
        .join(df_pl_ref.lazy(), on="user_id", how="left")
        .filter((pl.col("tier")=="Gold") & (pl.col("amount") > 50))
        .group_by("product")
        .agg([pl.col("amount").sum(), pl.col("amount").mean(), pl.len()])
        .collect())

for name, fn in [("Pandas", pandas_join_filter), ("Polars", polars_join_filter)]:
    t0 = time.time()
    result = fn()
    t1 = time.time()
    print(f"{name}: {(t1-t0)*1000:.1f}ms, rows={len(result)}")

print("\nSpeedups are typically 5-50x for large datasets on multi-core machines.")

import polars as pl
import pandas as pd
import numpy as np
import sys

np.random.seed(42)
n = 100_000

# Memory usage comparison
data = {
    "id": range(n),
    "name": [f"item_{i}" for i in range(n)],
    "value": np.random.randn(n),
    "category": np.random.choice(["A","B","C","D","E"], n),
    "flag": np.random.choice([True, False], n),
}

df_pd = pd.DataFrame(data)
df_pl = pl.DataFrame(data)

pd_memory = df_pd.memory_usage(deep=True).sum() / 1024**2
# Polars uses Arrow format - typically more efficient
print(f"Pandas memory: {pd_memory:.2f} MB")
print(f"Polars dtypes: {dict(zip(df_pl.columns, df_pl.dtypes))}")

# Pandas -> Polars migration cheatsheet
print("\nPandas -> Polars Migration:")
migrations = [
    ("df.groupby('col').mean()", "df.group_by('col').agg(pl.all().mean())"),
    ("df[df['x'] > 0]",          "df.filter(pl.col('x') > 0)"),
    ("df['col'].apply(fn)",       "df.with_columns(pl.col('col').map_elements(fn))"),
    ("pd.merge(df1, df2, on='k')","df1.join(df2, on='k', how='inner')"),
    ("df.rename({'a':'b'},...)",  "df.rename({'a': 'b'})"),
    ("df.drop('col', axis=1)",   "df.drop('col')"),
    ("df.sort_values('col')",    "df.sort('col')"),
    ("df.head(10)",              "df.head(10)"),
]
for pandas_code, polars_code in migrations:
    print(f"  Pandas: {pandas_code}")
    print(f"  Polars: {polars_code}\n")

import polars as pl
import numpy as np
import tempfile, os, time

np.random.seed(42)
n = 100_000

df = pl.DataFrame({
    "id": range(n),
    "category": np.random.choice(list("ABCDE"), n),
    "value": np.random.randn(n).round(4),
    "date": pl.date_range(pl.date(2024,1,1), period=f"{n}d", interval="1d", eager=True)[:n],
    "active": np.random.choice([True, False], n),
})

with tempfile.TemporaryDirectory() as tmpdir:
    # Write formats
    csv_path = os.path.join(tmpdir, "data.csv")
    parquet_path = os.path.join(tmpdir, "data.parquet")
    ipc_path = os.path.join(tmpdir, "data.arrow")

    df.write_csv(csv_path)
    df.write_parquet(parquet_path)
    df.write_ipc(ipc_path)

    sizes = {
        "CSV": os.path.getsize(csv_path),
        "Parquet": os.path.getsize(parquet_path),
        "Arrow IPC": os.path.getsize(ipc_path),
    }
    print("File sizes:")
    for fmt, size in sizes.items():
        print(f"  {fmt}: {size/1024:.1f} KB ({size/sizes['CSV']*100:.0f}% of CSV)")

    # Read speeds
    for fmt, read_fn, path in [
        ("CSV",     lambda: pl.read_csv(csv_path), csv_path),
        ("Parquet", lambda: pl.read_parquet(parquet_path), parquet_path),
        ("Arrow",   lambda: pl.read_ipc(ipc_path), ipc_path),
    ]:
        t0 = time.time()
        result = read_fn()
        t_ms = (time.time()-t0)*1000
        print(f"  {fmt} read: {t_ms:.1f}ms, shape={result.shape}")

print("\nRecommendation: Use Parquet for storage (5-10x smaller + faster reads)")

import polars as pl
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

np.random.seed(42)
n = 2000

# Create Polars DataFrame
df = pl.DataFrame({
    "age": np.random.randint(18, 70, n),
    "income": np.random.exponential(50000, n).round(0),
    "credit_score": np.random.randint(300, 850, n),
    "debt_ratio": np.random.uniform(0, 1, n).round(3),
    "employment_years": np.random.randint(0, 40, n),
    "category": np.random.choice(["A","B","C"], n),
})

# Feature engineering in Polars
df_features = df.with_columns([
    (pl.col("income") / (pl.col("age") + 1)).alias("income_per_age"),
    (pl.col("credit_score") / 850).alias("credit_norm"),
    pl.col("category").cast(pl.Categorical).cast(pl.Int8).alias("category_code"),
]).drop("category")

# Target
target = (df["credit_score"] > 650).cast(pl.Int32)

# Convert to numpy for sklearn
X = df_features.to_numpy()
y = target.to_numpy()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

scaler = StandardScaler()
X_train_sc = scaler.fit_transform(X_train)
X_test_sc = scaler.transform(X_test)

model = RandomForestClassifier(n_estimators=100, random_state=42)
model.fit(X_train_sc, y_train)
print(classification_report(y_test, model.predict(X_test_sc)))

# Feature importance back to Polars
importances = pl.DataFrame({
    "feature": df_features.columns,
    "importance": model.feature_importances_,
}).sort("importance", descending=True)
print("Feature importances:")
print(importances)

import polars as pl
import numpy as np
import tempfile, os

np.random.seed(42)
n = 1000

# Simulate messy input data
data_rows = []
for i in range(n):
    row = f"{i},{np.random.choice(['A','B','C'])},"
    val = np.random.randn()
    if np.random.random() < 0.05: row += "NULL"  # missing value
    elif np.random.random() < 0.02: row += "N/A"  # invalid
    else: row += f"{val:.4f}"
    row += f",{np.random.randint(1, 100)}"
    data_rows.append(row)

with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False) as f:
    f.write("id,category,value,count\n")
    f.write("\n".join(data_rows))
    csv_path = f.name

def run_etl(path):
    df = pl.read_csv(path, null_values=["NULL", "N/A", ""])

    # Validate schema
    n_before = len(df)
    df = df.filter(pl.col("value").is_not_null())
    n_after = len(df)
    print(f"Removed {n_before - n_after} null rows ({(n_before-n_after)/n_before*100:.1f}%)")

    # Type casting and validation
    df = df.with_columns([
        pl.col("value").cast(pl.Float64),
        pl.col("count").cast(pl.Int32),
        pl.col("category").cast(pl.Categorical),
    ])

    # Remove outliers (|z-score| > 3)
    mean_val = df["value"].mean()
    std_val = df["value"].std()
    df = df.filter(((pl.col("value") - mean_val) / std_val).abs() <= 3)

    # Aggregate
    result = df.group_by("category").agg([
        pl.col("value").mean().round(4).alias("avg_value"),
        pl.col("count").sum().alias("total_count"),
        pl.len().alias("n_rows"),
    ]).sort("category")

    return df, result

clean_df, summary = run_etl(csv_path)
print("\nETL Summary:")
print(summary)
os.unlink(csv_path)