uv pip install microcalibrate

from microcalibrate import Calibration
import numpy as np
import pandas as pd

# Survey data (1,000 households)
weights = np.ones(1000)  # Initial weights

# Estimates (how much each household contributes to targets)
estimate_matrix = pd.DataFrame({
    'total_income': household_incomes,      # Each household's income
    'total_employed': household_employment  # 1 if employed, 0 if not
})

# Known population targets (benchmarks)
targets = np.array([
    50_000_000,  # Total income in population
    600,         # Total employed people
])

# Calibrate
cal = Calibration(
    weights=weights,
    targets=targets,
    estimate_matrix=estimate_matrix,
    l0_lambda=0.01  # Sparsity penalty
)

# Optimize weights
new_weights = cal.calibrate(max_iter=1000)

# Check results
achieved = (estimate_matrix.values.T @ new_weights)
print(f"Target: {targets}")
print(f"Achieved: {achieved}")
print(f"Non-zero weights: {(new_weights > 0).sum()} / {len(weights)}")

# L0 encourages many weights to be exactly zero
cal = Calibration(
    weights=weights,
    targets=targets,
    estimate_matrix=estimate_matrix,
    l0_lambda=0.01  # Higher = more sparse
)

# Without L0
cal_dense = Calibration(..., l0_lambda=0.0)
weights_dense = cal_dense.calibrate()

# With L0
cal_sparse = Calibration(..., l0_lambda=0.01)
weights_sparse = cal_sparse.calibrate()

print(f"Dense: {(weights_dense > 0).sum()} households")
print(f"Sparse: {(weights_sparse > 0).sum()} households")
# Sparse might use 60% fewer households while matching same targets

from microcalibrate import tune_hyperparameters

# Find best l0_lambda using cross-validation
best_lambda, results = tune_hyperparameters(
    weights=weights,
    targets=targets,
    estimate_matrix=estimate_matrix,
    lambda_min=1e-4,
    lambda_max=1e-1,
    n_trials=50
)

print(f"Best lambda: {best_lambda}")

from microcalibrate import evaluate_robustness

# Holdout validation
robustness = evaluate_robustness(
    weights=weights,
    targets=targets,
    estimate_matrix=estimate_matrix,
    l0_lambda=0.01,
    n_folds=5
)

print(f"Mean error: {robustness['mean_error']}")
print(f"Std error: {robustness['std_error']}")

git clone https://github.com/PolicyEngine/microcalibrate
cd microcalibrate

tree microcalibrate/

# Key modules:
ls microcalibrate/
# - calibration.py - Main Calibration class
# - hyperparameter_tuning.py - Optuna integration
# - evaluation.py - Robustness testing
# - target_analysis.py - Target diagnostics

# Main calibration algorithm
cat microcalibrate/calibration.py

# Hyperparameter tuning
cat microcalibrate/hyperparameter_tuning.py

# Robustness evaluation
cat microcalibrate/evaluation.py

cat pyproject.toml

# Internal to microcalibrate
from l0 import HardConcrete

# Create gates for sample selection
gates = HardConcrete(
    n_items=len(weights),
    temperature=temperature,
    init_mean=0.999
)

# Apply gates during optimization
effective_weights = weights * gates()

# L0 penalty encourages gates → 0 or 1
# Result: Many households get weight = 0 (sparse)

grep -n "HardConcrete\|l0" microcalibrate/calibration.py

# Target matching loss
target_loss = sum((achieved_targets - desired_targets)^2)

# L0 penalty (number of non-zero weights)
l0_penalty = l0_lambda * count_nonzero(weights)

# Total loss
total_loss = target_loss + l0_penalty

make test

# Or
pytest tests/ -v

cat tests/test_calibration.py
cat tests/test_hyperparameter_tuning.py

cd ../policyengine-us-data

# Find usage
grep -r "microcalibrate" policyengine_us_data/
grep -r "Calibration" policyengine_us_data/

from microcalibrate import Calibration

cal = Calibration(
    weights=initial_weights,
    targets=benchmark_values,
    estimate_matrix=contributions,
    l0_lambda=0.01
)

calibrated_weights = cal.calibrate(max_iter=1000)

from microcalibrate import tune_hyperparameters, Calibration

# Find best lambda
best_lambda, results = tune_hyperparameters(
    weights=weights,
    targets=targets,
    estimate_matrix=estimate_matrix
)

# Use best lambda
cal = Calibration(..., l0_lambda=best_lambda)
calibrated_weights = cal.calibrate()

# Multiple population targets
estimate_matrix = pd.DataFrame({
    'total_population': population_counts,
    'total_income': incomes,
    'total_employed': employment_indicators,
    'total_children': child_counts
})

targets = np.array([
    331_000_000,   # US population
    15_000_000_000_000,  # Total income
    160_000_000,   # Employed people
    73_000_000     # Children
])

cal = Calibration(weights, targets, estimate_matrix, l0_lambda=0.01)

import time

start = time.time()
weights = cal.calibrate()
print(f"Calibration took {time.time() - start:.1f}s")

# Try:
# - More iterations
# - Lower l0_lambda
# - Better initialization

cal = Calibration(..., l0_lambda=0.001)  # Lower sparsity penalty
weights = cal.calibrate(max_iter=5000)  # More iterations

# Check achieved vs desired
achieved = (estimate_matrix.values.T @ weights)
error = np.abs(achieved - targets) / targets
print(f"Relative errors: {error}")

# If large errors, l0_lambda may be too high

# Lower l0_lambda
cal = Calibration(..., l0_lambda=0.0001)