In [ ]:
Copied!
import marimo as mo
import marimo as mo
mloda demo: How can we make feature engineering shareable?¶
Define dummy data as plugin¶
In [ ]:
Copied!
import numpy as np
from mloda.provider import FeatureGroup, DataCreator
class DummyData(FeatureGroup):
@classmethod
def calculate_feature(cls, data, features):
n_samples = features.get_options_key("n_samples") or 100
return {
"age": np.random.randint(18, 80, n_samples),
"weight": np.random.normal(70, 15, n_samples),
"state": np.random.choice(["CA", "NY", "TX", "FL"], n_samples),
"gender": np.random.choice(["M", "F"], n_samples),
}
@classmethod
def input_data(cls):
return DataCreator({"age", "weight", "state", "gender"})
import numpy as np
from mloda.provider import FeatureGroup, DataCreator
class DummyData(FeatureGroup):
@classmethod
def calculate_feature(cls, data, features):
n_samples = features.get_options_key("n_samples") or 100
return {
"age": np.random.randint(18, 80, n_samples),
"weight": np.random.normal(70, 15, n_samples),
"state": np.random.choice(["CA", "NY", "TX", "FL"], n_samples),
"gender": np.random.choice(["M", "F"], n_samples),
}
@classmethod
def input_data(cls):
return DataCreator({"age", "weight", "state", "gender"})
Request mlodaAPI to create features¶
In [ ]:
Copied!
# We load dependencies.
from mloda.user import mloda, PluginLoader
PluginLoader.all()
# Load plugins into namespace so compute frameworks register.
_result = mloda.run_all(
["age", "weight", "state", "gender"], compute_frameworks=["PandasDataFrame", "PyArrowTable"]
)
print(_result)
# We load dependencies.
from mloda.user import mloda, PluginLoader
PluginLoader.all()
# Load plugins into namespace so compute frameworks register.
_result = mloda.run_all(
["age", "weight", "state", "gender"], compute_frameworks=["PandasDataFrame", "PyArrowTable"]
)
print(_result)
[ age state gender weight 0 55 NY F 92.048332 1 36 TX M 74.012635 2 40 FL M 71.355146 3 77 CA M 55.414713 4 36 TX M 60.180648 .. ... ... ... ... 95 63 CA M 47.563638 96 42 TX M 61.098454 97 35 NY F 82.741975 98 35 FL M 48.767208 99 63 CA M 92.380329 [100 rows x 4 columns]]
Chain features - automatic dependency resolution¶
In [ ]:
Copied!
# Load plugin into namespace again
_result = mloda.run_all(["age__sum_aggr"], compute_frameworks=["PolarsLazyDataFrame"])
print(_result)
# Load plugin into namespace again
_result = mloda.run_all(["age__sum_aggr"], compute_frameworks=["PolarsLazyDataFrame"])
print(_result)
[shape: (100, 1) ┌───────────────┐ │ age__sum_aggr │ │ --- │ │ i64 │ ╞═══════════════╡ │ 4846 │ │ 4846 │ │ 4846 │ │ 4846 │ │ 4846 │ │ … │ │ 4846 │ │ 4846 │ │ 4846 │ │ 4846 │ │ 4846 │ └───────────────┘]
As long as the plugins exists, we can run any datatransformation.
What is behind the "age__sum_aggr" syntax?¶
In [ ]:
Copied!
from mloda.user import Feature, Options
feature = Feature(
name="CustomConfiguration",
options=Options(context={"aggregation_type": "sum", "in_features": Feature("age", options={"n_samples": 5})}),
)
_result = mloda.run_all([feature], compute_frameworks=["PolarsLazyDataFrame"])
print(_result)
from mloda.user import Feature, Options
feature = Feature(
name="CustomConfiguration",
options=Options(context={"aggregation_type": "sum", "in_features": Feature("age", options={"n_samples": 5})}),
)
_result = mloda.run_all([feature], compute_frameworks=["PolarsLazyDataFrame"])
print(_result)
[shape: (5, 1) ┌─────────────────────┐ │ CustomConfiguration │ │ --- │ │ i64 │ ╞═════════════════════╡ │ 246 │ │ 246 │ │ 246 │ │ 246 │ │ 246 │ └─────────────────────┘]
How the chaining essentially works¶
class FeatureGroup(ABC):
def input_features(self, options: Options, feature_name: FeatureName) -> set[Feature] | None:
# In principle, the resolver checks if the feature group depends on another input feature
# -> then adds it to the chain of features which need to be resolved
if feature_name contains "input_feature__sum_aggr":
return input_feature
# How does mloda knows a feature matches a feature group?
# Customizable, but some good guesses
@classmethod
def match_feature_group_criteria(
cls,
feature_name: FeatureName | str,
options: Options,
data_access_collection: DataAccessCollection | None = None,
) -> bool:
Now we have chaining and matching. Why do we do this?¶
class FeatureGroup(ABC):
@classmethod
def calculate_feature(cls, data: Any, features: FeatureSet) -> Any:
\"\"\"
This function should be used to calculate the feature.
\"\"\"
# data is the incoming data from other feature dependencies or data via mloda
# features is the configuration
Business knowledge is in the data and in the configuration, but not in the plugin definition.¶
Big idea¶
Separate business logic from transformation logic:
- Plugins = generic transformations (shareable across companies)
- Data + Config = your business knowledge (stays private)
→ Stop rewriting "sum of a column" at every company
→ Build a shared ecosystem of feature engineering plugins