In [ ]:

Copied!

import marimo as mo
import marimo as mo

mloda demo: How can we make feature engineering shareable?¶

Define dummy data as plugin¶

In [ ]:

Copied!





import numpy as np
from mloda.provider import FeatureGroup, DataCreator

class DummyData(FeatureGroup):
    @classmethod
    def calculate_feature(cls, data, features):
        n_samples = features.get_options_key("n_samples") or 100
        return {
            "age": np.random.randint(18, 80, n_samples),
            "weight": np.random.normal(70, 15, n_samples),
            "state": np.random.choice(["CA", "NY", "TX", "FL"], n_samples),
            "gender": np.random.choice(["M", "F"], n_samples),
        }

    @classmethod
    def input_data(cls):
        return DataCreator({"age", "weight", "state", "gender"})
import numpy as np
from mloda.provider import FeatureGroup, DataCreator

class DummyData(FeatureGroup):
    @classmethod
    def calculate_feature(cls, data, features):
        n_samples = features.get_options_key("n_samples") or 100
        return {
            "age": np.random.randint(18, 80, n_samples),
            "weight": np.random.normal(70, 15, n_samples),
            "state": np.random.choice(["CA", "NY", "TX", "FL"], n_samples),
            "gender": np.random.choice(["M", "F"], n_samples),
        }

    @classmethod
    def input_data(cls):
        return DataCreator({"age", "weight", "state", "gender"})

Request mlodaAPI to create features¶

In [ ]:

Copied!





# We load dependencies.
from mloda.user import mloda, PluginLoader

PluginLoader.all()
# Load plugins into namespace so compute frameworks register.
_result = mloda.run_all(
    ["age", "weight", "state", "gender"], compute_frameworks=["PyArrowTable", "PandasDataFrame"]
)
print(_result)
# We load dependencies.
from mloda.user import mloda, PluginLoader

PluginLoader.all()
# Load plugins into namespace so compute frameworks register.
_result = mloda.run_all(
    ["age", "weight", "state", "gender"], compute_frameworks=["PyArrowTable", "PandasDataFrame"]
)
print(_result)

[       weight  age state gender
0   77.965069   33    TX      F
1   55.146970   70    CA      M
2   85.173198   62    CA      F
3   65.185641   43    TX      F
4   57.418842   63    FL      F
..        ...  ...   ...    ...
95  54.515487   19    NY      M
96  70.420550   45    TX      M
97  92.889109   64    TX      M
98  67.705347   21    CA      F
99  66.535843   50    FL      M

[100 rows x 4 columns]]

Alternative options to consume data¶

Apidata
Files
DBs
Streams
...

This is not the heart of mloda.

Chain features - automatic dependency resolution¶

In [ ]:

Copied!

# Load plugin into namespace again
_result = mloda.run_all(["age__sum_aggr"], compute_frameworks=["PolarsLazyDataFrame"])
print(_result)
# Load plugin into namespace again
_result = mloda.run_all(["age__sum_aggr"], compute_frameworks=["PolarsLazyDataFrame"])
print(_result)

[shape: (100, 1)
┌───────────────┐
│ age__sum_aggr │
│ ---           │
│ i64           │
╞═══════════════╡
│ 5010          │
│ 5010          │
│ 5010          │
│ 5010          │
│ 5010          │
│ …             │
│ 5010          │
│ 5010          │
│ 5010          │
│ 5010          │
│ 5010          │
└───────────────┘]

As long as the plugins exists, we can run any datatransformation.

What is behind the "age__sum_aggr" syntax?¶

In [ ]:

Copied!





from mloda.user import Feature, Options

feature = Feature(
    name="CustomConfiguration",
    options=Options(context={"aggregation_type": "sum", "in_features": Feature("age", options={"n_samples": 5})}),
)
_result = mloda.run_all([feature], compute_frameworks=["PolarsLazyDataFrame"])
print(_result)
from mloda.user import Feature, Options

feature = Feature(
    name="CustomConfiguration",
    options=Options(context={"aggregation_type": "sum", "in_features": Feature("age", options={"n_samples": 5})}),
)
_result = mloda.run_all([feature], compute_frameworks=["PolarsLazyDataFrame"])
print(_result)

[shape: (5, 1)
┌─────────────────────┐
│ CustomConfiguration │
│ ---                 │
│ i64                 │
╞═════════════════════╡
│ 253                 │
│ 253                 │
│ 253                 │
│ 253                 │
│ 253                 │
└─────────────────────┘]

How the chaining essentially works¶

class FeatureGroup(ABC):

    def input_features(self, options: Options, feature_name: FeatureName) -> Optional[Set[Feature]]:

        # In principle, the resolver checks if the feature group depends on another input feature
        # -> then adds it to the chain of features which need to be resolved
        if feature_name contains "input_feature__sum_aggr":
            return input_feature

    # How does mloda knows a feature matches a feature group?
    # Customizable, but some good guesses
    @classmethod
    def match_feature_group_criteria(
        cls,
        feature_name: Union[FeatureName, str],
        options: Options,
        data_access_collection: Optional[DataAccessCollection] = None,
    ) -> bool:

Now we have chaining and matching. Why do we do this?¶

class FeatureGroup(ABC):

    @classmethod
    def calculate_feature(cls, data: Any, features: FeatureSet) -> Any:
        \"\"\"
        This function should be used to calculate the feature.
        \"\"\"

        # data is the incoming data from other feature dependencies or data via mloda

        # features is the configuration

Business knowledge is in the data and in the configuration, but not in the plugin definition.¶

Big idea¶

Separate business logic from transformation logic:

Plugins = generic transformations (shareable across companies)
Data + Config = your business knowledge (stays private)

→ Stop rewriting "sum of a column" at every company

→ Build a shared ecosystem of feature engineering plugins