Tune Multi-timestamp MLP Downscaler¶

Import Packages¶

In [1]:

Copied!





from pathlib import Path
from typing import Any

import optuna
import pandas as pd
from pingi.notebook import display_caption
from sklearn.neural_network import MLPRegressor

from s3lst_ds.downscaling.tune.tune import tune
from s3lst_ds.downscaling.tune.tune_config import TuneConfig
from pathlib import Path
from typing import Any

import optuna
import pandas as pd
from pingi.notebook import display_caption
from sklearn.neural_network import MLPRegressor

from s3lst_ds.downscaling.tune.tune import tune
from s3lst_ds.downscaling.tune.tune_config import TuneConfig

Tune Downscaler¶

Define hyperparameter suggesting function¶

The function below will be used by optuna to get suggestions for the hyperparameters values during tuning, namely:

the downscaler's numerical predictor scaling method, scale (either "standardize" or "min_max_normalize");
the downscaler's regularization strength of the Lasso predictor selector model, lasso_alpha (between 1e-5 and 1e1);
the initial learning rate of the Multi-Layer Perceptron, learning_rate_init (between 1e-3 and 1e-2).

In [2]:

Copied!





def params_tune_getter(
    trial: optuna.trial.Trial | optuna.trial.FrozenTrial,
) -> dict[str, Any]:
    """
    Get `DownscalerEstimator`'s hyperparameter values suggested by `trial` for the
    optimized tuning method `optuna.study.Study.optimize()`.

    The suggestions may be obtained using `optuna`'s [suggest
    methods](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial):
    - [`suggest_float`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial.suggest_float):
    for continuous hyperparameters;
    - [`suggest_int`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial.suggest_int):
    for integer hyperparameters;
    - [`suggest_categorical`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial.suggest_categorical):
    for categorical hyperparameters.

    WARNING: Note that there is no suggest method for iterables. Each of their
    components must be suggested separately.

    Parameters
    ----------
    trial : optuna.trial.Trial or optuna.trial.FrozenTrial

    Returns
    -------

    params_tune : dict[str, Any]
        Suggested values for `DownscalerEstimator`'s tunable hyperparameters. The keys
        correspond to the hyperparameters' full access paths with each step separated by
        double underscores. (e.g. `"base_model__formula"` in which `"base_model"` is a
        parameter of the downscaler estimator and `"formula"` is a parameter of the
        former)
    """

    params_tune = {
        "scale": trial.suggest_categorical(
            name="scale",
            choices=["standardize", "min_max_normalize"],
        ),
        "lasso_alpha": trial.suggest_float(
            "lasso_alpha",
            1e-5,
            1e1,
            log=True,
        ),
        "base_model__learning_rate_init": trial.suggest_float(
            name="base_model__learning_rate_init",
            low=1e-3,
            high=1e-2,
            log=True,
        ),
    }

    return params_tune
def params_tune_getter(
    trial: optuna.trial.Trial | optuna.trial.FrozenTrial,
) -> dict[str, Any]:
    """
    Get `DownscalerEstimator`'s hyperparameter values suggested by `trial` for the
    optimized tuning method `optuna.study.Study.optimize()`.

    The suggestions may be obtained using `optuna`'s [suggest
    methods](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial):
    - [`suggest_float`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial.suggest_float):
    for continuous hyperparameters;
    - [`suggest_int`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial.suggest_int):
    for integer hyperparameters;
    - [`suggest_categorical`](https://optuna.readthedocs.io/en/stable/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial.suggest_categorical):
    for categorical hyperparameters.

    WARNING: Note that there is no suggest method for iterables. Each of their
    components must be suggested separately.

    Parameters
    ----------
    trial : optuna.trial.Trial or optuna.trial.FrozenTrial

    Returns
    -------

    params_tune : dict[str, Any]
        Suggested values for `DownscalerEstimator`'s tunable hyperparameters. The keys
        correspond to the hyperparameters' full access paths with each step separated by
        double underscores. (e.g. `"base_model__formula"` in which `"base_model"` is a
        parameter of the downscaler estimator and `"formula"` is a parameter of the
        former)
    """

    params_tune = {
        "scale": trial.suggest_categorical(
            name="scale",
            choices=["standardize", "min_max_normalize"],
        ),
        "lasso_alpha": trial.suggest_float(
            "lasso_alpha",
            1e-5,
            1e1,
            log=True,
        ),
        "base_model__learning_rate_init": trial.suggest_float(
            name="base_model__learning_rate_init",
            low=1e-3,
            high=1e-2,
            log=True,
        ),
    }

    return params_tune

Configure¶

With the configuration below, one intends to:

use the Sentine-3 products found at directory ./data;
masking out the data outside of the Lisbon metropolitan area (represented by a WKT string);
using 5 processors in the wrangling of the data (reprojecting, combining, masking and transforming);
performing the optimized cross-validated hyperparameter tuning with 3 folds;
using a random seed number of 42 in the cross-validation splits;
using a multi-timestamp downscaling Multi-Layer Perceptron model;
with "FVC", "NDWI", "season" as predictors;
standardising the numerical predictors ("FVC", "NDWI");
dummy-encoding the categorical predictors ("season");
considering timestamp-specific standardization of the spatio-temporal variables ("FVC", "NDWI" and "LST");
using a Lasso predictor selector model;
using 5 processors in prediction and scoring;
training with all timestamps and predicting (downscaling) for timestamps "20210716T104226" and "20210717T111751";
scoring with metrics $R^2$ (r2), Root Mean Square Error (rmse), Mean Absolute Error (mae) and Mean Bias Error (mbe);
using Root Mean Square Error (rmse) as metric for selecting the best hyperparameters (note that the score computed by the best scorer considers as units the ones resulting from the downscaler_transform);
considering residual correction of the fine predictions;
using function params_tune_getter to obtained suggested hyperparameter values for the tuning;
using a random seed number of 42 in the tuner;
performing 5 hyperparameter tuning trials;
using 10 processors in the tuning;
not returning the data batcher;
logging the whole process into the terminal.

In [3]:

Copied!





tune_config = TuneConfig(
    data_batcher_data_wrangler_path_sentinel3=Path("./data"),
    data_batcher_data_wrangler_aoi=(
        "POLYGON ((-9.53 38.57, -9.05 38.57, -9.05 38.92, -9.53 38.92, -9.53 38.57))"
    ),
    data_batcher_data_wrangler_max_workers=5,
    data_batcher_n_cross_val_folds=3,
    data_batcher_rnd_seed=42,
    downscaler_base_model=MLPRegressor(
        random_state=42,
        batch_size=1024,
        hidden_layer_sizes=(8,),
        activation="relu",
        solver="adam",
        learning_rate_init=1e-2,
        beta_1=0.9,
        beta_2=0.999,
        epsilon=1e-08,
        verbose=False,
        shuffle=True,
        max_iter=1000,
        early_stopping=True,  # NOTE: R2 is used has validation scorer
        validation_fraction=0.2,
        n_iter_no_change=10,
        tol=1e-3,
    ),
    downscaler_X=["FVC", "NDWI", "season"],
    downscaler_masks=["aoi"],
    downscaler_scale="standardize",
    downscaler_encode="dummy",
    downscaler_transform="standardize",
    downscaler_lasso_sel=True,
    downscaler_max_workers=5,
    scorers=["r2", "rmse", "mae", "mbe"],
    best_scorer="rmse",
    correct=True,
    params_tune_getter=params_tune_getter,
    tune_rnd_seed=42,
    tune_n_trials=5,
    tune_n_jobs=10,
    out_data_batcher=False,
    log_mode="console",
)
tune_config = TuneConfig(
    data_batcher_data_wrangler_path_sentinel3=Path("./data"),
    data_batcher_data_wrangler_aoi=(
        "POLYGON ((-9.53 38.57, -9.05 38.57, -9.05 38.92, -9.53 38.92, -9.53 38.57))"
    ),
    data_batcher_data_wrangler_max_workers=5,
    data_batcher_n_cross_val_folds=3,
    data_batcher_rnd_seed=42,
    downscaler_base_model=MLPRegressor(
        random_state=42,
        batch_size=1024,
        hidden_layer_sizes=(8,),
        activation="relu",
        solver="adam",
        learning_rate_init=1e-2,
        beta_1=0.9,
        beta_2=0.999,
        epsilon=1e-08,
        verbose=False,
        shuffle=True,
        max_iter=1000,
        early_stopping=True,  # NOTE: R2 is used has validation scorer
        validation_fraction=0.2,
        n_iter_no_change=10,
        tol=1e-3,
    ),
    downscaler_X=["FVC", "NDWI", "season"],
    downscaler_masks=["aoi"],
    downscaler_scale="standardize",
    downscaler_encode="dummy",
    downscaler_transform="standardize",
    downscaler_lasso_sel=True,
    downscaler_max_workers=5,
    scorers=["r2", "rmse", "mae", "mbe"],
    best_scorer="rmse",
    correct=True,
    params_tune_getter=params_tune_getter,
    tune_rnd_seed=42,
    tune_n_trials=5,
    tune_n_jobs=10,
    out_data_batcher=False,
    log_mode="console",
)

Tune¶

In [4]:

Copied!

tune_out = tune(tune_config)
tune_out = tune(tune_config)

INFO     Tuning multi-timestamp downscaler                                              

INFO     The data will now be wrangled.                                                 
INFO     Getting a SingleDataWrangler instance for each timestamp and performing        
         wrangling of the respective data...                                            
           0% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 0/11  [ 0:00:00 < -:--:-- , ? timestamp/s ]

         100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11/11  [ 0:00:00 < 0:00:00 , 63 timestamp/s ]:--:-- , ? timestamp/s ]
INFO     Data wrangled.                                                                 

INFO     The data will now be batched.                                                  
⠋        Batching the data...
INFO     Data batched.                                                                  

INFO     The downscaler will now be tuned.                                              
INFO     A new study created in memory with name: hparam_tuning                         
INFO     Performing tuning trials...                                                    
INFO     Trial 0 finished with value: 0.8965808504038417 and parameters: {'scale':      rial/s ]
         'min_max_normalize', 'lasso_alpha': 0.24658329458549094,                       
         'base_model__learning_rate_init': 0.003968793330444372}. Best is trial 0 with  
         value: 0.8965808504038417.                                                     
INFO     Trial 1 finished with value: 0.7514857925326367 and parameters: {'scale':      :01 < -:--:-- , ? trial/s ]
         'standardize', 'lasso_alpha': 2.231010801867923e-05,                           
         'base_model__learning_rate_init': 0.007348118405270454}. Best is trial 1 with  
         value: 0.7514857925326367.                                                     
INFO     Trial 2 finished with value: 0.8563495141236307 and parameters: {'scale':      :01 < 0:00:02 , 2 trial/s ]
         'min_max_normalize', 'lasso_alpha': 1.3289448722869181e-05,                    
         'base_model__learning_rate_init': 0.009330606024425666}. Best is trial 1 with  
         value: 0.7514857925326367.                                                     
INFO     Trial 3 finished with value: 0.7953167631214547 and parameters: {'scale':      :02 < 0:00:02 , 2 trial/s ]
         'standardize', 'lasso_alpha': 0.00012329623163659834,                          
         'base_model__learning_rate_init': 0.0015254729458052604}. Best is trial 1 with 
         value: 0.7514857925326367.                                                     
INFO     Trial 4 finished with value: 0.8993432256002651 and parameters: {'scale':      :03 < 0:00:01 , 1 trial/s ]
         'min_max_normalize', 'lasso_alpha': 0.0039054412752107894,                     
         'base_model__learning_rate_init': 0.001955370866274525}. Best is trial 1 with  
         value: 0.7514857925326367.                                                     
         100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 5/5  [ 0:00:03 < 0:00:00 , 1 trial/s ][0m , 1 trial/s ]
INFO     Downscaler tuned having as best hyperparameters                                
         {'scale': 'standardize', 'lasso_alpha': 2.231010801867923e-05,                 
         'base_model__learning_rate_init': 0.007348118405270454}                        
         which were found at trial 1 with cross-validation rmse of 0.751486.            

INFO     The downscaler will now be trained.                                            
⠙        Training the downscaler...0m
INFO     Downscaler trained.                                                            

INFO     The downscaler will now be scored with respect to the coarse training data     
         using Sentinel-3 as ground truth.                                              
INFO     Predicting raw target and scoring...                                           
         100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11/11  [ 0:00:00 < 0:00:00 , ? timestamp/s ]
INFO     Downscaler scored with respect to coarse training data using Sentinel-3 as     
         ground truth.                                                                  

INFO     The downscaler attained the following best hyperparameters:                    
                                                                                        
                       Best hyperparameters                                             
         ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓                               
         ┃ Hyperparameter                 ┃       Value ┃                               
         ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩                               
         │ scale                          │ standardize │                               
         │ lasso_alpha                    │   2.231e-05 │                               
         │ base_model__learning_rate_init │   0.0073481 │                               
         └────────────────────────────────┴─────────────┘                               
                                                                                        

INFO     The downscaler attained the following metrics:                                 
                                                                                        
                                  Metrics                                               
         ┏━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━┓                     
         ┃ Batch     ┃ Grid   ┃ Ground truth ┃ Metric ┃     Value ┃                     
         ┡━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━┩                     
         │ cross_val │ coarse │ sentinel     │ rmse   │   0.75149 │                     
         │ train     │ coarse │ sentinel     │ r2     │   0.66615 │                     
         │           │        │              │ rmse   │    3.4266 │                     
         │           │        │              │ mae    │    2.5646 │                     
         │           │        │              │ mbe    │ -0.090619 │                     
         └───────────┴────────┴──────────────┴────────┴───────────┘

Parse Results¶

In [5]:

Copied!





# Tuned downscaler
downscaler = tune_out["downscaler"]  # type: ignore

# Tuned hyperparameters
params = tune_out["params"]  # type: ignore
# Scores
score = tune_out["score"]  # type: ignore
# Tuned downscaler
downscaler = tune_out["downscaler"]  # type: ignore

# Tuned hyperparameters
params = tune_out["params"]  # type: ignore
# Scores
score = tune_out["score"]  # type: ignore

Show scores and tuned hyperparameters¶

In [6]:

Copied!





display_caption("Scores")
score = (
    pd.Series(
        {
            (batch, grid, ground_truth, scorer): score[batch][grid][ground_truth][  # type: ignore
                scorer
            ]  # type: ignore
            for batch in score  # type: ignore
            for grid in score[batch]  # type: ignore
            for ground_truth in score[batch][grid]  # type: ignore
            for scorer in score[batch][grid][ground_truth]  # type: ignore
        }
    )
    .to_frame()
    .rename(
        columns={
            0: "Score",
        }
    )
)

score.index.names = ["Batch", "Grid", "Ground Truth", "Scorer"]
display(score)

display_caption("Tuned Hyperparameters")
pd.Series(params).to_frame().reset_index().rename(
    columns={
        "index": "Name",
        0: "Value",
    }
)
display_caption("Scores")
score = (
    pd.Series(
        {
            (batch, grid, ground_truth, scorer): score[batch][grid][ground_truth][  # type: ignore
                scorer
            ]  # type: ignore
            for batch in score  # type: ignore
            for grid in score[batch]  # type: ignore
            for ground_truth in score[batch][grid]  # type: ignore
            for scorer in score[batch][grid][ground_truth]  # type: ignore
        }
    )
    .to_frame()
    .rename(
        columns={
            0: "Score",
        }
    )
)

score.index.names = ["Batch", "Grid", "Ground Truth", "Scorer"]
display(score)

display_caption("Tuned Hyperparameters")
pd.Series(params).to_frame().reset_index().rename(
    columns={
        "index": "Name",
        0: "Value",
    }
)

Scores

				Score
Batch	Grid	Ground Truth	Scorer
cross_val	coarse	sentinel	rmse	0.751486
train	coarse	sentinel	r2	0.666152
			rmse	3.426589
			mae	2.564593
			mbe	-0.090619

Tuned Hyperparameters

Out[6]:

	Name	Value
0	scale	standardize
1	lasso_alpha	0.000022
2	base_model__learning_rate_init	0.007348