
# System modules
import os
import sys
from time import time

# Append source directory to system path
src_path = os.path.abspath(os.path.join("../src"))
if src_path not in sys.path:
    sys.path.append(src_path)

# Helper functions
import data.helpers as data_helpers

# numpy and pandas for data manipulation
import numpy as np
import pandas as pd

# matplotlib and seaborn for plotting
import plotly.express as px
import plotly.graph_objects as go

# Necessary to export to HTML
import plotly.io as pio
pio.renderers.default='jupyterlab' # 'notebook' for Jupyter Notebook
import plotly as py
py.offline.init_notebook_mode()


# Accelerate the development cycle
SAMPLE_FRAC: float = 0.5

# Prevent excessive memory usage used by plotly
DRAW_PLOTS: bool = True


# Download and unzip CSV files
!cd .. && make dataset && cd notebooks

>>> Downloading and extracting data files...
Data files already downloaded.
>>> OK.


# Load customers data
customers_df = pd.read_csv(
    "../data/raw/olist_customers_dataset.csv",
    dtype={
        # Nominal qualitative data
        "customer_id": "category",
        "customer_unique_id": "category",
        "customer_city": "category",
        "customer_state": "category",
        "customer_zip_code_prefix": "category",
    },
)

# Load geolocation data
geolocation_df = pd.read_csv(
    "../data/raw/olist_geolocation_dataset.csv",
    dtype={
        # Nominal qualitative data
        "geolocation_zip_code_prefix": "category",
        "geolocation_city": "category",
        "geolocation_state": "category",
        # Continuous quantitative data
        "geolocation_lat": float,
        "geolocation_lng": float,
    },
)

# Load order items data
order_items_df = pd.read_csv(
    "../data/raw/olist_order_items_dataset.csv",
    dtype={
        # Nominal qualitative data
        "order_id": "category",
        "order_item_id": "category",
        "product_id": "category",
        "seller_id": "category",
        # Date data
        "shipping_limit_date": str,
        # Continuous quantitative data
        "price": float,
        "freight_value": float,
    },
    parse_dates=["shipping_limit_date"],
)

# Load order payments
order_payments_df = pd.read_csv(
    "../data/raw/olist_order_payments_dataset.csv",
    dtype={
        # Nominal qualitative data
        "order_id": "category",
        "payment_type": "category",
        # Discrete quantitative data
        "payment_sequential": int,
        "payment_installments": int,
        # Continuous quantitative data
        "payment_value": float,
    },
)

# Load order reviews
order_reviews_df = pd.read_csv(
    "../data/raw/olist_order_reviews_dataset.csv",
    dtype={
        # Nominal qualitative data
        "review_id": "category",
        "order_id": "category",
        # Discrete quantitative data
        "review_score": int,
        # Text data
        "review_comment_title": str,
        "review_comment_message": str,
        # Date data
        "review_creation_date": str,
        "review_answer_timestamp": str,
    },
    parse_dates=["review_creation_date", "review_answer_timestamp"],
)

# Load orders data
orders_df = pd.read_csv(
    "../data/raw/olist_orders_dataset.csv",
    dtype={
        # Nominal qualitative data
        "order_id": "category",
        "customer_id": "category",
        "order_status": "category",
        # Date data
        "order_purchase_timestamp": str,
        "order_approved_at": str,
        "order_delivered_carrier_date": str,
        "order_delivered_customer_date": str,
        "order_estimated_delivery_date": str,
    },
    parse_dates=[
        "order_purchase_timestamp",
        "order_approved_at",
        "order_delivered_carrier_date",
        "order_delivered_customer_date",
        "order_estimated_delivery_date",
    ],
)

# Load products data
products_df = pd.read_csv(
    "../data/raw/olist_products_dataset.csv",
    dtype={
        # Nominal qualitative data
        "product_id": "category",
        "product_category_name": "category",
        # Discrete quantitative data
        # Nullable : https://pandas.pydata.org/pandas-docs/stable/user_guide/gotchas.html#support-for-integer-na
        "product_name_lenght": pd.Int64Dtype(),
        "product_description_lenght": pd.Int64Dtype(),
        "product_photos_qty": pd.Int64Dtype(),
        # Continuous quantitative data
        "product_weight_g": float,
        "product_length_cm": float,
        "product_height_cm": float,
        "product_width_cm": float,
    },
)

# Load sellers data
sellers_df = pd.read_csv(
    "../data/raw/olist_sellers_dataset.csv",
    dtype={
        # Nominal qualitative data
        "seller_id": "category",
        "seller_city": "category",
        "seller_state": "category",
        "seller_zip_code_prefix": "category",
    },
)

# Load category name translation data
category_translation_df = pd.read_csv(
    "../data/raw/product_category_name_translation.csv"
)


# Compute the sum of the payment values for each order
merged_orders_payments_df = (
    orders_df[["order_id", "customer_id", "order_purchase_timestamp"]]
    .merge(
        order_payments_df[["order_id", "payment_value"]],
        how="left",
        left_on="order_id",
        right_on="order_id",
        validate="1:m",
    )
    .groupby("order_id")
    .agg(
        customer_id=("customer_id", "first"),
        order_purchase_timestamp=("order_purchase_timestamp", "first"),
        payment_value=("payment_value", "sum"),  # Total order value
    )
)

# Compute the RFM variables
rfm_df = (
    customers_df[["customer_id", "customer_unique_id"]]
    .merge(
        merged_orders_payments_df,
        how="left",
        left_on="customer_id",
        right_on="customer_id",
        validate="1:1",
    )
    .groupby("customer_unique_id")
    .agg(
        recency=(  # Recency : last purchase date
            "order_purchase_timestamp",
            "max",
        ),
        frequency=(  # Frequency : total number of purchases
            "customer_id",
            "count",
        ),
        monetary=("payment_value", "mean"),  # Monetary : average purchase value
    )
)

# Recency : Transform last purchase date to days since last purchase (from the very last order recorded)
rfm_df["recency"] = (
    (rfm_df["recency"] - rfm_df["recency"].max()) / np.timedelta64(1, "D")
).values

# Reduce memory usage
rfm_df = data_helpers.reduce_dataframe_memory_usage(rfm_df)

rfm_df.describe(include="all", datetime_is_numeric=True)


# Plot the RFM variables scatter matrix to see their correlation
if DRAW_PLOTS:
    fig = px.scatter_3d(
        rfm_df.sample(  # sample of the data for performances reasons
            n=1000,
            random_state=42,
        ),
        x="recency",
        y="frequency",
        z="monetary",
        title="Clients in RFM space",
        opacity=0.5,
        width=1200,
        height=800,
    )
    fig.show()


# Plot the RFM variables histograms and boxes with Plotly
if DRAW_PLOTS:
    for col in rfm_df.columns:
        fig = px.histogram(
            rfm_df[col],
            marginal="box",
            width=800,
        )
        fig.show()


# Plot the RFM variables scatter matrix to see their correlation
if DRAW_PLOTS:
    fig = px.scatter_matrix(
        rfm_df.sample(  # sample of the data for performances reasons
            n=1000,
            random_state=42,
        ),
        width=1200,
        height=800,
    )
    fig.update_traces(
        diagonal_visible=False,
        showupperhalf=False,
    )
    fig.show()


# Plot the Recency x Monetary trendline to see if there is a correlation
if DRAW_PLOTS:
    fig = px.scatter(
        rfm_df.sample(  # sample of the data for performances reasons
            n=1000,
            random_state=42,
        ),
        x="recency",
        y="monetary",
        labels={
            "recency": "Recency : days since last purchase",
            "frequency": "Frequency : total number of purchases",
            "monetary": "Monetary : average purchase value",
        },
        # color="frequency",
        # size="frequency",
        trendline="ols",
        trendline_color_override="red",
        marginal_x="histogram",
        marginal_y="histogram",
        width=1200,
        height=800,
    )
    fig.show()


from sklearn.preprocessing import StandardScaler


def rfm_fit(
    df: pd.DataFrame,
) -> StandardScaler:
    """Fit a Standard Scaler to the RFM variables of the dataframe.

    - Transform Rececy to positive values.
    - Transform RFM values with log+1 function.
    - Fit the Standard Scaler to the transformed RFM variables.

    Args:
        df: untransformed RFM DataFrame to fit our scaler.

    Returns:
        Fitted Standard Scaler.
    """
    df = df.copy()
    df["recency"] = -df["recency"]  # Invert the recency to have positive values
    return StandardScaler().fit(np.log1p(df.astype(float)))


def rfm_transform(
    df: pd.DataFrame,
    standard_scaler: StandardScaler,
) -> pd.DataFrame:
    """Transform the RFM variables of the dataframe with a fitted Standard Scaler.

    - Transform Rececy to positive values.
    - Transform RFM values with log+1 function.
    - Transform the values with the fitted the Standard Scaler.

    Args:
        df: untransformed RFM DataFrame to transform.
        standard_scaler: fitted Standard Scaler

    Returns:
        Transformed RFM DataFrame.
    """
    df = df.copy()
    df["recency"] = -df[
        "recency"
    ]  # Invert the recency to have positive values to apply log+1 function
    return pd.DataFrame(
        standard_scaler.transform(np.log1p(df.astype(float))),
        columns=df.columns,
    )


def rfm_inverse_transform(
    df: pd.DataFrame,
    standard_scaler: StandardScaler,
) -> pd.DataFrame:
    """Revert transformed RFM variables of the dataframe with a fitted Standard Scaler.

    - Inverse-transform the values with the fitted the Standard Scaler.
    - Transform RFM values with exp-1 function.
    - Transform Rececy back to negative values.

    Args:
        df: untransformed RFM DataFrame to transform.
        standard_scaler: fitted Standard Scaler

    Returns:
        Untransformed RFM DataFrame.
    """
    df = np.expm1(
        pd.DataFrame(
            standard_scaler.inverse_transform(df),
            columns=df.columns,
        )
    )
    df["recency"] = -df["recency"]  # Revert the recency back to negative values
    return df


# Fit our Standard Scaler on raw RFM data
standard_scaler = rfm_fit(rfm_df)

# Scale RFM data
scaled_rfm_df = rfm_transform(
    rfm_df,
    standard_scaler=standard_scaler,
)

# Clean RFM data
scaled_rfm_df = (
    scaled_rfm_df[scaled_rfm_df < 20]  # Remove data where z-score > 20
    .dropna()  # Remove empty data
    .drop_duplicates()  # Remove duplaicate data
    .sample(frac=SAMPLE_FRAC, random_state=42)  # Sample for performance reasons
)

scaled_rfm_df.describe(include="all", datetime_is_numeric=True)


raw_rfm_df = rfm_df.copy()

# Un-scale the RFM variables
rfm_df = rfm_inverse_transform(
    scaled_rfm_df,
    standard_scaler=standard_scaler,
)

rfm_df.describe(include="all", datetime_is_numeric=True)


from sklearn.neighbors import NearestNeighbors


# Compute the nearest neighbors of each customer and their distance
if DRAW_PLOTS:
    knn = NearestNeighbors(n_neighbors=2).fit(scaled_rfm_df)
    distances, indices = knn.kneighbors(scaled_rfm_df)
    distances = np.sort(distances[:1000], axis=0)
    distances = distances[:, 1]

    # Plot the nearest neighbors distances
    fig = px.line(
        distances,
        labels={
            "index": "Couple of customers",
            "value": "Euclidian distance",
        },
        title="Customers distances in scaled RFM space",
        width=800,
    )
    fig.show()


from scipy.cluster.hierarchy import dendrogram, linkage


# Plot the hierarchical clustering of the scaled RFM variables
if DRAW_PLOTS:
    dendrogram(
        linkage(scaled_rfm_df.sample(frac=0.5, random_state=42), method="ward"),
        truncate_mode="level",
        p=2,
    )


from sklearn.decomposition import PCA


if DRAW_PLOTS:
    pca = PCA(n_components=2, random_state=42)
    data_pca = pca.fit_transform(scaled_rfm_df)

    # Plot the data in the PCA space
    fig = px.scatter(
        x=data_pca[:1000, 0],
        y=data_pca[:1000, 1],
        trendline="ols",
        title="PCA 2D",
        opacity=0.5,
        width=1200,
        height=800,
        labels={
            "x": f"PCA 1 (Explained Variance ={ round(pca.explained_variance_ratio_[0], 3) })",
            "y": f"PCA 2 (Explained Variance={ round(pca.explained_variance_ratio_[1], 3) })",
        },
    )

    # Plot the feature importances in the PCA space
    loadings = pca.components_.T * np.sqrt(pca.explained_variance_)
    for i, feature in enumerate(rfm_df.columns):
        fig.add_shape(
            type="line",
            x0=0,
            y0=0,
            x1=loadings[i, 0],
            y1=loadings[i, 1],
            line=dict(color="red", width=3),
            name=feature,
        )
        fig.add_annotation(
            x=loadings[i, 0],
            y=loadings[i, 1],
            ax=0,
            ay=0,
            xanchor="center",
            yanchor="bottom",
            text=feature,
            name=feature,
        )

    fig.show()


from sklearn.base import ClusterMixin
from sklearn.metrics import (
    silhouette_score,  # higher is better : https://scikit-learn.org/stable/modules/clustering.html#silhouette-coefficient
    davies_bouldin_score,  # lower is better : https://scikit-learn.org/stable/modules/clustering.html#davies-bouldin-index
    calinski_harabasz_score,  # higher is better : https://scikit-learn.org/stable/modules/clustering.html#calinski-harabasz-index
)


# Let's store the scores of the different clustering algorithms
models_results = pd.DataFrame(
    columns=[
        "model",  # Model name
        "n_clusters",  # Number of clusters found
        "labels",  # List of predicted clusters labels
        "cluster_centers",  # List of cluster centers coordinates
        "inertia",  # List of inertia values of clusters
        "time",  # Time spent for training and prediction
        "silhouette_score",
        "davies_bouldin_score",
        "calinski_harabasz_score",
        "meta_score",  # Meta-score of the model : sum of standardized scores
    ]
)


def process_model(
    model_class: ClusterMixin,
    model_args: dict,
    param_name: str,
    param_range: list,
    fit_df: pd.DataFrame,
    pred_df: pd.DataFrame,
    verbose: bool = False,
) -> dict:
    """Fit and predict a model with a given parameter range.

    - For each value in the hyper-parameter range :
        - Create a clustering model with the given parameters.
        - Fit the model.
        - Predict the cluster labels.
        - Compute and return the results.

    Args:
        model_class: class of the clustering model
        model_args: fixed model parameters
        param_name: name of the hyper-parameter that we want to fine-tune
        param_range: range of values of the hyper-parameter that we want to fine-tune
        fit_df: dataframe on which the model will be fitted
        pred_df: dataframe on which the model will predict the cluster labels
        verbose: if True, print the results of the model

    Returns:
        A dictionary of results for each model.
    """
    results = pd.DataFrame(
        columns=[
            "model",  # Model name
            "n_clusters",  # Number of clusters found
            "labels",  # List of predicted clusters labels
            "cluster_centers",  # List of cluster centers coordinates
            "inertia",  # List of inertia values of clusters
            "time",  # Time spent for training and prediction
            "silhouette_score",
            "davies_bouldin_score",
            "calinski_harabasz_score",
            "meta_score",  # Meta-score of the model : sum of standardized scores
        ]
    )

    # Iterate over the values of the hyper-parameter
    for param_value in param_range:
        # Merge the fixed model parameters with the hyper-parameter value
        model_args[param_name] = param_value
        model = model_class(**model_args)

        if verbose:
            print(f">>> Model : { model }")

        # Fit the model and predict the cluster labels
        if hasattr(model, "fit") and hasattr(model, "predict"):
            # If the model has distict fit and predict methods
            start_time = time()
            model.fit(fit_df)
            fit_time = (time() - start_time) / fit_df.shape[0]

            start_time = time()
            predicted_labels = model.predict(pred_df)
            pred_time = (time() - start_time) / pred_df.shape[0]

            fit_pred_time = fit_time + pred_time
        elif hasattr(model, "fit_predict"):
            # If the model has a single fit_predict method
            start_time = time()
            predicted_labels = model.fit_predict(pred_df)
            fit_pred_time = time() - start_time
        else:
            # If the model has no fit or predict methods
            raise ValueError(f"{model} is not a clustering model.")

        # Number of clusters found
        n_clusters = predicted_labels.max() + 1
        if verbose:
            print(f"Number of clusters : { n_clusters }")

        # If the model found less than 2 or more tan 10 clusters, we skip it
        if not 1 < n_clusters < 11:
            continue

        # Compute the model results
        result = {
            "model": str(model)
            .replace(", random_state=42", "")
            .replace(", n_jobs=-1", "")
            .replace("(random_state=42", "(")
            .replace("(n_jobs=-1", "("),
            "n_clusters": n_clusters,
            "labels": predicted_labels,
            "cluster_centers": model.cluster_centers_
            if hasattr(model, "cluster_centers_")
            else None,
            "inertia": model.inertia_ if hasattr(model, "inertia_") else None,
            "time": fit_pred_time,
            "silhouette_score": silhouette_score(
                pred_df, predicted_labels, random_state=42
            ),
            "davies_bouldin_score": davies_bouldin_score(
                pred_df, predicted_labels
            ),
            "calinski_harabasz_score": calinski_harabasz_score(
                pred_df, predicted_labels
            ),
        }

        if verbose:
            print(f"Score : { round(result['silhouette_score'], 3) }")

        # Append model results to the results dataframe
        results = results.append(result, ignore_index=True)

    # Standardize model scores
    results[
        [
            "standard_silhouette_score",
            "standard_davies_bouldin_score",
            "standard_calinski_harabasz_score",
        ]
    ] = StandardScaler().fit_transform(
        X=results[
            [
                "silhouette_score",
                "davies_bouldin_score",
                "calinski_harabasz_score",
            ]
        ]
    )

    # Compute the meta-score of the model, based on standardized scores
    results["meta_score"] = (
        results["standard_silhouette_score"]
        - results["standard_davies_bouldin_score"]
        + results["standard_calinski_harabasz_score"]
    )

    return results


def plot_scores(results: pd.DataFrame) -> None:
    """Plot the scores of the different clustering model's hyper-parameters.

    Args:
        results (pd.DataFrame): Model results for each hyper-parameter value.
    """
    fig = px.line(
        results,
        x="model",
        y=[
            "standard_silhouette_score",
            "standard_davies_bouldin_score",
            "standard_calinski_harabasz_score",
            "meta_score",
        ],
        title="Clustering model evaluation",
        markers=True,
        width=800,
    )
    fig.show()


def plot_clusters(
    model_name: str, rfm_df: pd.DataFrame, labels: list, cluster_centers=None
) -> None:
    """Plot the RFM data colorized by cluster labels.

    Args:
        model_name (string): Name of the model.
        rfm_df (pd.DataFrame): RFM data points.
        labels (list): List of the labels of each RFM data point.
        cluster_centers (list, optional): Coordinates of each cluster center in the RFM space. Defaults to None.
    """
    fig = px.scatter_3d(
        rfm_df,
        x="recency",
        y="frequency",
        z="monetary",
        title=f"Clustering : { model_name }",
        color=labels,
        opacity=0.5,
        width=1200,
        height=800,
    )

    if cluster_centers is not None:
        fig.add_trace(
            go.Scatter3d(
                x=cluster_centers["recency"],
                y=cluster_centers["frequency"],
                z=cluster_centers["monetary"],
                mode="markers",
                marker_symbol="x",
                hovertemplate="recency: %{x}, frequency: %{y}, monetary: %{z}",
                text="Cluster Center",
                name="Cluster Center",
            )
        )

    fig.show()


def plot_boxes(model_name: str, rfm_df: pd.DataFrame, labels: list) -> None:
    """Plot the RFM variables box and whiskers colorized by cluster labels.

    Args:
        model_name (string): Name of the model.
        rfm_df (pd.DataFrame): RFM data points.
        labels (list): List of the labels of each RFM data point.
    """
    rfm_df = rfm_df.copy()
    rfm_df["labels"] = labels

    fig = px.box(
        rfm_df,
        title=f"{ model_name } - Recency : days since last purchase",
        x="recency",
        color="labels",
        width=800,
    )
    fig.update_traces(boxmean="sd")
    fig.update_traces(notched=True)
    fig.show()

    fig = px.box(
        rfm_df,
        title=f"{ model_name } - Frequency : total number of purchases",
        x="frequency",
        color="labels",
        width=800,
    )
    fig.update_traces(boxmean="sd")
    fig.update_traces(notched=True)
    fig.show()

    fig = px.box(
        rfm_df,
        title=f"{ model_name } - Monetary : average purchase value",
        x="monetary",
        color="labels",
        width=800,
    )
    fig.update_traces(boxmean="sd")
    fig.update_traces(notched=True)
    fig.show()


## KMeans : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html
# Parameters : number of clusters
# Scalability : Very large n_samples, medium n_clusters with MiniBatch code
# Usecase : General-purpose, even cluster size, flat geometry, not too many clusters, inductive
# Geometry (metric used) : Distances between points
from sklearn.cluster import KMeans


results = process_model(
    model_class=KMeans,
    model_args={"random_state": 42},
    param_name="n_clusters",
    param_range=list(range(2, 11)),
    fit_df=scaled_rfm_df,
    pred_df=scaled_rfm_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[
    0
]

models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_df,
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=second_best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(
            second_best_result["cluster_centers"], columns=rfm_df.columns
        ),
        standard_scaler=standard_scaler,
    )
    if second_best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_df,
    labels=second_best_result["labels"],
)


## MiniBatchKMeans : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.MiniBatchKMeans.html
# Parameters : number of clusters
# Scalability : Very large n_samples, medium n_clusters with MiniBatch code
# Usecase : General-purpose, even cluster size, flat geometry, not too many clusters, inductive
# Geometry (metric used) : Distances between points
from sklearn.cluster import MiniBatchKMeans


results = process_model(
    model_class=MiniBatchKMeans,
    model_args={"random_state": 42},
    param_name="n_clusters",
    param_range=list(range(2, 11)),
    fit_df=scaled_rfm_df,
    pred_df=scaled_rfm_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[
    0
]

models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_df,
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=second_best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(
            second_best_result["cluster_centers"], columns=rfm_df.columns
        ),
        standard_scaler=standard_scaler,
    )
    if second_best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_df,
    labels=second_best_result["labels"],
)


## AffinityPropagation : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.AffinityPropagation.html
# Parameters : damping, sample preference
# Scalability : Not scalable with n_samples
# Usecase : Many clusters, uneven cluster size, non-flat geometry, inductive
# Geometry (metric used) : Graph distance (e.g. nearest-neighbor graph)
from sklearn.cluster import AffinityPropagation


results = process_model(
    model_class=AffinityPropagation,
    model_args={"random_state": 42},
    param_name="damping",
    param_range=[round(e, 3) for e in np.linspace(0.5, 0.999, 10)],
    fit_df=scaled_rfm_df.sample(frac=0.2, random_state=42),
    pred_df=scaled_rfm_df,
)
plot_scores(results)

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/cluster/_affinity_propagation.py:250: ConvergenceWarning:

Affinity propagation did not converge, this model will not have any cluster centers.

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/cluster/_affinity_propagation.py:528: ConvergenceWarning:

This model does not have any cluster centers because affinity propagation did not converge. Labeling every sample as '-1'.

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/cluster/_affinity_propagation.py:250: ConvergenceWarning:

Affinity propagation did not converge, this model will not have any cluster centers.

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/cluster/_affinity_propagation.py:528: ConvergenceWarning:

This model does not have any cluster centers because affinity propagation did not converge. Labeling every sample as '-1'.

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/cluster/_affinity_propagation.py:250: ConvergenceWarning:

Affinity propagation did not converge, this model will not have any cluster centers.

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/cluster/_affinity_propagation.py:528: ConvergenceWarning:

This model does not have any cluster centers because affinity propagation did not converge. Labeling every sample as '-1'.

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/cluster/_affinity_propagation.py:250: ConvergenceWarning:

Affinity propagation did not converge, this model will not have any cluster centers.

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/cluster/_affinity_propagation.py:528: ConvergenceWarning:

This model does not have any cluster centers because affinity propagation did not converge. Labeling every sample as '-1'.


best_result = results.sort_values(by="meta_score", ascending=False).iloc[0]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_df,
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=second_best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(
            second_best_result["cluster_centers"], columns=rfm_df.columns
        ),
        standard_scaler=standard_scaler,
    )
    if second_best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_df,
    labels=second_best_result["labels"],
)


## AgglomerativeClustering : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.AgglomerativeClustering.html
# Parameters : number of clusters or distance threshold, linkage type, distance
# Scalability : Large n_samples and n_clusters
# Usecase : Many clusters, possibly connectivity constraints, non Euclidean distances, transductive
# Geometry (metric used) : Any pairwise distance
from sklearn.cluster import AgglomerativeClustering

fit_pred_df = scaled_rfm_df.sample(frac=0.5, random_state=42)
results = process_model(
    model_class=AgglomerativeClustering,
    model_args={},
    param_name="n_clusters",
    param_range=list(range(2, 11)),
    fit_df=fit_pred_df,
    pred_df=fit_pred_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[
    0
]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df.head(1000), standard_scaler=standard_scaler
    ),
    labels=best_result["labels"][:1000],
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df, standard_scaler=standard_scaler
    ),
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="silhouette_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df.head(1000), standard_scaler=standard_scaler
    ),
    labels=second_best_result["labels"][:1000],
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df, standard_scaler=standard_scaler
    ),
    labels=second_best_result["labels"],
)


## MeanShift : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.MeanShift.html
# Parameters : bandwidth
# Scalability : Not scalable with n_samples
# Usecase : Many clusters, uneven cluster size, non-flat geometry, inductive
# Geometry (metric used) : Distances between points
from sklearn.cluster import MeanShift


results = process_model(
    model_class=MeanShift,
    model_args={"n_jobs": -1},
    param_name="bandwidth",
    param_range=[round(e, 1) for e in np.linspace(2, 6, 10)],
    fit_df=scaled_rfm_df.sample(frac=0.1, random_state=42),
    pred_df=scaled_rfm_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[
    0
]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_df,
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=second_best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(second_best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if second_best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_df,
    labels=second_best_result["labels"],
)


## SpectralClustering : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.SpectralClustering.html
# Parameters : number of clusters
# Scalability : Medium n_samples, small n_clusters
# Usecase : Few clusters, even cluster size, non-flat geometry, transductive
# Geometry (metric used) : Graph distance (e.g. nearest-neighbor graph)
from sklearn.cluster import SpectralClustering


fit_pred_df = scaled_rfm_df.sample(frac=0.2, random_state=42)
results = process_model(
    model_class=SpectralClustering,
    model_args={
        "random_state": 42,
        "n_jobs": -1,
    },
    param_name="n_clusters",
    param_range=list(range(2, 11)),
    fit_df=fit_pred_df,
    pred_df=fit_pred_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[0]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df.head(1000), standard_scaler=standard_scaler
    ),
    labels=best_result["labels"][:1000],
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df, standard_scaler=standard_scaler
    ),
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df.head(1000), standard_scaler=standard_scaler
    ),
    labels=second_best_result["labels"][:1000],
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df, standard_scaler=standard_scaler
    ),
    labels=second_best_result["labels"],
)


## DBSCAN : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html
# Parameters : neighborhood size
# Scalability : Very large n_samples, medium n_clusters
# Usecase : Non-flat geometry, uneven cluster sizes, transductive
# Geometry (metric used) : Distances between nearest points
from sklearn.cluster import DBSCAN


fit_pred_df = scaled_rfm_df.sample(frac=0.5, random_state=42)
results = process_model(
    model_class=DBSCAN,
    model_args={
        "n_jobs": -1,
    },
    param_name="eps",
    param_range=[round(e, 2) for e in np.linspace(0.5, 2, 10)],
    fit_df=fit_pred_df,
    pred_df=fit_pred_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[0]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df.head(1000), standard_scaler=standard_scaler
    ),
    labels=best_result["labels"][:1000],
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df, standard_scaler=standard_scaler
    ),
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df.head(1000), standard_scaler=standard_scaler
    ),
    labels=second_best_result["labels"][:1000],
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df, standard_scaler=standard_scaler
    ),
    labels=second_best_result["labels"],
)


## OPTICS : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.OPTICS.html
# Parameters : minimum cluster membership
# Scalability : Very large n_samples, large n_clusters
# Usecase : Non-flat geometry, uneven cluster sizes, variable cluster density, outlier removal, transductive
# Geometry (metric used) : Distances between points
from sklearn.cluster import OPTICS


fit_pred_df = scaled_rfm_df.sample(frac=0.5, random_state=42)
results = process_model(
    model_class=OPTICS,
    model_args={
        "n_jobs": -1,
    },
    param_name="min_samples",
    param_range=[round(e, 4) for e in np.linspace(0.001, 0.01, 10)],
    fit_df=fit_pred_df,
    pred_df=fit_pred_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[
    0
]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df.head(1000), standard_scaler=standard_scaler
    ),
    labels=best_result["labels"][:1000],
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df, standard_scaler=standard_scaler
    ),
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df.head(1000), standard_scaler=standard_scaler
    ),
    labels=second_best_result["labels"][:1000],
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_inverse_transform(
        df=fit_pred_df, standard_scaler=standard_scaler
    ),
    labels=second_best_result["labels"],
)


## Birch : https://scikit-learn.org/stable/modules/generated/sklearn.cluster.Birch.html
# Parameters : branching factor, threshold, optional global clusterer
# Scalability : Large n_clusters and n_samples
# Usecase : Large dataset, outlier removal, data reduction, inductive
# Geometry (metric used) : Euclidean distance between points
from sklearn.cluster import Birch


results = process_model(
    model_class=Birch,
    model_args={},
    param_name="n_clusters",
    param_range=list(range(2, 11)),
    fit_df=scaled_rfm_df.sample(frac=0.2, random_state=42),
    pred_df=scaled_rfm_df,
)
plot_scores(results)

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names

/home/clement/Workspace/oc_p5/env/lib/python3.9/site-packages/sklearn/base.py:441: UserWarning:

X does not have valid feature names, but Birch was fitted with feature names


best_result = results.sort_values(by="meta_score", ascending=False).iloc[
    0
]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_df,
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=second_best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(second_best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if second_best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_df,
    labels=second_best_result["labels"],
)


## GaussianMixture : https://scikit-learn.org/stable/modules/generated/sklearn.mixture.GaussianMixture.html
# Parameters : many
# Scalability : Not scalable
# Usecase : Flat geometry, good for density estimation, inductive
# Geometry (metric used) : Mahalanobis distances to  centers
from sklearn.mixture import GaussianMixture


results = process_model(
    model_class=GaussianMixture,
    model_args={
        "random_state": 42,
    },
    param_name="n_components",
    param_range=list(range(2, 11)),
    fit_df=scaled_rfm_df,
    pred_df=scaled_rfm_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[
    0
]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_df,
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=second_best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(
            second_best_result["cluster_centers"], columns=rfm_df.columns
        ),
        standard_scaler=standard_scaler,
    )
    if second_best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_df,
    labels=second_best_result["labels"],
)


## BayesianGaussianMixture : https://scikit-learn.org/stable/modules/generated/sklearn.mixture.BayesianGaussianMixture.html
# Parameters : many
# Scalability : Not scalable
# Usecase : Flat geometry, good for density estimation, inductive
# Geometry (metric used) : Mahalanobis distances to  centers
from sklearn.mixture import BayesianGaussianMixture


results = process_model(
    model_class=BayesianGaussianMixture,
    model_args={
        "random_state": 42,
    },
    param_name="n_components",
    param_range=list(range(2, 11)),
    fit_df=scaled_rfm_df,
    pred_df=scaled_rfm_df,
)
plot_scores(results)


best_result = results.sort_values(by="meta_score", ascending=False).iloc[
    0
]
models_results = models_results.append(best_result, ignore_index=True)

plot_clusters(
    model_name=best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(best_result["cluster_centers"], columns=rfm_df.columns),
        standard_scaler=standard_scaler,
    )
    if best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=best_result["model"],
    rfm_df=rfm_df,
    labels=best_result["labels"],
)


second_best_result = results.sort_values(
    by="meta_score", ascending=False
).iloc[1]

plot_clusters(
    model_name=second_best_result["model"],
    rfm_df=rfm_df.head(1000),
    labels=second_best_result["labels"][:1000],
    cluster_centers=rfm_inverse_transform(
        df=pd.DataFrame(
            second_best_result["cluster_centers"], columns=rfm_df.columns
        ),
        standard_scaler=standard_scaler,
    )
    if second_best_result["cluster_centers"] is not None
    else None,
)
plot_boxes(
    model_name=second_best_result["model"],
    rfm_df=rfm_df,
    labels=second_best_result["labels"],
)


from sklearn.cluster import KMeans
from sklearn.metrics.cluster import adjusted_rand_score


# Scale raw RFM data (cancel the sampling)
scaled_rfm_df = rfm_transform(
    raw_rfm_df,
    standard_scaler=standard_scaler,
)
# We want to keep the Recency unscaled as the Time variable
scaled_rfm_df["time"] = raw_rfm_df["recency"].values
scaled_rfm_df.describe(include="all", datetime_is_numeric=True)


results = pd.DataFrame(
    columns=[
        "time",
        "monthly_score",
        "quarterly_score",
        "semesterly_score",
        "yearly_score",
    ]
)

# We will observe the performances of each model every week over the timespan of the dataset
for time in range(-720, 1, 7):
    # refecence model : always up-to-date with the latest data
    weekly_kmeans = KMeans(n_clusters=4, random_state=42).fit(
        scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
            ["recency", "frequency", "monetary"]
        ]
    )
    # this model's predictions are considered the ground truth
    labels_true = weekly_kmeans.predict(
        scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
            ["recency", "frequency", "monetary"]
        ]
    )

    if time % 30 < 7:
        # update model every 30 days
        monthly_kmeans = KMeans(n_clusters=4, random_state=42).fit(
            scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
                ["recency", "frequency", "monetary"]
            ]
        )
    monthly_labels_pred = monthly_kmeans.predict(
        scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
            ["recency", "frequency", "monetary"]
        ]
    )

    if time % 90 < 7:
        # update model every 90 days
        quarterly_kmeans = KMeans(n_clusters=4, random_state=42).fit(
            scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
                ["recency", "frequency", "monetary"]
            ]
        )
    quarterly_labels_pred = quarterly_kmeans.predict(
        scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
            ["recency", "frequency", "monetary"]
        ]
    )

    if time % 180 < 7:
        # update model every 180 days
        semesterly_kmeans = KMeans(n_clusters=4, random_state=42).fit(
            scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
                ["recency", "frequency", "monetary"]
            ]
        )
    semesterly_labels_pred = semesterly_kmeans.predict(
        scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
            ["recency", "frequency", "monetary"]
        ]
    )

    if time % 360 < 7:
        # update model every 360 days
        yearly_kmeans = KMeans(n_clusters=4, random_state=42).fit(
            scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
                ["recency", "frequency", "monetary"]
            ]
        )
    yearly_labels_pred = yearly_kmeans.predict(
        scaled_rfm_df.loc[scaled_rfm_df["time"] <= time][
            ["recency", "frequency", "monetary"]
        ]
    )

    # store the results
    result = {
        "time": time,
        "monthly_score": adjusted_rand_score(labels_true, monthly_labels_pred),
        "quarterly_score": adjusted_rand_score(
            labels_true, quarterly_labels_pred
        ),
        "semesterly_score": adjusted_rand_score(
            labels_true, semesterly_labels_pred
        ),
        "yearly_score": adjusted_rand_score(labels_true, yearly_labels_pred),
    }
    results = results.append(result, ignore_index=True)


# Let's plot the performance of each model over the time
fig = px.line(
    results,
    x="time",
    y=[
        "monthly_score",
        "quarterly_score",
        "semesterly_score",
        "yearly_score",
    ],
    title=f"Score evolution per update frequency",
    width=800,
)
fig.show()


# Let'sdesribe the performance of each model
results.describe()

	recency	frequency	monetary
count	96096.000000	96096.000000	96096.000000
mean	-288.201355	1.034809	161.400116
std	153.416321	0.214384	222.307526
min	-772.843750	1.000000	0.000000
25%	-397.351402	1.000000	62.457500
50%	-268.910431	1.000000	105.825001
75%	-163.885746	1.000000	177.210007
max	0.000000	17.000000	13664.080078

	recency	frequency	monetary
count	48046.000000	48046.000000	48046.000000
mean	0.001405	0.000470	0.005966
std	0.998669	0.992188	1.000536
min	-7.864206	-0.173593	-5.883551
25%	-0.632576	-0.173593	-0.690505
50%	0.159996	-0.173593	-0.044974
75%	0.789658	-0.173593	0.596246
max	1.800247	17.540794	6.010884

	recency	frequency	monetary
count	48046.000000	48046.000000	48046.000000
mean	-288.311412	1.034675	162.366538
std	153.220305	0.205463	225.297732
min	-743.782043	1.000000	0.000000
25%	-397.560471	1.000000	62.912499
50%	-268.966934	1.000000	106.160004
75%	-164.331200	1.000000	178.052498
max	-0.884907	7.000000	13664.080078

	time	monthly_score	quarterly_score	semesterly_score	yearly_score
count	103.000000	103.000000	103.000000	103.000000	103.000000
mean	-363.000000	0.952781	0.871807	0.794783	0.792857
std	209.142695	0.113214	0.233071	0.308742	0.309098
min	-720.000000	0.364820	0.204146	0.204146	0.201496
25%	-541.500000	0.967217	0.897053	0.895828	0.909408
50%	-363.000000	0.991010	0.979839	0.954838	0.953029
75%	-184.500000	1.000000	0.992035	0.983036	0.978589
max	-6.000000	1.000000	1.000000	1.000000	1.000000

Fréquence de mise à jour	Performance moyenne cible	Performance minimale garantie	Montant Annuel
Mensuelle	0.95	0.35	12000 € HT
Trimestrielle	0.87	0.20	4000 € HT
Semestrielle	0.79	0.20	2000 € HT
Annuelle	0.79	0.20	1000 € HT

Olist : Construire un modèle de segmentation client¶

Contexte¶

Chargement des modules du projet¶

Chargement des données¶

Construction des variables RFM (Recency, Frequency, Monetary)¶

Distributions des variables¶

Scaling, Cleaning & Sampling¶

EDA & Visualization¶

Distances¶

Dendrogram¶

PCA (Principal Component Analysis)¶

Évaluation des modèles¶

KMeans¶

Meilleur résultat : 4 clusters¶

Second meilleur résultat : 7 clusters¶

Mini-Batch K-Means¶

Meilleur résultat : 4 clusters¶

Second meilleur résultat : 7 clusters¶

Affinity Propagation¶

Meilleur résultat : 4 clusters¶

Second meilleur résultat : 4 clusters¶

Agglomerative Clustering¶

Meilleur résultat : 2 clusters¶

Second meilleur résultat : 4 clusters¶

Mean Shift¶

Meilleur résultat : 2 clusters¶

Second meilleur résultat : 2 clusters¶

Spectral clustering¶

Meilleur résultat : 2 clusters¶

Second meilleur résultat : 3 clusters¶

DBSCAN (Density-Based Spatial Clustering of Applications with Noise)¶

Meilleur résultat : 4 clusters¶

Second meilleur résultat : 4 clusters¶

OPTICS (Ordering Points To Identify the Clustering Structure)¶

Meilleur résultat : 2 clusters¶

Second meilleur résultat : 2 clusters¶

Birch¶

Meilleur résultat : 2 clusters¶

Second meilleur résultat : 3 clusters¶

GMM (Gaussian Mixture Model)¶

Meilleur résultat : 2 clusters¶

Second meilleur résultat : 4 clusters¶

Bayesian Gaussian Mixture¶

Meilleur résultat : 2 clusters¶

Second meilleur résultat : 2 clusters¶

Comparaison¶

Contrat de maintenance¶