A plethora of dimensionality reduction techniques have emerged over the past decades, leaving researchers and analysts with a wide variety of choices for reducing their data, all the more so given some techniques come with additional hyper-parametrization (e.g., t-SNE, UMAP, etc.). Recent studies are showing that people often use dimensionality reduction as a black-box regardless of the specific properties the method itself preserves. Hence, evaluating and comparing 2D embeddings is usually qualitatively decided, by setting embeddings side-by-side and letting human judgment decide which embedding is the best. In this work, we propose a quantitative way of evaluating embeddings, that nonetheless places human perception at the center. We run a comparative study, where we ask people to select “good” and “misleading” views between scatterplots of low-dimensional embeddings of image datasets, simulating the way people usually select embeddings. We use the study data as labels for a set of quality metrics for a supervised machine learning model whose purpose is to discover and quantify what exactly people are looking for when deciding between embeddings. With the model as a proxy for human judgments, we use it to rank embeddings on new datasets, explain why they are relevant, and quantify the degree of subjectivity when people select preferred embeddings.
%0 Journal Article
%1 morariu2023dumbledr
%A Morariu, Cristina
%A Bibal, Adrien
%A Cutura, Rene
%A Frénay, Beno\^ıt
%A Sedlmair, Michael
%D 2023
%J IEEE Transactions on Visualization and Computer Graphics
%K visus:cuturare sfbtrr161 from:christinawarren visus:sedlmaml 2023 a08 visus
%N 1
%P 745-755
%R 10.1109/TVCG.2022.3209449
%T Predicting User Preferences of Dimensionality Reduction Embedding Quality
%U https://ieeexplore.ieee.org/document/9904619
%V 29
%X A plethora of dimensionality reduction techniques have emerged over the past decades, leaving researchers and analysts with a wide variety of choices for reducing their data, all the more so given some techniques come with additional hyper-parametrization (e.g., t-SNE, UMAP, etc.). Recent studies are showing that people often use dimensionality reduction as a black-box regardless of the specific properties the method itself preserves. Hence, evaluating and comparing 2D embeddings is usually qualitatively decided, by setting embeddings side-by-side and letting human judgment decide which embedding is the best. In this work, we propose a quantitative way of evaluating embeddings, that nonetheless places human perception at the center. We run a comparative study, where we ask people to select “good” and “misleading” views between scatterplots of low-dimensional embeddings of image datasets, simulating the way people usually select embeddings. We use the study data as labels for a set of quality metrics for a supervised machine learning model whose purpose is to discover and quantify what exactly people are looking for when deciding between embeddings. With the model as a proxy for human judgments, we use it to rank embeddings on new datasets, explain why they are relevant, and quantify the degree of subjectivity when people select preferred embeddings.
@article{morariu2023dumbledr,
abstract = {A plethora of dimensionality reduction techniques have emerged over the past decades, leaving researchers and analysts with a wide variety of choices for reducing their data, all the more so given some techniques come with additional hyper-parametrization (e.g., t-SNE, UMAP, etc.). Recent studies are showing that people often use dimensionality reduction as a black-box regardless of the specific properties the method itself preserves. Hence, evaluating and comparing 2D embeddings is usually qualitatively decided, by setting embeddings side-by-side and letting human judgment decide which embedding is the best. In this work, we propose a quantitative way of evaluating embeddings, that nonetheless places human perception at the center. We run a comparative study, where we ask people to select “good” and “misleading” views between scatterplots of low-dimensional embeddings of image datasets, simulating the way people usually select embeddings. We use the study data as labels for a set of quality metrics for a supervised machine learning model whose purpose is to discover and quantify what exactly people are looking for when deciding between embeddings. With the model as a proxy for human judgments, we use it to rank embeddings on new datasets, explain why they are relevant, and quantify the degree of subjectivity when people select preferred embeddings.},
added-at = {2023-07-12T07:34:19.000+0200},
author = {Morariu, Cristina and Bibal, Adrien and Cutura, Rene and Fr{\'e}nay, Beno{\^\i}t and Sedlmair, Michael},
biburl = {https://puma.ub.uni-stuttgart.de/bibtex/226429a9d050847f9a54987d98dd28675/visus},
doi = {10.1109/TVCG.2022.3209449},
interhash = {191cd2881d3ed5c526971c8f37ff8a03},
intrahash = {26429a9d050847f9a54987d98dd28675},
journal = {IEEE Transactions on Visualization and Computer Graphics},
keywords = {visus:cuturare sfbtrr161 from:christinawarren visus:sedlmaml 2023 a08 visus},
number = 1,
pages = {745-755},
timestamp = {2023-07-12T07:34:19.000+0200},
title = {Predicting User Preferences of Dimensionality Reduction Embedding Quality},
url = {https://ieeexplore.ieee.org/document/9904619},
volume = 29,
year = 2023
}