Skip to content

Commit

Permalink
Categorical Exif Metadata support for Vision Dashboard (#2348)
Browse files Browse the repository at this point in the history 
* rai vision insights metadata ckpt

* deepcopy support

* dashboard rendering ckpt

* Flyout Info fix

* mock data update with cat feats

* backend ckpt

* python lint fixes

* isort fix

* MO test updates

* OD MO test fix

* auto lint fixes
  • Loading branch information
@Advitya17
Advitya17 committed Sep 25, 2023
1 parent 5833a88 commit 057cbac
Show file tree
Hide file tree
Showing 14 changed files with 326 additions and 212 deletions.
275 changes: 139 additions & 136 deletions apps/dashboard/src/model-assessment-vision/__mock_data__/fridge.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,144 +6,147 @@ import { DatasetTaskType, IDataset } from "@responsible-ai/core-ui";
import { fridgeImages } from "./fridgeImages";

export const fridge: IDataset = {
categorical_features: [],
categorical_features: ["Make", "Software", "Model"],
class_names: ["can", "carton", "milk_bottle", "water_bottle"],
feature_names: ["mean_pixel_value"],
feature_metadata: {
categorical_features: ["Make", "Software", "Model"]
},
feature_names: ["mean_pixel_value", "Make", "Software", "Model"],
features: [
[95.31284825289677],
[84.69935660795275],
[96.82651920382118],
[139.81288994028156],
[118.74137849634106],
[96.03234639705477],
[116.1944440258964],
[156.94601031637208],
[138.63600484176374],
[116.77090019638274],
[104.20986333569394],
[116.08712864325142],
[97.7599067809805],
[102.52772110637315],
[79.23872406467069],
[78.73751914857283],
[75.8992330525713],
[107.98595587164303],
[115.18021104866375],
[100.59847414126499],
[82.81142636149492],
[88.68177055866119],
[100.34771096076865],
[102.71283720322852],
[116.43946539695933],
[96.60582819774888],
[95.60207532860207],
[111.82308175247739],
[94.76322168145815],
[89.44031622141526],
[74.21323247999759],
[104.50651779749977],
[105.64691990497285],
[110.85784702236553],
[100.75961748057519],
[113.76419204323435],
[109.15102485673101],
[114.39268110155147],
[108.32878689710247],
[106.11869654095158],
[81.28403071807776],
[97.64177276859483],
[91.13476074956931],
[70.44286869478304],
[81.33078990060321],
[77.63631423247497],
[75.88688354151913],
[108.65089276296955],
[130.8879654044932],
[119.59721447908348],
[152.05574356733518],
[109.87165709865847],
[110.36740850121295],
[82.79173083008122],
[94.12558349782272],
[110.79903064273911],
[124.7931281108583],
[114.98461584823782],
[104.79512408275197],
[93.91704612231983],
[95.86216844717003],
[99.59804923129467],
[89.33219722653327],
[75.59860171470761],
[104.68140993766983],
[105.644306156172],
[116.64628605582091],
[105.68025775862502],
[111.10920085785607],
[121.85357080074937],
[102.63033234388577],
[85.63116408255107],
[83.89496285804692],
[124.50242339315221],
[95.28945208713165],
[97.20876439595983],
[76.99963134288627],
[80.8539143449807],
[89.48208396743361],
[93.05765114188276],
[82.82937102274725],
[102.87860934902386],
[137.36997905585608],
[139.65304142118825],
[116.45327145519109],
[24.462534090737865],
[82.24885962400992],
[101.50880608334464],
[116.19573683708269],
[109.43424392644938],
[101.70703512287733],
[116.01029326824074],
[88.71978644004801],
[84.29723908970824],
[113.93166181988036],
[109.22723843677329],
[100.98208949226775],
[84.5715561448712],
[82.80138924465473],
[107.8279084484759],
[97.26840849619037],
[111.17454558239285],
[97.09293172811789],
[119.7738092726807],
[107.37994485211024],
[98.976736430254],
[136.69238728082732],
[100.68018744255428],
[95.20174986564608],
[108.4939201711694],
[113.95875962451218],
[108.08554452262922],
[100.69176448134364],
[113.45939698946766],
[109.1880322047604],
[92.21307678012667],
[91.18546667269376],
[109.26651699388752],
[76.73858092124098],
[71.06574753517059],
[73.74995102987931],
[81.30849568811809],
[115.89991110039628],
[105.08086348134866],
[118.20760819885385],
[88.28068367311063],
[88.5644421675431],
[124.86965208612715],
[96.44391037714527],
[97.88054103193856],
[85.75104394252163],
[133.50404367632507],
[130.90057006243063],
[105.48499003018568]
[95.31284825289677, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[84.69935660795275, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[96.82651920382118, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[139.81288994028156, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[118.74137849634106, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[96.03234639705477, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[116.1944440258964, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[156.94601031637208, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[138.63600484176374, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[116.77090019638274, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[104.20986333569394, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[116.08712864325142, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[97.7599067809805, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[102.52772110637315, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[79.23872406467069, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[78.73751914857283, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[75.8992330525713, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[107.98595587164303, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[115.18021104866375, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[100.59847414126499, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[82.81142636149492, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[88.68177055866119, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[100.34771096076865, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[102.71283720322852, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[116.43946539695933, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[96.60582819774888, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[95.60207532860207, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[111.82308175247739, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[94.76322168145815, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[89.44031622141526, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[74.21323247999759, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[104.50651779749977, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[105.64691990497285, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[110.85784702236553, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[100.75961748057519, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[113.76419204323435, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[109.15102485673101, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[114.39268110155147, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[108.32878689710247, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[106.11869654095158, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[81.28403071807776, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[97.64177276859483, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[91.13476074956931, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[70.44286869478304, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[81.33078990060321, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[77.63631423247497, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[75.88688354151913, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[108.65089276296955, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[130.8879654044932, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[119.59721447908348, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[152.05574356733518, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[109.87165709865847, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[110.36740850121295, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[82.79173083008122, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[94.12558349782272, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[110.79903064273911, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[124.7931281108583, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[114.98461584823782, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[104.79512408275197, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[93.91704612231983, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[95.86216844717003, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[99.59804923129467, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[89.33219722653327, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[75.59860171470761, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[104.68140993766983, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[105.644306156172, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[116.64628605582091, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[105.68025775862502, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[111.10920085785607, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[121.85357080074937, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[102.63033234388577, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[85.63116408255107, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[83.89496285804692, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[124.50242339315221, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[95.28945208713165, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[97.20876439595983, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[76.99963134288627, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[80.8539143449807, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[89.48208396743361, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[93.05765114188276, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[82.82937102274725, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[102.87860934902386, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[137.36997905585608, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[139.65304142118825, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[116.45327145519109, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[24.462534090737865, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[82.24885962400992, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[101.50880608334464, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[116.19573683708269, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[109.43424392644938, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[101.70703512287733, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[116.01029326824074, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[88.71978644004801, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[84.29723908970824, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[113.93166181988036, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[109.22723843677329, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[100.98208949226775, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[84.5715561448712, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[82.80138924465473, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[107.8279084484759, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[97.26840849619037, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[111.17454558239285, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[97.09293172811789, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[119.7738092726807, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[107.37994485211024, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[98.976736430254, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[136.69238728082732, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[100.68018744255428, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[95.20174986564608, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[108.4939201711694, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[113.95875962451218, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[108.08554452262922, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[100.69176448134364, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[113.45939698946766, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[109.1880322047604, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[92.21307678012667, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[91.18546667269376, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[109.26651699388752, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[76.73858092124098, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[71.06574753517059, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[73.74995102987931, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[81.30849568811809, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[115.89991110039628, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[105.08086348134866, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[118.20760819885385, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[88.28068367311063, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[88.5644421675431, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[124.86965208612715, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[96.44391037714527, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[97.88054103193856, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[85.75104394252163, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[133.50404367632507, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[130.90057006243063, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[105.48499003018568, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"]
],
images: fridgeImages,
predicted_y: [
Expand Down
17 changes: 10 additions & 7 deletions apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeBinary.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,18 @@ import { fridge } from "./fridge";
import { fridgeImages } from "./fridgeImages";

export const fridgeBinary: IDataset = {
categorical_features: [],
categorical_features: ["Make", "Software", "Model"],
class_names: ["can", "carton"],
feature_names: ["mean_pixel_value"],
feature_metadata: {
categorical_features: ["Make", "Software", "Model"]
},
feature_names: ["mean_pixel_value", "Make", "Software", "Model"],
features: [
fridge.features[0],
fridge.features[1],
fridge.features[2],
fridge.features[3],
fridge.features[34]
[fridge.features[0][0], "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[fridge.features[1][0], "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[fridge.features[2][0], "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[fridge.features[3][0], "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[fridge.features[34][0], "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"]
],
images: [
fridgeImages[0],
Expand Down
27 changes: 15 additions & 12 deletions apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeMultilabel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,23 @@ import { DatasetTaskType, IDataset } from "@responsible-ai/core-ui";
import { fridgeMultilabelImages } from "./fridgeMultilabelImages";

export const fridgeMultilabel: IDataset = {
categorical_features: [],
categorical_features: ["Make", "Software", "Model"],
class_names: ["can", "carton", "milk_bottle", "water_bottle"],
feature_names: ["mean_pixel_value"],
feature_metadata: {
categorical_features: ["Make", "Software", "Model"]
},
feature_names: ["mean_pixel_value", "Make", "Software", "Model"],
features: [
[100.65330159819138],
[103.05057662873294],
[98.1199917352222],
[104.26360328264136],
[102.08869290131815],
[99.20243590283671],
[102.64630060922646],
[97.02252452853655],
[96.48332701438913],
[96.88715067773184]
[100.65330159819138, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[103.05057662873294, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[98.1199917352222, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[104.26360328264136, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[102.08869290131815, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[99.20243590283671, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[102.64630060922646, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[97.02252452853655, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[96.48332701438913, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[96.88715067773184, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"]
],
images: fridgeMultilabelImages,
predicted_y: [
Expand Down
17 changes: 10 additions & 7 deletions apps/dashboard/src/model-assessment-vision/__mock_data__/fridgeObjectDetection.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,18 @@ import { DatasetTaskType, IDataset } from "@responsible-ai/core-ui";
import { fridgeObjectDetectionImages } from "./fridgeObjectDetectionImages";

export const fridgeObjectDetection: IDataset = {
categorical_features: [],
categorical_features: ["Make", "Software", "Model"],
class_names: ["can", "carton", "milk_bottle", "water_bottle"],
feature_names: ["mean_pixel_value"],
feature_metadata: {
categorical_features: ["Make", "Software", "Model"]
},
feature_names: ["mean_pixel_value", "Make", "Software", "Model"],
features: [
[96.30899737412763],
[95.32630225415797],
[100.3762680516188],
[92.000130390912],
[95.33849179841164]
[96.30899737412763, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[95.32630225415797, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[100.3762680516188, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[92.000130390912, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"],
[95.33849179841164, "Google", "HDR+ 1.0.220943774z", "Pixel 2 XL"]
],
imageDimensions: [
[499, 666],
Expand Down
2 changes: 2 additions & 0 deletions ...e2e/src/lib/describer/modelAssessment/datasets/FridgeImageClassificationModelDebugging.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ export const FridgeImageClassificationModelDebugging = {
modelOverviewData: {
featureCohortView: {
firstFeatureToSelect: "mean_pixel_value",
multiFeatureCohorts: 3,
secondFeatureToSelect: "Make",
singleFeatureCohorts: 3
},
hasModelOverviewComponent: true,
Expand Down
2 changes: 2 additions & 0 deletions libs/e2e/src/lib/describer/modelAssessment/datasets/FridgeMultilabelModelDebugging.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ export const FridgeMultilabelModelDebugging = {
modelOverviewData: {
featureCohortView: {
firstFeatureToSelect: "mean_pixel_value",
multiFeatureCohorts: 3,
secondFeatureToSelect: "Make",
singleFeatureCohorts: 3
},
hasModelOverviewComponent: true,
Expand Down
Loading

0 comments on commit 057cbac

Please sign in to comment.