Nested Exif Metadata support + Gate fixes for py 3.7 and keras version (

#2545) * test ckpt * gate updates * removed 3.7 * fridge test fixes * test fixes * other gate test fixes * added keras suffix * test fixes * tf keras save fix * pegged tf to avoid keras 3.0 install * tf keras update * removed .keras from path * keras version update * version updates * version updates * version updates * keras install gate * tf-keras install update * test updates * tf-keras support * removed py 3.7 * lint fixes * keras gate for nbk tests * removed py 3.7 from nbk tests * removed commented code
microsoft · Apr 2, 2024 · 76e0de4 · 76e0de4
1 parent e2c7dd0
commit 76e0de4
Show file tree

Hide file tree

Showing 10 changed files with 125 additions and 49 deletions.
diff --git a/.github/workflows/CI-e2e-notebooks-text-vision.yml b/.github/workflows/CI-e2e-notebooks-text-vision.yml
@@ -56,7 +56,7 @@ jobs:
       matrix:
         # TODO: add macos
         operatingSystem: [ubuntu-latest, windows-latest]
-        pythonVersion: [3.7, 3.8, 3.9, "3.10"]
+        pythonVersion: [3.8, 3.9, "3.10"]
         flights: [""]
         notebookGroup: ["vis_nb_group_1", "text_nb_group_1"]
 
@@ -99,6 +99,12 @@ jobs:
           pip install --upgrade setuptools
           pip install --upgrade "pip-tools<=7.1.0"
 
+      - name: Install backwards-compatible keras for transformers
+        shell: bash -l {0}
+        run: |
+          pip install tf-keras
+          pip install keras==2.15
+
       - name: Install dependencies
         shell: bash -l {0}
         run: |

diff --git a/.github/workflows/CI-notebook-text.yml b/.github/workflows/CI-notebook-text.yml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       matrix:
         operatingSystem: [ubuntu-latest, windows-latest]
-        pythonVersion: [3.7, 3.8, 3.9, "3.10"]
+        pythonVersion: [3.8, 3.9, "3.10"]
 
     runs-on: ${{ matrix.operatingSystem }}
 
@@ -49,18 +49,12 @@ jobs:
         run: yarn buildall
 
       - if: ${{ matrix.operatingSystem != 'macos-latest' }}
-        name: Install pytorch on non-MacOS with python 3.7
+        name: Install pytorch on non-MacOS
         shell: bash -l {0}
         run: |
           conda install --yes --quiet "pytorch==1.13.1" "torchvision<0.15" captum cpuonly "numpy<1.24.0" -c pytorch
 
-      - if: ${{ matrix.operatingSystem == 'macos-latest' && matrix.pythonVersion == '3.7' }}
-        name: Install Anaconda packages on MacOS with python 3.7
-        shell: bash -l {0}
-        run: |
-          conda install --yes --quiet "pytorch==1.13.1" "torchvision<0.15" captum "numpy<1.24.0" -c pytorch
-
-      - if: ${{ matrix.operatingSystem == 'macos-latest' && matrix.pythonVersion != '3.7' }}
+      - if: ${{ matrix.operatingSystem == 'macos-latest' }}
         name: Install Anaconda packages on MacOS, which should not include cpuonly according to official docs
         shell: bash -l {0}
         run: |
@@ -73,6 +67,12 @@ jobs:
           pip install --upgrade setuptools
           pip install --upgrade "pip-tools<=7.1.0"
 
+      - name: Install backwards-compatible keras for transformers
+        shell: bash -l {0}
+        run: |
+          pip install tf-keras
+          pip install keras==2.15
+
       - name: Install dependencies
         shell: bash -l {0}
         run: |

diff --git a/.github/workflows/CI-notebook-vision.yml b/.github/workflows/CI-notebook-vision.yml
@@ -20,7 +20,7 @@ jobs:
     strategy:
       matrix:
         operatingSystem: [ubuntu-latest]
-        pythonVersion: [3.7, 3.8, 3.9, "3.10"]
+        pythonVersion: [3.8, 3.9, "3.10"]
 
     runs-on: ${{ matrix.operatingSystem }}
 
@@ -48,25 +48,13 @@ jobs:
       - name: Build Typescript
         run: yarn buildall
 
-      - if: ${{ matrix.operatingSystem != 'macos-latest' && matrix.pythonVersion == '3.7' }}
-        name: Install pytorch on non-MacOS with python 3.7
-        shell: bash -l {0}
-        run: |
-          conda install --yes --quiet "pytorch==1.13.1" "torchvision<0.15" cpuonly "numpy<1.24.0" -c pytorch
-
-      - if: ${{ matrix.operatingSystem == 'macos-latest' && matrix.pythonVersion == '3.7' }}
-        name: Install Anaconda packages on MacOS with python 3.7
-        shell: bash -l {0}
-        run: |
-          conda install --yes --quiet "pytorch==1.13.1" "torchvision<0.15" "numpy<1.24.0" -c pytorch
-
-      - if: ${{ matrix.operatingSystem != 'macos-latest' && matrix.pythonVersion != '3.7' }}
+      - if: ${{ matrix.operatingSystem != 'macos-latest' }}
         name: Install pytorch on non-MacOS
         shell: bash -l {0}
         run: |
           conda install --yes --quiet "pytorch<2.1,>1.13.1" "torchvision<0.16" cpuonly "numpy<1.24.0" -c pytorch
 
-      - if: ${{ matrix.operatingSystem == 'macos-latest' && matrix.pythonVersion != '3.7' }}
+      - if: ${{ matrix.operatingSystem == 'macos-latest' }}
         name: Install Anaconda packages on MacOS, which should not include cpuonly according to official docs
         shell: bash -l {0}
         run: |

diff --git a/.github/workflows/CI-responsibleai-text-vision-pytest.yml b/.github/workflows/CI-responsibleai-text-vision-pytest.yml
@@ -72,6 +72,12 @@ jobs:
         run: |
           conda install --yes --quiet matplotlib -c conda-forge
 
+      - name: Install backwards-compatible keras for transformers
+        shell: bash -l {0}
+        run: |
+          pip install tf-keras
+          pip install keras==2.15
+
       - name: Install dependencies
         shell: bash -l {0}
         run: |

diff --git a/libs/e2e/src/lib/describer/modelAssessment/datasets/FridgeObjectDetectionModelDebugging.ts b/libs/e2e/src/lib/describer/modelAssessment/datasets/FridgeObjectDetectionModelDebugging.ts
@@ -3,7 +3,7 @@
 
 import { getOS } from "../../../../util/getOS";
 
-const FeatureCohorts = getOS() === "Linux" ? [3, 6] : 6;
+const FeatureCohorts = getOS() === "Linux" ? [2, 6] : 6;
 
 export const FridgeObjectDetectionModelDebugging = {
   causalAnalysisData: {

diff --git a/...modelOverview/ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent.ts b/...modelOverview/ensureAllModelOverviewFeatureCohortsViewElementsAfterSelectionArePresent.ts
@@ -63,7 +63,7 @@ function assertNumberOfChartRowsEqual(
       .should("be.gte", expectedNumberOfCohorts[0])
       .and("be.lte", expectedNumberOfCohorts[1]);
   } else if (isVision) {
-    cy.get(getChartItems(chartIdentifier)).its("length").should("be.gt", 2);
+    cy.get(getChartItems(chartIdentifier)).its("length").should("be.gte", 2);
   } else {
     cy.get(getChartItems(chartIdentifier)).should(
       "have.length",

diff --git a/responsibleai_vision/requirements-dev.txt b/responsibleai_vision/requirements-dev.txt
@@ -21,9 +21,9 @@ sphinx==3.1.1
 sphinx-gallery==0.8.1
 pydata-sphinx-theme==0.3.0
 
+tensorflow<=2.15
 transformers
 datasets
-tensorflow
 opencv-python
 
 fastai

diff --git a/responsibleai_vision/tests/common_vision_utils.py b/responsibleai_vision/tests/common_vision_utils.py
@@ -318,6 +318,46 @@ def load_fridge_object_detection_dataset(automl_format=False):
     return data
 
 
+def load_clearsight_object_detection_dataset(automl_format=False):
+    # create data folder if it doesnt exist.
+    os.makedirs("data", exist_ok=True)
+
+    # download data
+    download_url = ("https://publictestdatasets.blob.core.windows.net/" +
+                    "computervision/clearsight_mini.zip")
+    data_file = "./data/clearsight_mini.zip"
+    retrieve_unzip_file(download_url, data_file)
+
+    # dummy function to load labels
+    labels = load_fridge_object_detection_dataset_labels(automl_format)[:2]
+    if automl_format:
+        image_details = load_image_details()
+        columns = [ImageColumns.IMAGE.value,
+                   ImageColumns.IMAGE_DETAILS.value,
+                   ImageColumns.LABEL.value]
+    else:
+        columns = [ImageColumns.IMAGE.value,
+                   ImageColumns.LABEL.value]
+    features = []
+    for i, file in enumerate(os.listdir("./data/clearsight_mini")):
+        image_path = "./data/clearsight_mini/" + file
+        if automl_format:
+            row = {
+                ImageColumns.IMAGE.value: image_path,
+                ImageColumns.IMAGE_DETAILS.value: image_details[i],
+                ImageColumns.LABEL.value: labels[i]
+            }
+        else:
+            row = {
+                ImageColumns.IMAGE.value: image_path,
+                ImageColumns.LABEL.value: labels[i]
+            }
+        features.append(row)
+
+    data = pd.DataFrame(features, columns=columns)
+    return data
+
+
 class ImageTransformEnum(Enum):
     '''
     Possible modifications to images

diff --git a/responsibleai_vision/tests/test_feature_extractors.py b/responsibleai_vision/tests/test_feature_extractors.py
@@ -2,7 +2,8 @@
 # Licensed under the MIT License.
 
 import pytest
-from common_vision_utils import (load_flowers_dataset, load_fridge_dataset,
+from common_vision_utils import (load_clearsight_object_detection_dataset,
+                                 load_flowers_dataset, load_fridge_dataset,
                                  load_fridge_object_detection_dataset,
                                  load_imagenet_dataset)
 
@@ -13,22 +14,42 @@
 
 MEAN_PIXEL_VALUE = ExtractedFeatures.MEAN_PIXEL_VALUE.value
 FRIDGE_METADATA_FEATURES = [
-    'SensingMethod', 'GPSVersionID', 'ISOSpeedRatings', 'SceneType',
-    'SceneCaptureType', 'SubjectDistance', 'CustomRendered',
-    'SubjectDistanceRange', 'DigitalZoomRatio', 'ApertureValue',
-    'ImageWidth', 'GPSDOP', 'MaxApertureValue', 'ColorSpace',
-    'FocalLengthIn35mmFilm', 'ExposureMode', 'Saturation', 'ExposureTime',
-    'ExifImageHeight', 'FNumber', 'YCbCrPositioning', 'Make', 'MeteringMode',
-    'ExposureBiasValue', 'ExposureProgram', 'ComponentsConfiguration',
-    'ExifImageWidth', 'ExifInteroperabilityOffset', 'BrightnessValue',
-    'ImageLength', 'FlashPixVersion', 'SubsecTimeOriginal', 'Model',
-    'SubsecTimeDigitized', 'ResolutionUnit', 'DateTimeOriginal', 'XResolution',
-    'FocalLength', 'Sharpness', 'GPSLongitude', 'Contrast', 'Software',
-    'GPSLatitude', 'MakerNote', 'GPSDateStamp', 'GPSAltitude',
-    'GPSProcessingMethod', 'GPSTimeStamp', 'GPSLatitudeRef', 'WhiteBalance',
-    'GPSLongitudeRef', 'Flash', 'SubsecTime', 'YResolution',
-    'DateTimeDigitized', 'DateTime', 'GPSAltitudeRef', 'Orientation',
-    'ShutterSpeedValue', 'ExifVersion']
+    'FlashPixVersion', 'WhiteBalance', 'Make', 'ExifImageHeight',
+    'ExposureProgram', 'Software', 'CustomRendered', 'Contrast', 'XResolution',
+    'GPSTimeStamp', 'Orientation', 'SubjectDistanceRange', 'MakerNote',
+    'SubsecTimeDigitized', 'ExifInteroperabilityOffset', 'ColorSpace',
+    'BrightnessValue', 'ExifImageWidth', 'SubsecTimeOriginal',
+    'YCbCrPositioning', 'ISOSpeedRatings', 'SubsecTime', 'ExposureBiasValue',
+    'ComponentsConfiguration', 'ResolutionUnit', 'ImageLength',
+    'GPSLatitudeRef', 'ExifVersion', 'Flash', 'ExposureMode', 'ApertureValue',
+    'GPSDOP', 'ImageWidth', 'SubjectDistance', 'GPSLongitudeRef', 'FNumber',
+    'DigitalZoomRatio', 'FocalLength', 'SceneCaptureType',
+    'FocalLengthIn35mmFilm', 'ShutterSpeedValue', 'DateTimeOriginal',
+    'Sharpness', 'GPSAltitude', 'GPSLatitude', 'Saturation', 'ExposureTime',
+    'GPSLongitude', 'SceneType', 'GPSDateStamp', 'GPSVersionID',
+    'GPSAltitudeRef', 'YResolution', 'GPSProcessingMethod', 'DateTime',
+    'MeteringMode', 'SensingMethod', 'DateTimeDigitized', 'Model',
+    'MaxApertureValue'
+]
+CLEARSIGHT_METADATA_FEATURES = [
+    'FocalLength', 'Make', 'OffsetTimeOriginal', 'FocalPlaneYResolution',
+    'ExposureMode', 'LensSerialNumber', 'ApertureValue', 'GPSLatitudeRef',
+    'DateTimeDigitized', 'Orientation', 'OffsetTime',
+    'FocalPlaneResolutionUnit', 'ExposureProgram', 'ComponentsConfiguration',
+    'Flash', 'BodySerialNumber', 'SubsecTimeDigitized', 'CameraOwnerName',
+    'SubsecTime', 'ExifInteroperabilityOffset', 'ExposureBiasValue',
+    'ISOSpeedRatings', 'SubsecTimeOriginal', 'FNumber', 'WhiteBalance',
+    'GPSLatitude', 'XResolution', 'GPSSatellites', 'ShutterSpeedValue',
+    'GPSLongitudeRef', 'ExposureTime', 'DigitalZoomRatio', 'MeteringMode',
+    'ExifImageHeight', 'GPSTimeStamp', 'LensSpecification', 'Artist',
+    'FlashPixVersion', 'YCbCrPositioning', 'SceneCaptureType', 'DateTime',
+    'LensModel', 'OffsetTimeDigitized', 'GPSStatus', 'CustomRendered',
+    'GPSVersionID', 'SensitivityType', 'DateTimeOriginal', 'Copyright',
+    'ColorSpace', 'RecommendedExposureIndex', 'ResolutionUnit', 'GPSAltitude',
+    'Model', 'YResolution', 'ExifVersion', 'UserComment',
+    'FocalPlaneXResolution', 'GPSAltitudeRef', 'MakerNote', 'GPSMapDatum',
+    'GPSLongitude', 'GPSDateStamp', 'ExifImageWidth'
+]
 
 
 def validate_extracted_features(extracted_features, feature_names,
@@ -95,3 +116,14 @@ def test_extract_features_mixed_exif_XPComment_metadata(self):
         validate_extracted_features(extracted_features, feature_names,
                                     expected_feature_names, data,
                                     feature_metadata)
+
+    def test_extract_features_clearsight_metadata(self):
+        data = load_clearsight_object_detection_dataset()
+        feature_metadata = FeatureMetadata()
+        extracted_features, feature_names = extract_dataset_features(
+            data, feature_metadata=feature_metadata)
+        expected_feature_names = [MEAN_PIXEL_VALUE]
+        expected_feature_names += CLEARSIGHT_METADATA_FEATURES
+        validate_extracted_features(extracted_features, feature_names,
+                                    expected_feature_names, data,
+                                    feature_metadata)
diff --git a/responsibleai_vision/tests/test_image_utils.py b/responsibleai_vision/tests/test_image_utils.py
@@ -1,15 +1,15 @@
 # Copyright (c) Microsoft Corporation
 # Licensed under the MIT License.
 
-import platform
 from collections import Counter
 from http.client import HTTPMessage
 from math import isclose
 from unittest.mock import Mock, patch
 from urllib.parse import urlparse
 
 import numpy as np
-from common_vision_utils import load_fridge_object_detection_dataset
+from common_vision_utils import (load_clearsight_object_detection_dataset,
+                                 load_fridge_object_detection_dataset)
 
 from responsibleai_vision.common.constants import ImageColumns
 from responsibleai_vision.utils.image_reader import \
@@ -96,8 +96,12 @@ def test_retry_sessions_retries_on_conn_failure(self, request_mock):
     def test_get_all_exif_feature_names(self):
         image_dataset = load_fridge_object_detection_dataset().head(2)
         exif_feature_names = get_all_exif_feature_names(image_dataset)
-        num_features = 49 if platform.system() == "Linux" else 60
-        assert len(exif_feature_names) == num_features
+        assert len(exif_feature_names) == 60
+
+    def test_get_all_clearsight_feature_names(self):
+        image_dataset = load_clearsight_object_detection_dataset().head(2)
+        exif_feature_names = get_all_exif_feature_names(image_dataset)
+        assert len(exif_feature_names) == 64
 
     def test_generate_od_error_labels(self):
         true_y = np.array([[[3, 142, 257, 395, 463, 0]],