From 27ca26a417093e1f4d65ab4588485524baa07f3a Mon Sep 17 00:00:00 2001
From: Mathieu Beligon <mathieu@feedly.com>
Date: Thu, 17 Sep 2020 12:10:28 +0200
Subject: [PATCH] [robots@robots] (minor) make the evaluation objects generic
---
.../dataset/image_dataset_generator.py | 27 ---------------
.../image_pipeline_evaluation_reporter.py | 27 +++++++++------
.../evaluation/image_pipeline_evaluator.py | 34 ++++++++++---------
3 files changed, 34 insertions(+), 54 deletions(-)
delete mode 100644 robots-at-robots/research/robots_at_robots/dataset/image_dataset_generator.py
diff --git a/robots-at-robots/research/robots_at_robots/dataset/image_dataset_generator.py b/robots-at-robots/research/robots_at_robots/dataset/image_dataset_generator.py
deleted file mode 100644
index f9b321b..0000000
--- a/robots-at-robots/research/robots_at_robots/dataset/image_dataset_generator.py
+++ /dev/null
@@ -1,27 +0,0 @@
-from abc import abstractmethod
-from pathlib import Path
-from typing import Generic, Iterable, List, Tuple, TypeVar
-
-from polystar.common.models.image import Image, load_image
-from research.common.dataset.directory_roco_dataset import DirectoryROCODataset
-
-T = TypeVar("T")
-
-
-class ImageDatasetGenerator(Generic[T]):
- def from_roco_datasets(
- self, datasets: Iterable[DirectoryROCODataset]
- ) -> Tuple[List[Path], List[Image], List[T], List[int]]:
- images_path, images, labels, dataset_sizes = [], [], [], []
- for dataset in datasets:
- prev_total_size = len(images)
- for img_path, label in self.from_roco_dataset(dataset):
- images_path.append(img_path)
- images.append(load_image(img_path))
- labels.append(label)
- dataset_sizes.append(len(images) - prev_total_size)
- return images_path, images, labels, dataset_sizes
-
- @abstractmethod
- def from_roco_dataset(self, dataset: DirectoryROCODataset) -> Iterable[Tuple[Path, T]]:
- pass
diff --git a/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluation_reporter.py b/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluation_reporter.py
index 0e3cd25..dadcd94 100644
--- a/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluation_reporter.py
+++ b/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluation_reporter.py
@@ -2,24 +2,27 @@ from collections import Counter
from dataclasses import dataclass
from os.path import relpath
from pathlib import Path
-from typing import Any, Dict, Iterable, List, Tuple
+from typing import Any, Dict, Iterable, List, Tuple, Generic
import numpy as np
from pandas import DataFrame
from polystar.common.image_pipeline.image_pipeline import ImagePipeline
-from polystar.common.utils.dataframe import (format_df_column, format_df_row,
- format_df_rows)
+from polystar.common.utils.dataframe import format_df_column, format_df_row, format_df_rows
from polystar.common.utils.markdown import MarkdownFile
from polystar.common.utils.time import create_time_id
from research.common.constants import DSET_DIR, EVALUATION_DIR
from research.common.datasets.roco.roco_dataset import ROCOFileDataset
+from research.robots_at_robots.dataset.armor_value_dataset import ValueT
from research.robots_at_robots.evaluation.image_pipeline_evaluator import (
- ClassificationResults, ImagePipelineEvaluator, SetClassificationResults)
+ ClassificationResults,
+ ImagePipelineEvaluator,
+ SetClassificationResults,
+)
@dataclass
-class ImagePipelineEvaluationReporter:
- evaluator: ImagePipelineEvaluator
+class ImagePipelineEvaluationReporter(Generic[ValueT]):
+ evaluator: ImagePipelineEvaluator[ValueT]
evaluation_project: str
main_metric: Tuple[str, str] = ("f1-score", "weighted avg")
@@ -67,17 +70,17 @@ class ImagePipelineEvaluationReporter:
df["Repartition"] = (df["Total"] / total).map("{:.1%}".format)
mf.table(df)
- def _report_aggregated_results(self, mf: MarkdownFile, pipeline2results: Dict[str, ClassificationResults]):
+ def _report_aggregated_results(self, mf: MarkdownFile, pipeline2results: Dict[str, ClassificationResults[ValueT]]):
aggregated_results = self._aggregate_results(pipeline2results)
mf.title("Aggregated results", level=2)
mf.paragraph("On test set:")
mf.table(aggregated_results)
- def _report_pipelines_results(self, mf: MarkdownFile, pipeline2results: Dict[str, ClassificationResults]):
+ def _report_pipelines_results(self, mf: MarkdownFile, pipeline2results: Dict[str, ClassificationResults[ValueT]]):
for pipeline_name, results in pipeline2results.items():
self._report_pipeline_results(mf, pipeline_name, results)
- def _report_pipeline_results(self, mf: MarkdownFile, pipeline_name: str, results: ClassificationResults):
+ def _report_pipeline_results(self, mf: MarkdownFile, pipeline_name: str, results: ClassificationResults[ValueT]):
mf.title(pipeline_name, level=2)
mf.paragraph(results.full_pipeline_name)
@@ -93,7 +96,9 @@ class ImagePipelineEvaluationReporter:
)
@staticmethod
- def _report_pipeline_set_results(mf: MarkdownFile, results: SetClassificationResults, image_paths: List[Path]):
+ def _report_pipeline_set_results(
+ mf: MarkdownFile, results: SetClassificationResults[ValueT], image_paths: List[Path]
+ ):
mf.title("Metrics", level=4)
mf.paragraph(f"Inference time: {results.mean_inference_time: .2e} s/img")
df = DataFrame(results.report)
@@ -119,7 +124,7 @@ class ImagePipelineEvaluationReporter:
).set_index("images")
)
- def _aggregate_results(self, pipeline2results: Dict[str, ClassificationResults]) -> DataFrame:
+ def _aggregate_results(self, pipeline2results: Dict[str, ClassificationResults[ValueT]]) -> DataFrame:
main_metric_name = f"{self.main_metric[0]} {self.main_metric[1]}"
df = DataFrame(columns=["pipeline", main_metric_name, "inf time"]).set_index("pipeline")
diff --git a/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluator.py b/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluator.py
index cb4797a..27afedb 100644
--- a/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluator.py
+++ b/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluator.py
@@ -2,7 +2,7 @@ import logging
from dataclasses import dataclass
from pathlib import Path
from time import time
-from typing import Any, Dict, Iterable, List, Sequence, Tuple
+from typing import Dict, Generic, Iterable, List, Sequence, Tuple
import numpy as np
from memoized_property import memoized_property
@@ -10,13 +10,13 @@ from polystar.common.image_pipeline.image_pipeline import ImagePipeline
from polystar.common.models.image import Image, load_images
from research.common.datasets.roco.directory_roco_dataset import \
DirectoryROCODataset
-from research.robots_at_robots.dataset.armor_value_dataset import \
- ArmorValueDatasetCache
+from research.robots_at_robots.dataset.armor_value_dataset import (
+ ArmorValueDatasetCache, ValueT)
from sklearn.metrics import classification_report, confusion_matrix
@dataclass
-class SetClassificationResults:
+class SetClassificationResults(Generic[ValueT]):
labels: np.ndarray
predictions: np.ndarray
mean_inference_time: float
@@ -34,23 +34,23 @@ class SetClassificationResults:
return np.where(self.labels != self.predictions)[0]
@memoized_property
- def unique_labels(self) -> List[Any]:
+ def unique_labels(self) -> List[ValueT]:
return sorted(set(self.labels) | set(self.predictions))
@dataclass
-class ClassificationResults:
- train_results: SetClassificationResults
- test_results: SetClassificationResults
+class ClassificationResults(Generic[ValueT]):
+ train_results: SetClassificationResults[ValueT]
+ test_results: SetClassificationResults[ValueT]
full_pipeline_name: str
-class ImagePipelineEvaluator:
+class ImagePipelineEvaluator(Generic[ValueT]):
def __init__(
self,
train_roco_datasets: List[DirectoryROCODataset],
test_roco_datasets: List[DirectoryROCODataset],
- image_dataset_cache: ArmorValueDatasetCache,
+ image_dataset_cache: ArmorValueDatasetCache[ValueT],
):
logging.info("Loading data")
self.train_roco_datasets = train_roco_datasets
@@ -63,22 +63,24 @@ class ImagePipelineEvaluator:
)
def evaluate_pipelines(self, pipelines: Iterable[ImagePipeline]) -> Dict[str, ClassificationResults]:
- return {str(pipeline): self.evaluate(pipeline) for pipeline in pipelines}
+ return {str(pipeline): self.evaluate_pipeline(pipeline) for pipeline in pipelines}
- def evaluate(self, pipeline: ImagePipeline) -> ClassificationResults:
+ def evaluate_pipeline(self, pipeline: ImagePipeline) -> ClassificationResults:
logging.info(f"Training pipeline {pipeline}")
pipeline.fit(self.train_images, self.train_labels)
logging.info(f"Infering")
- train_results = self._evaluate_on_set(pipeline, self.train_images, self.train_labels)
- test_results = self._evaluate_on_set(pipeline, self.test_images, self.test_labels)
+ train_results = self._evaluate_pipeline_on_set(pipeline, self.train_images, self.train_labels)
+ test_results = self._evaluate_pipeline_on_set(pipeline, self.test_images, self.test_labels)
return ClassificationResults(
train_results=train_results, test_results=test_results, full_pipeline_name=repr(pipeline),
)
@staticmethod
- def _evaluate_on_set(pipeline: ImagePipeline, images: List[Image], labels: List[Any]) -> SetClassificationResults:
+ def _evaluate_pipeline_on_set(
+ pipeline: ImagePipeline, images: List[Image], labels: List[ValueT]
+ ) -> SetClassificationResults:
t = time()
preds = pipeline.predict(images)
mean_time = (time() - t) / len(images)
@@ -87,7 +89,7 @@ class ImagePipelineEvaluator:
def load_datasets(
roco_datasets: List[DirectoryROCODataset], image_dataset_cache: ArmorValueDatasetCache,
-) -> Tuple[List[Path], List[Image], List[Any], List[int]]:
+) -> Tuple[List[Path], List[Image], List[ValueT], List[int]]:
dataset = image_dataset_cache.from_roco_datasets(roco_datasets)
dataset_sizes = [len(d) for d in dataset.datasets]
--
GitLab