From b79f7a7ee723267b2067bb21dcfe7d573873876f Mon Sep 17 00:00:00 2001
From: Mathieu Beligon <mathieu@feedly.com>
Date: Tue, 31 Mar 2020 17:00:42 -0400
Subject: [PATCH] [common] (image pipeline evaluation) switch from unique ROCO
dataset to list of directory ROCO datasets
---
.../image_dataset_generator.py | 14 +++++++--
.../image_pipeline_evaluation_reporter.py | 8 ++---
.../image_pipeline_evaluator.py | 13 +++++----
.../armor_color_pipeline_reporter_factory.py | 12 +++++---
.../armor_color/baseline_experiments.py | 5 ++--
.../armor_digit_pipeline_reporter_factory.py | 12 ++++----
.../armor_digit/baseline_experiments.py | 5 ++--
.../dataset/armor_color_dataset_factory.py | 14 ++++-----
.../dataset/armor_digit_dataset_factory.py | 24 +++++++--------
.../dataset/armor_image_dataset_factory.py | 29 +++++++++++++++++++
10 files changed, 85 insertions(+), 51 deletions(-)
create mode 100644 robots-at-robots/research/dataset/armor_image_dataset_factory.py
diff --git a/common/research_common/image_pipeline_evaluation/image_dataset_generator.py b/common/research_common/image_pipeline_evaluation/image_dataset_generator.py
index b65ce92..b526836 100644
--- a/common/research_common/image_pipeline_evaluation/image_dataset_generator.py
+++ b/common/research_common/image_pipeline_evaluation/image_dataset_generator.py
@@ -1,13 +1,21 @@
from abc import abstractmethod
-from typing import TypeVar, Generic, Tuple, List
+from typing import TypeVar, Generic, Tuple, List, Iterable
from polystar.common.models.image import Image
-from research_common.dataset.roco_dataset import ROCODataset
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
T = TypeVar("T")
class ImageDatasetGenerator(Generic[T]):
@abstractmethod
- def from_roco_dataset(self, dataset: ROCODataset) -> Tuple[List[Image], List[T]]:
+ def from_roco_dataset(self, dataset: DirectoryROCODataset) -> Tuple[List[Image], List[T]]:
pass
+
+ def from_roco_datasets(self, datasets: Iterable[DirectoryROCODataset]) -> Tuple[List[Image], List[T]]:
+ images, labels = [], []
+ for dataset in datasets:
+ imgs, lbls = self.from_roco_dataset(dataset)
+ images.extend(imgs)
+ labels.extend(lbls)
+ return images, labels
diff --git a/common/research_common/image_pipeline_evaluation/image_pipeline_evaluation_reporter.py b/common/research_common/image_pipeline_evaluation/image_pipeline_evaluation_reporter.py
index 5a43e92..e0dc15a 100644
--- a/common/research_common/image_pipeline_evaluation/image_pipeline_evaluation_reporter.py
+++ b/common/research_common/image_pipeline_evaluation/image_pipeline_evaluation_reporter.py
@@ -41,14 +41,14 @@ class ImagePipelineEvaluationReporter:
mf.title("Datasets", level=2)
mf.title("Training", level=3)
- self._report_dataset(mf, self.evaluator.train_roco_dataset, self.evaluator.train_labels)
+ self._report_dataset(mf, self.evaluator.train_roco_datasets, self.evaluator.train_labels)
mf.title("Testing", level=3)
- self._report_dataset(mf, self.evaluator.test_roco_dataset, self.evaluator.test_labels)
+ self._report_dataset(mf, self.evaluator.test_roco_datasets, self.evaluator.test_labels)
@staticmethod
- def _report_dataset(mf: MarkdownFile, roco_dataset: ROCODataset, labels: List[Any]):
- mf.paragraph(roco_dataset.dataset_name)
+ def _report_dataset(mf: MarkdownFile, roco_datasets: List[ROCODataset], labels: List[Any]):
+ mf.list([dataset.dataset_name for dataset in roco_datasets])
label2count = Counter(labels)
total = len(labels)
mf.paragraph(f"{total} images")
diff --git a/common/research_common/image_pipeline_evaluation/image_pipeline_evaluator.py b/common/research_common/image_pipeline_evaluation/image_pipeline_evaluator.py
index 3f98a72..d952269 100644
--- a/common/research_common/image_pipeline_evaluation/image_pipeline_evaluator.py
+++ b/common/research_common/image_pipeline_evaluation/image_pipeline_evaluator.py
@@ -7,6 +7,7 @@ from sklearn.metrics import classification_report
from polystar.common.image_pipeline.image_pipeline import ImagePipeline
from polystar.common.models.image import Image
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
from research_common.dataset.roco_dataset import ROCODataset
from research_common.image_pipeline_evaluation.image_dataset_generator import ImageDatasetGenerator
@@ -23,15 +24,15 @@ class ClassificationResults:
class ImagePipelineEvaluator:
def __init__(
self,
- train_roco_dataset: ROCODataset,
- test_roco_dataset: ROCODataset,
+ train_roco_datasets: List[DirectoryROCODataset],
+ test_roco_datasets: List[DirectoryROCODataset],
image_dataset_generator: ImageDatasetGenerator,
):
logging.info("Loading data")
- self.train_roco_dataset = train_roco_dataset
- self.test_roco_dataset = test_roco_dataset
- self.train_images, self.train_labels = image_dataset_generator.from_roco_dataset(train_roco_dataset)
- self.test_images, self.test_labels = image_dataset_generator.from_roco_dataset(test_roco_dataset)
+ self.train_roco_datasets = train_roco_datasets
+ self.test_roco_datasets = test_roco_datasets
+ self.train_images, self.train_labels = image_dataset_generator.from_roco_datasets(train_roco_datasets)
+ self.test_images, self.test_labels = image_dataset_generator.from_roco_datasets(test_roco_datasets)
def evaluate_pipelines(self, pipelines: Iterable[ImagePipeline]) -> Dict[str, ClassificationResults]:
return {str(pipeline): self.evaluate(pipeline) for pipeline in pipelines}
diff --git a/robots-at-robots/research/armor_color/armor_color_pipeline_reporter_factory.py b/robots-at-robots/research/armor_color/armor_color_pipeline_reporter_factory.py
index dac836d..d6fa16e 100644
--- a/robots-at-robots/research/armor_color/armor_color_pipeline_reporter_factory.py
+++ b/robots-at-robots/research/armor_color/armor_color_pipeline_reporter_factory.py
@@ -1,16 +1,20 @@
+from typing import List
+
from research.dataset.armor_color_dataset_factory import ArmorColorDatasetGenerator
-from research_common.dataset.roco_dataset import ROCODataset
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
from research_common.image_pipeline_evaluation.image_pipeline_evaluation_reporter import ImagePipelineEvaluationReporter
from research_common.image_pipeline_evaluation.image_pipeline_evaluator import ImagePipelineEvaluator
class ArmorColorPipelineReporterFactory:
@staticmethod
- def from_roco_datasets(train_roco_dataset: ROCODataset, test_roco_dataset: ROCODataset):
+ def from_roco_datasets(
+ train_roco_datasets: List[DirectoryROCODataset], test_roco_datasets: List[DirectoryROCODataset]
+ ):
return ImagePipelineEvaluationReporter(
evaluator=ImagePipelineEvaluator(
- train_roco_dataset=train_roco_dataset,
- test_roco_dataset=test_roco_dataset,
+ train_roco_datasets=train_roco_datasets,
+ test_roco_datasets=test_roco_datasets,
image_dataset_generator=ArmorColorDatasetGenerator(),
),
evaluation_project="armor-color",
diff --git a/robots-at-robots/research/armor_color/baseline_experiments.py b/robots-at-robots/research/armor_color/baseline_experiments.py
index d85243d..87dfe72 100644
--- a/robots-at-robots/research/armor_color/baseline_experiments.py
+++ b/robots-at-robots/research/armor_color/baseline_experiments.py
@@ -6,14 +6,13 @@ from polystar.common.image_pipeline.models.random_model import RandomModel
from polystar.common.image_pipeline.models.red_blue_channels_comparison_model import RedBlueComparisonModel
from research.armor_color.armor_color_pipeline_reporter_factory import ArmorColorPipelineReporterFactory
from research_common.dataset.twitch.twitch_roco_datasets import TwitchROCODataset
-from research_common.dataset.union_dataset import UnionDataset
if __name__ == "__main__":
logging.getLogger().setLevel("INFO")
reporter = ArmorColorPipelineReporterFactory.from_roco_datasets(
- train_roco_dataset=UnionDataset(TwitchROCODataset.TWITCH_470151286, TwitchROCODataset.TWITCH_470150052),
- test_roco_dataset=TwitchROCODataset.TWITCH_470152289,
+ train_roco_datasets=[TwitchROCODataset.TWITCH_470151286, TwitchROCODataset.TWITCH_470150052],
+ test_roco_datasets=[TwitchROCODataset.TWITCH_470152289],
)
red_blue_comparison_pipeline = ImagePipeline(
diff --git a/robots-at-robots/research/armor_digit/armor_digit_pipeline_reporter_factory.py b/robots-at-robots/research/armor_digit/armor_digit_pipeline_reporter_factory.py
index 7e8afa1..5546f27 100644
--- a/robots-at-robots/research/armor_digit/armor_digit_pipeline_reporter_factory.py
+++ b/robots-at-robots/research/armor_digit/armor_digit_pipeline_reporter_factory.py
@@ -1,7 +1,7 @@
-from typing import Iterable
+from typing import Iterable, List
from research.dataset.armor_digit_dataset_factory import ArmorDigitDatasetGenerator
-from research_common.dataset.roco_dataset import ROCODataset
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
from research_common.image_pipeline_evaluation.image_pipeline_evaluation_reporter import ImagePipelineEvaluationReporter
from research_common.image_pipeline_evaluation.image_pipeline_evaluator import ImagePipelineEvaluator
@@ -9,14 +9,14 @@ from research_common.image_pipeline_evaluation.image_pipeline_evaluator import I
class ArmorDigitPipelineReporterFactory:
@staticmethod
def from_roco_datasets(
- train_roco_dataset: ROCODataset,
- test_roco_dataset: ROCODataset,
+ train_roco_datasets: List[DirectoryROCODataset],
+ test_roco_datasets: List[DirectoryROCODataset],
acceptable_digits: Iterable[int] = (1, 2, 3, 4, 5, 7),
):
return ImagePipelineEvaluationReporter(
evaluator=ImagePipelineEvaluator(
- train_roco_dataset=train_roco_dataset,
- test_roco_dataset=test_roco_dataset,
+ train_roco_datasets=train_roco_datasets,
+ test_roco_datasets=test_roco_datasets,
image_dataset_generator=ArmorDigitDatasetGenerator(set(acceptable_digits)),
),
evaluation_project="armor-digit",
diff --git a/robots-at-robots/research/armor_digit/baseline_experiments.py b/robots-at-robots/research/armor_digit/baseline_experiments.py
index 9b9cc35..01460f6 100644
--- a/robots-at-robots/research/armor_digit/baseline_experiments.py
+++ b/robots-at-robots/research/armor_digit/baseline_experiments.py
@@ -4,14 +4,13 @@ from polystar.common.image_pipeline.image_pipeline import ImagePipeline
from polystar.common.image_pipeline.models.random_model import RandomModel
from research.armor_digit.armor_digit_pipeline_reporter_factory import ArmorDigitPipelineReporterFactory
from research_common.dataset.twitch.twitch_roco_datasets import TwitchROCODataset
-from research_common.dataset.union_dataset import UnionDataset
if __name__ == "__main__":
logging.getLogger().setLevel("INFO")
reporter = ArmorDigitPipelineReporterFactory.from_roco_datasets(
- train_roco_dataset=UnionDataset(TwitchROCODataset.TWITCH_470151286, TwitchROCODataset.TWITCH_470150052),
- test_roco_dataset=TwitchROCODataset.TWITCH_470152289,
+ train_roco_datasets=[TwitchROCODataset.TWITCH_470151286, TwitchROCODataset.TWITCH_470150052],
+ test_roco_datasets=[TwitchROCODataset.TWITCH_470152289],
)
random_pipeline = ImagePipeline(model=RandomModel(), custom_name="random")
diff --git a/robots-at-robots/research/dataset/armor_color_dataset_factory.py b/robots-at-robots/research/dataset/armor_color_dataset_factory.py
index c32fe98..83cf323 100644
--- a/robots-at-robots/research/dataset/armor_color_dataset_factory.py
+++ b/robots-at-robots/research/dataset/armor_color_dataset_factory.py
@@ -1,11 +1,9 @@
-from typing import Tuple, Sequence
+from pathlib import Path
-from polystar.common.models.image import Image
-from research.dataset.armor_dataset_factory import ArmorDatasetFactory
-from research_common.dataset.roco_dataset import ROCODataset
-from research_common.image_pipeline_evaluation.image_dataset_generator import ImageDatasetGenerator
+from polystar.common.models.object import ArmorColor
+from research.dataset.armor_image_dataset_factory import ArmorImageDatasetGenerator
-class ArmorColorDatasetGenerator(ImageDatasetGenerator[str]):
- def from_roco_dataset(self, dataset: ROCODataset) -> Tuple[Sequence[Image], Sequence[str]]:
- return zip(*[(armor_img, c.name) for (armor_img, c, n, k, p) in ArmorDatasetFactory.from_dataset(dataset)])
+class ArmorColorDatasetGenerator(ArmorImageDatasetGenerator[str]):
+ def _label(self, color: ArmorColor, digit: int, k: int, path: Path) -> str:
+ return color.name
diff --git a/robots-at-robots/research/dataset/armor_digit_dataset_factory.py b/robots-at-robots/research/dataset/armor_digit_dataset_factory.py
index d7f0b93..f1ed6f1 100644
--- a/robots-at-robots/research/dataset/armor_digit_dataset_factory.py
+++ b/robots-at-robots/research/dataset/armor_digit_dataset_factory.py
@@ -1,20 +1,16 @@
-from typing import Tuple, Sequence, Set
+from pathlib import Path
+from typing import Set
-from polystar.common.models.image import Image
-from research.dataset.armor_dataset_factory import ArmorDatasetFactory
-from research_common.dataset.roco_dataset import ROCODataset
-from research_common.image_pipeline_evaluation.image_dataset_generator import ImageDatasetGenerator
+from polystar.common.models.object import ArmorColor
+from research.dataset.armor_image_dataset_factory import ArmorImageDatasetGenerator
-class ArmorDigitDatasetGenerator(ImageDatasetGenerator[str]):
+class ArmorDigitDatasetGenerator(ArmorImageDatasetGenerator[int]):
def __init__(self, acceptable_digits: Set[int]):
self.acceptable_digits = acceptable_digits
- def from_roco_dataset(self, dataset: ROCODataset) -> Tuple[Sequence[Image], Sequence[int]]:
- return zip(
- *[
- (armor_img, digit)
- for (armor_img, c, digit, k, p) in ArmorDatasetFactory.from_dataset(dataset)
- if digit in self.acceptable_digits
- ]
- )
+ def _label(self, color: ArmorColor, number: int, k: int, path: Path) -> int:
+ return number
+
+ def _valid_label(self, label: int) -> bool:
+ return label in self.acceptable_digits
diff --git a/robots-at-robots/research/dataset/armor_image_dataset_factory.py b/robots-at-robots/research/dataset/armor_image_dataset_factory.py
new file mode 100644
index 0000000..9ce0a1a
--- /dev/null
+++ b/robots-at-robots/research/dataset/armor_image_dataset_factory.py
@@ -0,0 +1,29 @@
+from abc import abstractmethod
+from pathlib import Path
+from typing import TypeVar, Tuple, List
+
+from polystar.common.models.image import Image
+from polystar.common.models.object import ArmorColor
+from research.dataset.armor_dataset_factory import ArmorDatasetFactory
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
+from research_common.image_pipeline_evaluation.image_dataset_generator import ImageDatasetGenerator
+
+T = TypeVar("T")
+
+
+class ArmorImageDatasetGenerator(ImageDatasetGenerator[T]):
+ def from_roco_dataset(self, dataset: DirectoryROCODataset) -> Tuple[List[Image], List[T]]:
+ images, labels = [], []
+ for (armor_img, color, digit, k, path) in ArmorDatasetFactory.from_dataset(dataset):
+ label = self._label(color, digit, k, path)
+ if self._valid_label(label):
+ images.append(armor_img)
+ labels.append(label)
+ return images, labels
+
+ @abstractmethod
+ def _label(self, color: ArmorColor, digit: int, k: int, path: Path) -> T:
+ pass
+
+ def _valid_label(self, label: T) -> bool:
+ return True
--
GitLab