diff --git a/common/README.md b/common/README.md
index 5c1153e7b4b709bc41cc8c1f6641df7cc1e923a0..a4fbdcaf17019aa71d1731abf0bf7c8221e2579e 100644
--- a/common/README.md
+++ b/common/README.md
@@ -44,7 +44,20 @@ It will enable the relative imports.
#### ROCO
-Telechargez le dossier [PolySTAR/RoboMaster/EÌquipe-Computer vision/DJI ROCO](https://drive.google.com/drive/folders/1AM3PqwwHzlK3tAS-1R5Qk3edPv0T4NzB) du drive, contenant tous les datasets fournis par DJI, et unzippez le dans le dossier [dataset/dji_roco](../dataset/dji_roco).
+Download the directory [PolySTAR/RoboMaster/EÌquipe-Computer vision/DJI ROCO](https://drive.google.com/drive/folders/1AM3PqwwHzlK3tAS-1R5Qk3edPv0T4NzB) in Drive, with the 4 datasets given by DJI, and unzip in folder [dataset/dji_roco](../dataset/dji_roco).
+
+The dji_roco directory should look like:
+
+
+
+#### TWITCH
+
+Download the directory [PolySTAR/RoboMaster/EÌquipe-Computer vision/datasets/twitch/v1](https://drive.google.com/drive/folders/1TaxdzB57U9wII9K2VDOEP8vUMm94_cR7) in Drive, with the 8 labelized videos from twitch, and unzip in folder [dataset/twitch/v1](../dataset/twitch/v1).
+
+The twitch/v1 directory should look like:
+
+
+
## Dataset creation
diff --git a/common/doc/dataset_dji_repo.png b/common/doc/dataset_dji_repo.png
new file mode 100644
index 0000000000000000000000000000000000000000..b34b590d26696a0652221e5abeed5833cbdf5d14
Binary files /dev/null and b/common/doc/dataset_dji_repo.png differ
diff --git a/common/doc/dataset_twitch_repo.png b/common/doc/dataset_twitch_repo.png
new file mode 100644
index 0000000000000000000000000000000000000000..5d1bd5958d432f370a722753716fb851119752bf
Binary files /dev/null and b/common/doc/dataset_twitch_repo.png differ
diff --git a/common/poetry.lock b/common/poetry.lock
index 4e43268da15169a8c4d31bf39da2450e37807273..9e094771826283b4ca1bb9f359d03b272c60188a 100644
Binary files a/common/poetry.lock and b/common/poetry.lock differ
diff --git a/common/polystar/common/image_pipeline/models/model_abc.py b/common/polystar/common/image_pipeline/models/model_abc.py
index c4a4c38cc5fc0582176fc306f810598dc25c790b..00d480831b8521649d7bd27d6de18c1eac2161c8 100644
--- a/common/polystar/common/image_pipeline/models/model_abc.py
+++ b/common/polystar/common/image_pipeline/models/model_abc.py
@@ -3,11 +3,11 @@ from typing import Any, List, Sequence
class ModelABC(ABC):
- def fit(self, features: Any, labels: List[Any]) -> "ModelABC":
+ def fit(self, features: List[Any], labels: List[Any]) -> "ModelABC":
return self
@abstractmethod
- def predict(self, features: Any) -> Sequence[Any]:
+ def predict(self, features: List[Any]) -> Sequence[Any]:
pass
@abstractmethod
diff --git a/common/polystar/common/image_pipeline/models/random_model.py b/common/polystar/common/image_pipeline/models/random_model.py
index 5c2f798070ad16c9a1ad7745edc13fb0204ad583..3c938d9ba4d1f41485e45c959c3b46ed8c104824 100644
--- a/common/polystar/common/image_pipeline/models/random_model.py
+++ b/common/polystar/common/image_pipeline/models/random_model.py
@@ -12,14 +12,14 @@ class RandomModel(ModelABC):
labels_: np.ndarray = field(init=False, default=None)
weights_: np.ndarray = field(init=False, default=None)
- def fit(self, features: Any, labels: List[Any]) -> "RandomModel":
+ def fit(self, features: List[Any], labels: List[Any]) -> "RandomModel":
counter = Counter(labels)
self.labels_ = np.asarray(list(counter.keys()))
occurrences = np.asarray(list(counter.values()))
self.weights_ = occurrences / occurrences.sum()
return self
- def predict(self, features: Any) -> Sequence[Any]:
+ def predict(self, features: List[Any]) -> Sequence[Any]:
return np.random.choice(self.labels_, size=len(features), replace=True, p=self.weights_)
def __str__(self) -> str:
diff --git a/common/polystar/common/models/image.py b/common/polystar/common/models/image.py
index 1bf4fc61f065337992ef42e9f181f79729679b7c..33d63c22ab531203a2fa9b6439769640bb9729d7 100644
--- a/common/polystar/common/models/image.py
+++ b/common/polystar/common/models/image.py
@@ -8,3 +8,7 @@ class Image(Array[int, ..., ..., 3]):
@staticmethod
def from_path(image_path: Path) -> "Image":
return cv2.cvtColor(cv2.imread(str(image_path)), cv2.COLOR_BGR2RGB)
+
+ def save(self, image_path: Path):
+ image_path.parent.mkdir(exist_ok=True, parents=True)
+ cv2.imwrite(str(image_path), cv2.cvtColor(self, cv2.COLOR_RGB2BGR))
diff --git a/common/polystar/common/models/object.py b/common/polystar/common/models/object.py
index aee53bcd44ab71cb0abfc2795a6739625bf268ee..6946967b9da3d0e5be4d1f7eaecf83cafdbccb8c 100644
--- a/common/polystar/common/models/object.py
+++ b/common/polystar/common/models/object.py
@@ -54,6 +54,8 @@ class ObjectFactory:
int(float(json["bndbox"]["ymax"])) - int(float(json["bndbox"]["ymin"])),
)
+ x, y = max(0, x), max(0, y)
+
if t is not ObjectType.Armor:
return Object(type=t, x=x, y=y, w=w, h=h)
diff --git a/common/research_common/dataset/twitch/twitch_roco_datasets.py b/common/research_common/dataset/twitch/twitch_roco_datasets.py
index a04bb75a3fac63e473d05d16463924f13a3f7f9e..368a7c48ee12d59cb1f91df26554f85e1f936a7d 100644
--- a/common/research_common/dataset/twitch/twitch_roco_datasets.py
+++ b/common/research_common/dataset/twitch/twitch_roco_datasets.py
@@ -1,15 +1,18 @@
from enum import Enum
-from research_common.constants import TWITCH_DSET_ROBOTS_VIEWS_DIR
+from research_common.constants import TWITCH_DSET_DIR
from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
class TwitchROCODataset(DirectoryROCODataset, Enum):
def __init__(self, competition_name: str):
- super().__init__(TWITCH_DSET_ROBOTS_VIEWS_DIR / competition_name, self.name)
+ super().__init__(TWITCH_DSET_DIR / "v1" / competition_name, self.name)
+ TWITCH_470149568 = "470149568"
TWITCH_470150052 = "470150052"
TWITCH_470151286 = "470151286"
+ TWITCH_470152289 = "470152289"
+ TWITCH_470152730 = "470152730"
TWITCH_470152838 = "470152838"
TWITCH_470153081 = "470153081"
- TWITCH_470152289 = "470152289"
+ TWITCH_470158483 = "470158483"
diff --git a/common/research_common/image_pipeline_evaluation/image_dataset_generator.py b/common/research_common/image_pipeline_evaluation/image_dataset_generator.py
index b65ce92537669be951a60a66fdeb763e7b24102d..c2848779ea75666d70dde2931dffa9ed096dce8b 100644
--- a/common/research_common/image_pipeline_evaluation/image_dataset_generator.py
+++ b/common/research_common/image_pipeline_evaluation/image_dataset_generator.py
@@ -1,13 +1,27 @@
from abc import abstractmethod
-from typing import TypeVar, Generic, Tuple, List
+from pathlib import Path
+from typing import TypeVar, Generic, Tuple, List, Iterable
from polystar.common.models.image import Image
-from research_common.dataset.roco_dataset import ROCODataset
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
T = TypeVar("T")
class ImageDatasetGenerator(Generic[T]):
+ def from_roco_datasets(
+ self, datasets: Iterable[DirectoryROCODataset]
+ ) -> Tuple[List[Path], List[Image], List[T], List[int]]:
+ images_path, images, labels, dataset_sizes = [], [], [], []
+ for dataset in datasets:
+ prev_total_size = len(images)
+ for img_path, label in self.from_roco_dataset(dataset):
+ images_path.append(img_path)
+ images.append(Image.from_path(img_path))
+ labels.append(label)
+ dataset_sizes.append(len(images) - prev_total_size)
+ return images_path, images, labels, dataset_sizes
+
@abstractmethod
- def from_roco_dataset(self, dataset: ROCODataset) -> Tuple[List[Image], List[T]]:
+ def from_roco_dataset(self, dataset: DirectoryROCODataset) -> Iterable[Tuple[Path, T]]:
pass
diff --git a/common/research_common/image_pipeline_evaluation/image_pipeline_evaluation_reporter.py b/common/research_common/image_pipeline_evaluation/image_pipeline_evaluation_reporter.py
index 5a43e92394ef0adc5419777208df1ecbdf63b277..336a970cca3d768371bdaf3de3d588acc1b2e3f7 100644
--- a/common/research_common/image_pipeline_evaluation/image_pipeline_evaluation_reporter.py
+++ b/common/research_common/image_pipeline_evaluation/image_pipeline_evaluation_reporter.py
@@ -1,18 +1,22 @@
from collections import Counter
from dataclasses import dataclass
+from os.path import relpath
+from pathlib import Path
from typing import Iterable, List, Any, Dict, Tuple
+import numpy as np
from pandas import DataFrame
from polystar.common.image_pipeline.image_pipeline import ImagePipeline
from polystar.common.utils.dataframe import format_df_rows, format_df_row, format_df_column
from polystar.common.utils.markdown import MarkdownFile
from polystar.common.utils.time import create_time_id
-from research_common.constants import EVALUATION_DIR
+from research_common.constants import EVALUATION_DIR, DSET_DIR
from research_common.dataset.roco_dataset import ROCODataset
from research_common.image_pipeline_evaluation.image_pipeline_evaluator import (
ImagePipelineEvaluator,
ClassificationResults,
+ SetClassificationResults,
)
@@ -41,18 +45,30 @@ class ImagePipelineEvaluationReporter:
mf.title("Datasets", level=2)
mf.title("Training", level=3)
- self._report_dataset(mf, self.evaluator.train_roco_dataset, self.evaluator.train_labels)
+ self._report_dataset(
+ mf, self.evaluator.train_roco_datasets, self.evaluator.train_dataset_sizes, self.evaluator.train_labels
+ )
mf.title("Testing", level=3)
- self._report_dataset(mf, self.evaluator.test_roco_dataset, self.evaluator.test_labels)
+ self._report_dataset(
+ mf, self.evaluator.test_roco_datasets, self.evaluator.test_dataset_sizes, self.evaluator.test_labels
+ )
@staticmethod
- def _report_dataset(mf: MarkdownFile, roco_dataset: ROCODataset, labels: List[Any]):
- mf.paragraph(roco_dataset.dataset_name)
- label2count = Counter(labels)
+ def _report_dataset(
+ mf: MarkdownFile, roco_datasets: List[ROCODataset], dataset_sizes: List[int], labels: List[Any]
+ ):
total = len(labels)
mf.paragraph(f"{total} images")
- mf.list(f"{label}: {count} ({count / total:.1%})" for label, count in sorted(label2count.items()))
+ df = DataFrame(
+ {
+ dataset.dataset_name: Counter(labels[start:end])
+ for dataset, start, end in zip(roco_datasets, np.cumsum([0] + dataset_sizes), np.cumsum(dataset_sizes))
+ }
+ ).fillna(0)
+ df["Total"] = sum([df[d.dataset_name] for d in roco_datasets])
+ df["Repartition"] = (df["Total"] / total).map("{:.1%}".format)
+ mf.table(df)
def _report_aggregated_results(self, mf: MarkdownFile, pipeline2results: Dict[str, ClassificationResults]):
aggregated_results = self._aggregate_results(pipeline2results)
@@ -64,28 +80,47 @@ class ImagePipelineEvaluationReporter:
for pipeline_name, results in pipeline2results.items():
self._report_pipeline_results(mf, pipeline_name, results)
- @staticmethod
- def _report_pipeline_results(mf: MarkdownFile, pipeline_name: str, results: ClassificationResults):
+ def _report_pipeline_results(self, mf: MarkdownFile, pipeline_name: str, results: ClassificationResults):
mf.title(pipeline_name, level=2)
mf.paragraph(results.full_pipeline_name)
mf.title("Train results", level=3)
-
- mf.paragraph(f"Inference time: {results.train_mean_inference_time: .2e} s/img")
- ImagePipelineEvaluationReporter._report_pipeline_set_report(mf, results.train_report)
+ ImagePipelineEvaluationReporter._report_pipeline_set_results(
+ mf, results.train_results, self.evaluator.train_images_paths
+ )
mf.title("Test results", level=3)
-
- mf.paragraph(f"Inference time: {results.test_mean_inference_time: .2e} s/img")
- ImagePipelineEvaluationReporter._report_pipeline_set_report(mf, results.test_report)
+ ImagePipelineEvaluationReporter._report_pipeline_set_results(
+ mf, results.test_results, self.evaluator.test_images_paths
+ )
@staticmethod
- def _report_pipeline_set_report(mf: MarkdownFile, set_report: Dict):
- df = DataFrame(set_report)
+ def _report_pipeline_set_results(mf: MarkdownFile, results: SetClassificationResults, image_paths: List[Path]):
+ mf.title("Metrics", level=4)
+ mf.paragraph(f"Inference time: {results.mean_inference_time: .2e} s/img")
+ df = DataFrame(results.report)
format_df_rows(df, ["precision", "recall", "f1-score"], "{:.1%}")
format_df_row(df, "support", int)
mf.table(df)
+ mf.title("Confusion Matrix:", level=4)
+ mf.table(DataFrame(results.confusion_matrix))
+ mf.title("10 Mistakes examples", level=4)
+ mistakes_idx = np.random.choice(results.mistakes, min(len(results.mistakes), 10), replace=False)
+ relative_paths = [
+ f", str(mf.markdown_path.parent))})" for idx in mistakes_idx
+ ]
+ images_names = [image_paths[idx].relative_to(DSET_DIR) for idx in mistakes_idx]
+ mf.table(
+ DataFrame(
+ {
+ "images": relative_paths,
+ "labels": results.labels[mistakes_idx],
+ "predictions": results.predictions[mistakes_idx],
+ "image names": images_names,
+ }
+ ).set_index("images")
+ )
def _aggregate_results(self, pipeline2results: Dict[str, ClassificationResults]) -> DataFrame:
main_metric_name = f"{self.main_metric[0]} {self.main_metric[1]}"
@@ -93,8 +128,8 @@ class ImagePipelineEvaluationReporter:
for pipeline_name, results in pipeline2results.items():
df.loc[pipeline_name] = [
- results.test_report[self.main_metric[1]][self.main_metric[0]],
- results.test_mean_inference_time,
+ results.test_results.report[self.main_metric[1]][self.main_metric[0]],
+ results.test_results.mean_inference_time,
]
df = df.sort_values(main_metric_name, ascending=False)
diff --git a/common/research_common/image_pipeline_evaluation/image_pipeline_evaluator.py b/common/research_common/image_pipeline_evaluation/image_pipeline_evaluator.py
index 3f98a72112e3a35d4b5a61545a96bcc45183bf80..24fdfa63bfc5f21bde2a16cfcd9963405ee0f2e1 100644
--- a/common/research_common/image_pipeline_evaluation/image_pipeline_evaluator.py
+++ b/common/research_common/image_pipeline_evaluation/image_pipeline_evaluator.py
@@ -1,37 +1,65 @@
import logging
from dataclasses import dataclass
from time import time
-from typing import List, Tuple, Dict, Any, Iterable
+from typing import List, Dict, Any, Iterable, Sequence
-from sklearn.metrics import classification_report
+import numpy as np
+from sklearn.metrics import classification_report, confusion_matrix
from polystar.common.image_pipeline.image_pipeline import ImagePipeline
from polystar.common.models.image import Image
-from research_common.dataset.roco_dataset import ROCODataset
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
from research_common.image_pipeline_evaluation.image_dataset_generator import ImageDatasetGenerator
+@dataclass
+class SetClassificationResults:
+ labels: np.ndarray
+ predictions: np.ndarray
+ mean_inference_time: float
+
+ @property
+ def report(self) -> Dict:
+ return classification_report(self.labels, self.predictions, output_dict=True)
+
+ @property
+ def confusion_matrix(self) -> Dict:
+ return confusion_matrix(self.labels, self.predictions)
+
+ @property
+ def mistakes(self) -> Sequence[int]:
+ return np.where(self.labels != self.predictions)[0]
+
+
@dataclass
class ClassificationResults:
- train_report: Dict
- train_mean_inference_time: float
- test_report: Dict
- test_mean_inference_time: float
+ train_results: SetClassificationResults
+ test_results: SetClassificationResults
full_pipeline_name: str
class ImagePipelineEvaluator:
def __init__(
self,
- train_roco_dataset: ROCODataset,
- test_roco_dataset: ROCODataset,
+ train_roco_datasets: List[DirectoryROCODataset],
+ test_roco_datasets: List[DirectoryROCODataset],
image_dataset_generator: ImageDatasetGenerator,
):
logging.info("Loading data")
- self.train_roco_dataset = train_roco_dataset
- self.test_roco_dataset = test_roco_dataset
- self.train_images, self.train_labels = image_dataset_generator.from_roco_dataset(train_roco_dataset)
- self.test_images, self.test_labels = image_dataset_generator.from_roco_dataset(test_roco_dataset)
+ self.train_roco_datasets = train_roco_datasets
+ self.test_roco_datasets = test_roco_datasets
+ (
+ self.train_images_paths,
+ self.train_images,
+ self.train_labels,
+ self.train_dataset_sizes,
+ ) = image_dataset_generator.from_roco_datasets(train_roco_datasets)
+ (
+ self.test_images_paths,
+ self.test_images,
+ self.test_labels,
+ self.test_dataset_sizes,
+ ) = image_dataset_generator.from_roco_datasets(test_roco_datasets)
def evaluate_pipelines(self, pipelines: Iterable[ImagePipeline]) -> Dict[str, ClassificationResults]:
return {str(pipeline): self.evaluate(pipeline) for pipeline in pipelines}
@@ -41,20 +69,16 @@ class ImagePipelineEvaluator:
pipeline.fit(self.train_images, self.train_labels)
logging.info(f"Infering")
- train_report, train_time = self._evaluate_on_set(pipeline, self.train_images, self.train_labels)
- test_report, test_time = self._evaluate_on_set(pipeline, self.test_images, self.test_labels)
+ train_results = self._evaluate_on_set(pipeline, self.train_images, self.train_labels)
+ test_results = self._evaluate_on_set(pipeline, self.test_images, self.test_labels)
return ClassificationResults(
- train_report=train_report,
- test_report=test_report,
- train_mean_inference_time=train_time,
- test_mean_inference_time=test_time,
- full_pipeline_name=repr(pipeline),
+ train_results=train_results, test_results=test_results, full_pipeline_name=repr(pipeline),
)
@staticmethod
- def _evaluate_on_set(pipeline: ImagePipeline, images: List[Image], labels: List[Any]) -> Tuple[Dict, float]:
+ def _evaluate_on_set(pipeline: ImagePipeline, images: List[Image], labels: List[Any]) -> SetClassificationResults:
t = time()
preds = pipeline.predict(images)
mean_time = (time() - t) / len(images)
- return classification_report(labels, preds, output_dict=True), mean_time
+ return SetClassificationResults(np.asarray(labels), np.asarray(preds), mean_time)
diff --git a/common/research_common/scripts/construct_dataset_from_manual_annotation.py b/common/research_common/scripts/construct_dataset_from_manual_annotation.py
index d3ab5188b70a695406761be7c49829f9e3741712..3178c9139ee0bba87e820e4859d74d3382e27306 100644
--- a/common/research_common/scripts/construct_dataset_from_manual_annotation.py
+++ b/common/research_common/scripts/construct_dataset_from_manual_annotation.py
@@ -3,8 +3,6 @@ from pathlib import Path
from shutil import move, rmtree
from zipfile import ZipFile
-from research_common.constants import TWITCH_DSET_DIR, TWITCH_ROBOTS_VIEWS_DIR, TWITCH_DSET_ROBOTS_VIEWS_DIR
-
def construct_dataset_from_manual_annotations(
source_images_directory: Path, source_annotations_directory: Path, destination_directory: Path
@@ -29,9 +27,3 @@ def _unzip_all_in_directory(source_directory: Path, destination_directory: Path,
for directory in destination_directory.glob("*"):
if directory.is_dir():
rmtree(str(directory))
-
-
-if __name__ == "__main__":
- construct_dataset_from_manual_annotations(
- TWITCH_ROBOTS_VIEWS_DIR, TWITCH_DSET_DIR / "robots-views-annotations", TWITCH_DSET_ROBOTS_VIEWS_DIR,
- )
diff --git a/common/research_common/scripts/construct_twith_datasets_from_manual_annotation.py b/common/research_common/scripts/construct_twith_datasets_from_manual_annotation.py
index 44f643434662f809a7f7d73d0b0ce18d5acd3f97..3989bba47072f03992f2b9d29b6a059c95423082 100644
--- a/common/research_common/scripts/construct_twith_datasets_from_manual_annotation.py
+++ b/common/research_common/scripts/construct_twith_datasets_from_manual_annotation.py
@@ -1,4 +1,5 @@
-from shutil import copy, move, rmtree
+from os import remove
+from shutil import copy, move, rmtree, make_archive
from research_common.constants import TWITCH_DSET_DIR, TWITCH_ROBOTS_VIEWS_DIR, TWITCH_DSET_ROBOTS_VIEWS_DIR
from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
@@ -38,8 +39,9 @@ def _separate_twitch_videos():
move(str(annotation.xml_path), str(annotations_path / annotation.xml_path.name))
if list((TWITCH_DSET_ROBOTS_VIEWS_DIR / "image").glob("*")):
raise Exception(f"Some images remains unmoved")
- if list((TWITCH_DSET_ROBOTS_VIEWS_DIR / "image_annotation").glob("*")):
- raise Exception(f"Some annotations remains unmoved")
+ for remaining_file in (TWITCH_DSET_ROBOTS_VIEWS_DIR / "image_annotation").glob("*"):
+ if remaining_file.name != ".DS_Store":
+ raise Exception(f"Some annotations remains unmoved")
rmtree(str(TWITCH_DSET_ROBOTS_VIEWS_DIR / "image"))
rmtree(str(TWITCH_DSET_ROBOTS_VIEWS_DIR / "image_annotation"))
@@ -61,6 +63,12 @@ def _make_separate_reports():
if __name__ == "__main__":
+
+ for zip_file in (TWITCH_DSET_DIR / "reviewed-robots-views-annotations").glob("*.zip"):
+ remove(str(zip_file))
+ for chunk_dir in (TWITCH_DSET_DIR / "reviewed-robots-views-annotations").glob("chunk_*"):
+ make_archive(chunk_dir, "zip", chunk_dir)
+
_construct_mixed_twitch_dset()
_correct_manual_annotations()
_extract_runes_images()
diff --git a/common/research_common/scripts/correct_annotations.py b/common/research_common/scripts/correct_annotations.py
index 2dfb1a8608a46a43e29c50040701143da1df1ea2..5b45d96fcc2167c732484a7cc4dd8808ff2af4a1 100644
--- a/common/research_common/scripts/correct_annotations.py
+++ b/common/research_common/scripts/correct_annotations.py
@@ -3,14 +3,25 @@ from pathlib import Path
class AnnotationFileCorrector:
- FINAL_ARMOR_NAME_PATTERN = re.compile(r"<name>(armor|amor)-(?P<color>\w{2,4})-(?P<num>\d)</name>", re.IGNORECASE)
- ABV_ARMOR_NAME_PATTERN = re.compile(r"<name>a(?P<color>\w)(?P<num>\d)</name>", re.IGNORECASE)
+ FINAL_ARMOR_NAME_PATTERN = re.compile(
+ r"<name>(armor|amor|amror)-(?P<color>\w{2,4})-(?P<num>\d)</name>", re.IGNORECASE
+ )
+ ABV_ARMOR_NAME_PATTERN = re.compile(r"<name>a{1,2}(?P<color>\w)(?P<num>\d)</name>", re.IGNORECASE)
ABV_RUNES_PATTERN = re.compile(r"<name>r(?P<color>\w)</name>", re.IGNORECASE)
ABV_BASE_PATTERN = re.compile(r"<name>b</name>", re.IGNORECASE)
ABV_WATCHER_PATTERN = re.compile(r"<name>w</name>", re.IGNORECASE)
- ABV_CAR_PATTERN = re.compile(r"<name>[cx]</name>", re.IGNORECASE)
-
- COLORS_MAP = {"r": "red", "red": "red", "b": "blue", "blue": "blue", "g": "grey", "grey": "grey", "gray": "grey"}
+ ABV_CAR_PATTERN = re.compile(r"<name>(c|x|robot)</name>", re.IGNORECASE)
+
+ COLORS_MAP = {
+ "r": "red",
+ "red": "red",
+ "b": "blue",
+ "bleu": "blue",
+ "blue": "blue",
+ "g": "grey",
+ "grey": "grey",
+ "gray": "grey",
+ }
def __init__(self, save_before: bool):
self.save_before = save_before
diff --git a/dataset/dji_roco/.gitignore b/dataset/dji_roco/.gitignore
index 837840cd27abb5677670c853f5c0b7bc2d2be0dd..fafbe0fec119e0d4a1e64b072d9a9b59bd033925 100644
--- a/dataset/dji_roco/.gitignore
+++ b/dataset/dji_roco/.gitignore
@@ -1,3 +1,5 @@
**/*.xml
**/*.jpg
**/*.png
+**/colors
+**/digits
diff --git a/dataset/twitch/.gitignore b/dataset/twitch/.gitignore
index 6413c17d7139a7238fa5ef7c0878ab3ba8186f77..d1462f9c9f1d0a25d6d03aabeb9dd8b1b2b2391c 100644
--- a/dataset/twitch/.gitignore
+++ b/dataset/twitch/.gitignore
@@ -1,3 +1,4 @@
/robots-views-annotations
/reviewed-robots-views-annotations
-/final-robots-views
\ No newline at end of file
+/final-robots-views
+/aerial-annotations
\ No newline at end of file
diff --git a/dataset/twitch/aerial-annotations/.gitignore b/dataset/twitch/v1/.gitignore
similarity index 100%
rename from dataset/twitch/aerial-annotations/.gitignore
rename to dataset/twitch/v1/.gitignore
diff --git a/drone-at-base/poetry.lock b/drone-at-base/poetry.lock
index 5d5ab9b3fe409ee080089ae7f272c78a9f47c25b..e8d0e4d738aeaf5eaa4438030e80abcae0c47a0b 100644
Binary files a/drone-at-base/poetry.lock and b/drone-at-base/poetry.lock differ
diff --git a/robots-at-robots/poetry.lock b/robots-at-robots/poetry.lock
index 5d5ab9b3fe409ee080089ae7f272c78a9f47c25b..e8d0e4d738aeaf5eaa4438030e80abcae0c47a0b 100644
Binary files a/robots-at-robots/poetry.lock and b/robots-at-robots/poetry.lock differ
diff --git a/robots-at-robots/research/armor_color/armor_color_pipeline_reporter_factory.py b/robots-at-robots/research/armor_color/armor_color_pipeline_reporter_factory.py
index dac836df8a0332dc044c6815b6109048ebe0bda3..d6fa16ec204c720a71e977e95492457d2e9624d1 100644
--- a/robots-at-robots/research/armor_color/armor_color_pipeline_reporter_factory.py
+++ b/robots-at-robots/research/armor_color/armor_color_pipeline_reporter_factory.py
@@ -1,16 +1,20 @@
+from typing import List
+
from research.dataset.armor_color_dataset_factory import ArmorColorDatasetGenerator
-from research_common.dataset.roco_dataset import ROCODataset
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
from research_common.image_pipeline_evaluation.image_pipeline_evaluation_reporter import ImagePipelineEvaluationReporter
from research_common.image_pipeline_evaluation.image_pipeline_evaluator import ImagePipelineEvaluator
class ArmorColorPipelineReporterFactory:
@staticmethod
- def from_roco_datasets(train_roco_dataset: ROCODataset, test_roco_dataset: ROCODataset):
+ def from_roco_datasets(
+ train_roco_datasets: List[DirectoryROCODataset], test_roco_datasets: List[DirectoryROCODataset]
+ ):
return ImagePipelineEvaluationReporter(
evaluator=ImagePipelineEvaluator(
- train_roco_dataset=train_roco_dataset,
- test_roco_dataset=test_roco_dataset,
+ train_roco_datasets=train_roco_datasets,
+ test_roco_datasets=test_roco_datasets,
image_dataset_generator=ArmorColorDatasetGenerator(),
),
evaluation_project="armor-color",
diff --git a/robots-at-robots/research/armor_color/baseline_experiments.py b/robots-at-robots/research/armor_color/baseline_experiments.py
index d85243d92d69e710cb0610ef8076c27a32c4e3f9..87dfe7290c0aea85fb50125d6afb67543825b447 100644
--- a/robots-at-robots/research/armor_color/baseline_experiments.py
+++ b/robots-at-robots/research/armor_color/baseline_experiments.py
@@ -6,14 +6,13 @@ from polystar.common.image_pipeline.models.random_model import RandomModel
from polystar.common.image_pipeline.models.red_blue_channels_comparison_model import RedBlueComparisonModel
from research.armor_color.armor_color_pipeline_reporter_factory import ArmorColorPipelineReporterFactory
from research_common.dataset.twitch.twitch_roco_datasets import TwitchROCODataset
-from research_common.dataset.union_dataset import UnionDataset
if __name__ == "__main__":
logging.getLogger().setLevel("INFO")
reporter = ArmorColorPipelineReporterFactory.from_roco_datasets(
- train_roco_dataset=UnionDataset(TwitchROCODataset.TWITCH_470151286, TwitchROCODataset.TWITCH_470150052),
- test_roco_dataset=TwitchROCODataset.TWITCH_470152289,
+ train_roco_datasets=[TwitchROCODataset.TWITCH_470151286, TwitchROCODataset.TWITCH_470150052],
+ test_roco_datasets=[TwitchROCODataset.TWITCH_470152289],
)
red_blue_comparison_pipeline = ImagePipeline(
diff --git a/robots-at-robots/research/armor_digit/armor_digit_pipeline_reporter_factory.py b/robots-at-robots/research/armor_digit/armor_digit_pipeline_reporter_factory.py
index 7e8afa1d6955d7022d21d7b12db5b7c2c8aa5e4e..5546f27d39414036d1d7290564acb9a26c03f9b4 100644
--- a/robots-at-robots/research/armor_digit/armor_digit_pipeline_reporter_factory.py
+++ b/robots-at-robots/research/armor_digit/armor_digit_pipeline_reporter_factory.py
@@ -1,7 +1,7 @@
-from typing import Iterable
+from typing import Iterable, List
from research.dataset.armor_digit_dataset_factory import ArmorDigitDatasetGenerator
-from research_common.dataset.roco_dataset import ROCODataset
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
from research_common.image_pipeline_evaluation.image_pipeline_evaluation_reporter import ImagePipelineEvaluationReporter
from research_common.image_pipeline_evaluation.image_pipeline_evaluator import ImagePipelineEvaluator
@@ -9,14 +9,14 @@ from research_common.image_pipeline_evaluation.image_pipeline_evaluator import I
class ArmorDigitPipelineReporterFactory:
@staticmethod
def from_roco_datasets(
- train_roco_dataset: ROCODataset,
- test_roco_dataset: ROCODataset,
+ train_roco_datasets: List[DirectoryROCODataset],
+ test_roco_datasets: List[DirectoryROCODataset],
acceptable_digits: Iterable[int] = (1, 2, 3, 4, 5, 7),
):
return ImagePipelineEvaluationReporter(
evaluator=ImagePipelineEvaluator(
- train_roco_dataset=train_roco_dataset,
- test_roco_dataset=test_roco_dataset,
+ train_roco_datasets=train_roco_datasets,
+ test_roco_datasets=test_roco_datasets,
image_dataset_generator=ArmorDigitDatasetGenerator(set(acceptable_digits)),
),
evaluation_project="armor-digit",
diff --git a/robots-at-robots/research/armor_digit/baseline_experiments.py b/robots-at-robots/research/armor_digit/baseline_experiments.py
index 0b7c979881f725679f8d5cfb7ce62f308b6897de..8b51e3991dc9d30f0e5043fcaa97bb16746e16d3 100644
--- a/robots-at-robots/research/armor_digit/baseline_experiments.py
+++ b/robots-at-robots/research/armor_digit/baseline_experiments.py
@@ -24,8 +24,8 @@ N_CLASSES = 9 # there is no 6s or 9s
# logging.getLogger().setLevel("INFO")
#
# reporter = ArmorDigitPipelineReporterFactory.from_roco_datasets(
-# train_roco_dataset=UnionDataset(TwitchROCODataset.TWITCH_470151286, TwitchROCODataset.TWITCH_470150052),
-# test_roco_dataset=TwitchROCODataset.TWITCH_470152289,
+# train_roco_datasets=[TwitchROCODataset.TWITCH_470151286, TwitchROCODataset.TWITCH_470150052],
+# test_roco_datasets=[TwitchROCODataset.TWITCH_470152289],
# )
#
# random_pipeline = ImagePipeline(model=RandomModel(), custom_name="random")
diff --git a/robots-at-robots/research/dataset/armor_color_dataset_factory.py b/robots-at-robots/research/dataset/armor_color_dataset_factory.py
index c32fe9820beb7df24861a6cb15de28e1b397dea8..c601fa168777f24dc93832706e6470836c7821b6 100644
--- a/robots-at-robots/research/dataset/armor_color_dataset_factory.py
+++ b/robots-at-robots/research/dataset/armor_color_dataset_factory.py
@@ -1,11 +1,11 @@
-from typing import Tuple, Sequence
+from pathlib import Path
-from polystar.common.models.image import Image
-from research.dataset.armor_dataset_factory import ArmorDatasetFactory
-from research_common.dataset.roco_dataset import ROCODataset
-from research_common.image_pipeline_evaluation.image_dataset_generator import ImageDatasetGenerator
+from polystar.common.models.object import Armor
+from research.dataset.armor_image_dataset_factory import ArmorImageDatasetGenerator
-class ArmorColorDatasetGenerator(ImageDatasetGenerator[str]):
- def from_roco_dataset(self, dataset: ROCODataset) -> Tuple[Sequence[Image], Sequence[str]]:
- return zip(*[(armor_img, c.name) for (armor_img, c, n, k, p) in ArmorDatasetFactory.from_dataset(dataset)])
+class ArmorColorDatasetGenerator(ArmorImageDatasetGenerator[str]):
+ task_name: str = "colors"
+
+ def _label_from_armor_info(self, armor: Armor, k: int, path: Path) -> str:
+ return armor.color.name
diff --git a/robots-at-robots/research/dataset/armor_dataset_factory.py b/robots-at-robots/research/dataset/armor_dataset_factory.py
index 75688971c9851a5dd321374a8491e1f69154576a..138de0d526bc80fa925aaaa7b01cefa07fb814e5 100644
--- a/robots-at-robots/research/dataset/armor_dataset_factory.py
+++ b/robots-at-robots/research/dataset/armor_dataset_factory.py
@@ -6,7 +6,7 @@ import matplotlib.pyplot as plt
from polystar.common.models.image import Image
from polystar.common.models.image_annotation import ImageAnnotation
-from polystar.common.models.object import ObjectType, Armor, ArmorColor, ArmorNumber
+from polystar.common.models.object import ObjectType, Armor
from polystar.common.target_pipeline.objects_validators.type_object_validator import TypeObjectValidator
from research_common.dataset.dji.dji_roco_datasets import DJIROCODataset
from research_common.dataset.roco_dataset import ROCODataset
@@ -14,36 +14,38 @@ from research_common.dataset.roco_dataset import ROCODataset
class ArmorDatasetFactory:
@staticmethod
- def from_image_annotation(
- image_annotation: ImageAnnotation,
- ) -> Iterable[Tuple[Image, ArmorColor, ArmorNumber, int, Path]]:
+ def from_image_annotation(image_annotation: ImageAnnotation) -> Iterable[Tuple[Image, Armor, int, Path]]:
img = image_annotation.image
armors: List[Armor] = TypeObjectValidator(ObjectType.Armor).filter(image_annotation.objects, img)
for i, obj in enumerate(armors):
croped_img = img[obj.y: obj.y + obj.h, obj.x: obj.x + obj.w]
- yield croped_img, obj.color, obj.numero, i, image_annotation.image_path
+ yield croped_img, obj, i, image_annotation.image_path
@staticmethod
- def from_dataset(dataset: ROCODataset) -> Iterable[Tuple[Image, ArmorColor, ArmorNumber, int, Path]]:
+ def from_dataset(dataset: ROCODataset) -> Iterable[Tuple[Image, Armor, int, Path]]:
for image_annotation in dataset.image_annotations:
for rv in ArmorDatasetFactory.from_image_annotation(image_annotation):
yield rv
def save_digits_img():
- for j, (digit_img, c, n, k, p) in enumerate(ArmorDatasetFactory.from_dataset(DJIROCODataset.Final)):
+ for j, (digit_img, armor, k, p) in enumerate(ArmorDatasetFactory.from_dataset(DJIROCODataset.Final)):
# we ignore the 6s, because the format of their pictures is not accurate and we don't need them
- if n != 6 and j >= 5925: # TODO change j value depending where you are in saving (to save time)
+ if armor.numero != 6 and j >= 5925: # TODO change j value depending where you are in saving (to save time)
# if j >= 50:
# break
+ # TODO avoid getting image with chinese symbols in their path
try:
- print(c, n, k, 'in', p)
+ print(armor.color, armor.numero, k, 'in', p)
plt.imshow(digit_img)
- plt.savefig(str(p.parents[1]) + '\\digits_found\\digit' + str(j) + '_' + c.name + '_' + str(n) + '.png')
+ plt.savefig(str(p.parents[1]) + '\\digits_found\\digit' + str(j) + '_' + armor.color.name +
+ '_' + str(armor.numero) + '.png')
plt.clf()
except ValueError:
# because some digit's pictures have zero-array values
continue
+ if j >= 5940:
+ break
if __name__ == "__main__":
diff --git a/robots-at-robots/research/dataset/armor_digit_dataset_factory.py b/robots-at-robots/research/dataset/armor_digit_dataset_factory.py
index d7f0b938b623b56959cf8c43777e7954973d2377..a38d4a14f9c48447515ff501353c051c117fabc5 100644
--- a/robots-at-robots/research/dataset/armor_digit_dataset_factory.py
+++ b/robots-at-robots/research/dataset/armor_digit_dataset_factory.py
@@ -1,20 +1,21 @@
-from typing import Tuple, Sequence, Set
+from pathlib import Path
+from typing import Set
-from polystar.common.models.image import Image
-from research.dataset.armor_dataset_factory import ArmorDatasetFactory
-from research_common.dataset.roco_dataset import ROCODataset
-from research_common.image_pipeline_evaluation.image_dataset_generator import ImageDatasetGenerator
+from polystar.common.models.object import Armor
+from research.dataset.armor_image_dataset_factory import ArmorImageDatasetGenerator
-class ArmorDigitDatasetGenerator(ImageDatasetGenerator[str]):
+class ArmorDigitDatasetGenerator(ArmorImageDatasetGenerator[int]):
+ task_name: str = "digits"
+
def __init__(self, acceptable_digits: Set[int]):
self.acceptable_digits = acceptable_digits
- def from_roco_dataset(self, dataset: ROCODataset) -> Tuple[Sequence[Image], Sequence[int]]:
- return zip(
- *[
- (armor_img, digit)
- for (armor_img, c, digit, k, p) in ArmorDatasetFactory.from_dataset(dataset)
- if digit in self.acceptable_digits
- ]
- )
+ def _label_from_str(self, label: str) -> int:
+ return int(label)
+
+ def _label_from_armor_info(self, armor: Armor, k: int, path: Path) -> int:
+ return armor.numero
+
+ def _valid_label(self, label: int) -> bool:
+ return label in self.acceptable_digits
diff --git a/robots-at-robots/research/dataset/armor_image_dataset_factory.py b/robots-at-robots/research/dataset/armor_image_dataset_factory.py
new file mode 100644
index 0000000000000000000000000000000000000000..dfdf35b8ec6ea55e0cffc1c6f9458a69e7190109
--- /dev/null
+++ b/robots-at-robots/research/dataset/armor_image_dataset_factory.py
@@ -0,0 +1,54 @@
+import json
+from abc import abstractmethod
+from pathlib import Path
+from typing import TypeVar, Tuple, Iterable
+
+import cv2
+
+from polystar.common.models.image import Image
+from polystar.common.models.object import Armor
+from polystar.common.utils.time import create_time_id
+from research.dataset.armor_dataset_factory import ArmorDatasetFactory
+from research_common.dataset.directory_roco_dataset import DirectoryROCODataset
+from research_common.image_pipeline_evaluation.image_dataset_generator import ImageDatasetGenerator
+
+T = TypeVar("T")
+
+
+class ArmorImageDatasetGenerator(ImageDatasetGenerator[T]):
+ task_name: str
+
+ def from_roco_dataset(self, dataset: DirectoryROCODataset) -> Iterable[Tuple[Image, T]]:
+ if not (dataset.dataset_path / self.task_name / ".lock").exists():
+ self._create_labelized_armor_images_from_roco(dataset)
+ return self._get_images_paths_and_labels(dataset)
+
+ def _create_labelized_armor_images_from_roco(self, dataset):
+ dset_path = dataset.dataset_path / self.task_name
+ dset_path.mkdir(exist_ok=True)
+ for (armor_img, armor, k, path) in ArmorDatasetFactory.from_dataset(dataset):
+ label = self._label_from_armor_info(armor, k, path)
+ cv2.imwrite(str(dset_path / f"{path.stem}-{k}-{label}.jpg"), cv2.cvtColor(armor_img, cv2.COLOR_RGB2BGR))
+ (dataset.dataset_path / self.task_name / ".lock").write_text(
+ json.dumps({"version": "0.0", "date": create_time_id()})
+ )
+
+ def _get_images_paths_and_labels(self, dataset: DirectoryROCODataset) -> Iterable[Tuple[Image, T]]:
+ return (
+ (image_path, self._label_from_filepath(image_path))
+ for image_path in (dataset.dataset_path / self.task_name).glob("*.jpg")
+ if self._valid_label(self._label_from_filepath(image_path))
+ )
+
+ def _label_from_filepath(self, image_path: Path) -> T:
+ return self._label_from_str(image_path.stem.split("-")[-1])
+
+ @abstractmethod
+ def _label_from_armor_info(self, armor: Armor, k: int, path: Path) -> T:
+ pass
+
+ def _valid_label(self, label: T) -> bool:
+ return True
+
+ def _label_from_str(self, label: str) -> T:
+ return label
diff --git a/robots-at-runes/poetry.lock b/robots-at-runes/poetry.lock
index 5d5ab9b3fe409ee080089ae7f272c78a9f47c25b..8c2d89db08a86cb509e2a2a9ecb4c6a68f847449 100644
Binary files a/robots-at-runes/poetry.lock and b/robots-at-runes/poetry.lock differ
diff --git a/robots-at-runes/pyproject.toml b/robots-at-runes/pyproject.toml
index 27e06de40cb50160e76e6991e4a4416e204bc476..3e9905b88674270ea3dbcb63f1ea3b121aa87fb8 100644
--- a/robots-at-runes/pyproject.toml
+++ b/robots-at-runes/pyproject.toml
@@ -8,6 +8,7 @@ packages = [{ include = "polystar" }]
[tool.poetry.dependencies]
python = "^3.7"
"polystar.common" = {path = "../common"}
+imutils = "^0.5.3"
[tool.poetry.dev-dependencies]
pytest = "^4.5"
diff --git a/robots-at-runes/research/constants.py b/robots-at-runes/research/dataset/__init__.py
similarity index 100%
rename from robots-at-runes/research/constants.py
rename to robots-at-runes/research/dataset/__init__.py
diff --git a/robots-at-runes/research/dataset/blend/__init__.py b/robots-at-runes/research/dataset/blend/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/robots-at-runes/research/dataset/blend/examples/.gitignore b/robots-at-runes/research/dataset/blend/examples/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..a13af165695e32f57c6c31c7a7e2c9f33ddf3881
--- /dev/null
+++ b/robots-at-runes/research/dataset/blend/examples/.gitignore
@@ -0,0 +1,2 @@
+/image
+/image_annotation
\ No newline at end of file
diff --git a/robots-at-runes/research/dataset/blend/examples/back1.jpg b/robots-at-runes/research/dataset/blend/examples/back1.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..59b770b853a95074c58a2c7d854d69efc787d4a5
Binary files /dev/null and b/robots-at-runes/research/dataset/blend/examples/back1.jpg differ
diff --git a/robots-at-runes/research/dataset/blend/examples/logo.png b/robots-at-runes/research/dataset/blend/examples/logo.png
new file mode 100644
index 0000000000000000000000000000000000000000..81182558975026552d3da02c9da45e0d65abe6b9
Binary files /dev/null and b/robots-at-runes/research/dataset/blend/examples/logo.png differ
diff --git a/robots-at-runes/research/dataset/blend/examples/logo.xml b/robots-at-runes/research/dataset/blend/examples/logo.xml
new file mode 100644
index 0000000000000000000000000000000000000000..b3d47c380c48ee78c5a264568ed62ee8271f5ec7
--- /dev/null
+++ b/robots-at-runes/research/dataset/blend/examples/logo.xml
@@ -0,0 +1,12 @@
+<annotation>
+ <point>
+ <x>432</x>
+ <y>76</y>
+ <label>green</label>
+ </point>
+ <point>
+ <x>432</x>
+ <y>13</y>
+ <label>blue</label>
+ </point>
+</annotation>
diff --git a/robots-at-runes/research/dataset/blend/image_blender.py b/robots-at-runes/research/dataset/blend/image_blender.py
new file mode 100644
index 0000000000000000000000000000000000000000..c19202e3d159e3298ede9df9695cf388a5f20c7f
--- /dev/null
+++ b/robots-at-runes/research/dataset/blend/image_blender.py
@@ -0,0 +1,96 @@
+from dataclasses import dataclass
+from random import random
+from typing import List, Tuple
+
+import cv2
+import numpy as np
+
+from polystar.common.models.image import Image
+from research.dataset.blend.labeled_image_modifiers.labeled_image_modifier_abc import LabeledImageModifierABC
+from research.dataset.labeled_image import LabeledImage, PointOfInterest
+
+
+@dataclass
+class ImageBlender:
+ background_size: Tuple[int, int]
+ object_modifiers: List[LabeledImageModifierABC]
+
+ def blend(self, background: Image, obj: LabeledImage) -> LabeledImage:
+ obj = self._modify_object(obj)
+ background = self._crop_background(background)
+ x, y = self._generate_position_of_object(background.shape, obj.image.shape)
+ return LabeledImage(
+ image=self._blend_obj_on_background(background, obj.image, x, y),
+ point_of_interests=[self._translate_poi(poi, x, y) for poi in obj.point_of_interests],
+ )
+
+ def _modify_object(self, obj: LabeledImage) -> LabeledImage:
+ for modifier in self.object_modifiers:
+ obj = modifier.randomly_modify(obj)
+ return obj
+
+ def _generate_position_of_object(
+ self, background_shape: Tuple[int, int, int], obj_shape: Tuple[int, int, int]
+ ) -> Tuple[int, int]:
+ return (
+ self._generate_position_of_object_amoung_axis(background_shape[1], obj_shape[1]),
+ self._generate_position_of_object_amoung_axis(background_shape[0], obj_shape[0]),
+ )
+
+ @staticmethod
+ def _generate_position_of_object_amoung_axis(background_size: int, obj_size: int) -> int:
+ return int(random() * (background_size - obj_size))
+
+ def _blend_obj_on_background(self, background: Image, obj_img: Image, x: int, y: int) -> Image:
+ background_roi = background[y : y + obj_img.shape[0], x : x + obj_img.shape[1], :]
+ mask = obj_img[:, :, 3]
+ obj_img = cv2.cvtColor(obj_img, cv2.COLOR_RGBA2RGB)
+ background_roi = background_roi.astype(np.float)
+ obj_img = obj_img.astype(np.float)
+ rv = background.copy()
+ for i in range(3):
+ rv[y : y + obj_img.shape[0], x : x + obj_img.shape[1], i] = (
+ (~mask * background_roi[:, :, i] + mask * obj_img[:, :, i]) / 255
+ ).astype(np.uint8)
+ return rv
+
+ @staticmethod
+ def _translate_poi(poi: PointOfInterest, x: int, y: int) -> PointOfInterest:
+ return PointOfInterest(poi.x + x, poi.y + y, poi.label)
+
+ def _crop_background(self, background: Image) -> Image:
+ h, w, _ = background.shape
+ x, y = int(random() * (h - self.background_size[1])), int(random() * (w - self.background_size[0]))
+ return background[y : y + self.background_size[1], x : x + self.background_size[0], :]
+
+
+if __name__ == "__main__":
+ from pathlib import Path
+
+ import matplotlib.pyplot as plt
+
+ from research.dataset.blend.labeled_image_modifiers.labeled_image_rotator import LabeledImageRotator
+ from research.dataset.blend.labeled_image_modifiers.labeled_image_scaler import LabeledImageScaler
+
+ EXAMPLES_DIR = Path(__file__).parent / "examples"
+
+ _obj = LabeledImage(
+ cv2.cvtColor(cv2.imread(str(EXAMPLES_DIR / "logo.png"), cv2.IMREAD_UNCHANGED), cv2.COLOR_BGRA2RGBA),
+ PointOfInterest.from_annotation_file(EXAMPLES_DIR / "logo.xml"),
+ )
+ _bg = cv2.cvtColor(cv2.imread(str(EXAMPLES_DIR / "back1.jpg")), cv2.COLOR_BGR2RGB)
+
+ _blender = ImageBlender(
+ background_size=(1_280, 720), object_modifiers=[LabeledImageScaler(1.5), LabeledImageRotator(180)]
+ )
+ for i in range(10):
+ res = _blender.blend(_bg, _obj)
+
+ res.save(EXAMPLES_DIR, f"test_{i}")
+
+ plt.imshow(res.image)
+ for poi in res.point_of_interests:
+ plt.plot([poi.x], [poi.y], f"{poi.label[0]}.")
+ plt.axis("off")
+ plt.tight_layout()
+ plt.show()
diff --git a/robots-at-runes/research/dataset/blend/labeled_image_modifiers/__init__.py b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_modifier_abc.py b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_modifier_abc.py
new file mode 100644
index 0000000000000000000000000000000000000000..e3ab2e7f4c7bd4bec333f24a3267e48b02b639c6
--- /dev/null
+++ b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_modifier_abc.py
@@ -0,0 +1,40 @@
+from abc import ABC, abstractmethod
+from random import random
+
+from polystar.common.models.image import Image
+from research.dataset.labeled_image import LabeledImage, PointOfInterest
+
+
+class LabeledImageModifierABC(ABC):
+ def randomly_modify(self, labeled_image: LabeledImage) -> LabeledImage:
+ return self.modify_from_factor(labeled_image, random() * 2 - 1)
+
+ def modify_from_factor(self, labeled_image: LabeledImage, factor: float) -> LabeledImage:
+ return self.modify_from_value(labeled_image, self._get_value_from_factor(factor))
+
+ def modify_from_value(self, labeled_image: LabeledImage, value: float) -> LabeledImage:
+ new_image = self._generate_modified_image(labeled_image.image, value)
+ return LabeledImage(
+ image=new_image,
+ point_of_interests=[
+ self._generate_modified_poi(poi, labeled_image.image, new_image, value)
+ for poi in labeled_image.point_of_interests
+ ],
+ )
+
+ @abstractmethod
+ def _get_value_from_factor(self, factor: float) -> float:
+ """
+ :param factor: a factor of modification, in range [-1, 1]
+ :return: the value of modification used by other function of this class
+ """
+
+ @abstractmethod
+ def _generate_modified_image(self, image: Image, value: float) -> Image:
+ pass
+
+ @abstractmethod
+ def _generate_modified_poi(
+ self, poi: PointOfInterest, original_image: Image, new_image: Image, value: float
+ ) -> PointOfInterest:
+ pass
diff --git a/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_rotator.py b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_rotator.py
new file mode 100644
index 0000000000000000000000000000000000000000..860972cbf0cc10519f982ef1a799c342bfc43361
--- /dev/null
+++ b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_rotator.py
@@ -0,0 +1,34 @@
+from dataclasses import dataclass
+
+import cv2
+import numpy as np
+from imutils import rotate_bound
+
+from polystar.common.models.image import Image
+from research.dataset.blend.labeled_image_modifiers.labeled_image_modifier_abc import LabeledImageModifierABC
+from research.dataset.labeled_image import PointOfInterest
+
+
+@dataclass
+class LabeledImageRotator(LabeledImageModifierABC):
+ max_angle_degrees: float
+
+ def _generate_modified_image(self, image: Image, angle: float) -> Image:
+ return rotate_bound(image, angle)
+
+ def _generate_modified_poi(
+ self, poi: PointOfInterest, original_image: Image, new_image: Image, angle_degrees: float
+ ) -> PointOfInterest:
+ angle_rads = np.deg2rad(angle_degrees)
+ sin, cos = np.sin(angle_rads), np.cos(angle_rads)
+ rotation_matrix = np.array([[cos, -sin], [sin, cos]])
+ prev_vector_to_center = np.array((poi.x - original_image.shape[1] / 2, poi.y - original_image.shape[0] / 2))
+ new_vector_to_center = np.dot(rotation_matrix, prev_vector_to_center)
+ return PointOfInterest(
+ int(new_vector_to_center[0] + new_image.shape[1] / 2),
+ int(new_vector_to_center[1] + new_image.shape[0] / 2),
+ poi.label,
+ )
+
+ def _get_value_from_factor(self, factor: float) -> float:
+ return self.max_angle_degrees * factor
diff --git a/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_scaler.py b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_scaler.py
new file mode 100644
index 0000000000000000000000000000000000000000..b98e25f4fa1a32e4197984864fe887dd7690b024
--- /dev/null
+++ b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_scaler.py
@@ -0,0 +1,28 @@
+from dataclasses import dataclass
+
+import cv2
+
+from polystar.common.models.image import Image
+from research.dataset.blend.labeled_image_modifiers.labeled_image_modifier_abc import LabeledImageModifierABC
+from research.dataset.labeled_image import LabeledImage, PointOfInterest
+
+
+@dataclass
+class LabeledImageScaler(LabeledImageModifierABC):
+ max_scale: float
+
+ def _generate_modified_image(self, image: Image, scale: float) -> Image:
+ return cv2.resize(
+ image, (int(image.shape[1] * scale), int(image.shape[0] * scale)), interpolation=cv2.INTER_AREA
+ )
+
+ def _generate_modified_poi(
+ self, poi: PointOfInterest, original_image: Image, new_image: Image, scale: float
+ ) -> PointOfInterest:
+ return PointOfInterest(int(poi.x * scale), int(poi.y * scale), poi.label)
+
+ def _get_value_from_factor(self, factor: float) -> float:
+ intensity = (self.max_scale - 1) * abs(factor) + 1
+ if factor > 0:
+ return intensity
+ return 1 / intensity
diff --git a/robots-at-runes/research/dataset/labeled_image.py b/robots-at-runes/research/dataset/labeled_image.py
new file mode 100644
index 0000000000000000000000000000000000000000..30d0dac6880f554786789cff8364ccac0d93a663
--- /dev/null
+++ b/robots-at-runes/research/dataset/labeled_image.py
@@ -0,0 +1,61 @@
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import List, Dict, Any
+from xml.dom.minidom import parseString
+
+import xmltodict
+from dicttoxml import dicttoxml
+
+from polystar.common.models.image import Image
+
+
+@dataclass
+class PointOfInterest:
+ x: int
+ y: int
+ label: str
+
+ def to_dict(self) -> Dict[str, int]:
+ return self.__dict__
+
+ @classmethod
+ def from_dict(cls, d: Dict[str, Any]):
+ return cls(x=int(d["x"]), y=int(d["y"]), label=d["label"])
+
+ @classmethod
+ def from_annotation_file(cls, annotation_path: Path) -> List["PointOfInterest"]:
+ points = xmltodict.parse(annotation_path.read_text())["annotation"]["point"]
+ return [PointOfInterest.from_dict(p) for p in points]
+
+
+@dataclass
+class LabeledImage:
+ image: Image
+ point_of_interests: List[PointOfInterest] = field(default_factory=list)
+
+ def save(self, directory_path: Path, name: str):
+ Image.save(self.image, directory_path / "image" / f"{name}.jpg")
+ self._save_annotation(directory_path / "image_annotation" / f"{name}.xml")
+
+ def _save_annotation(self, annotation_path: Path):
+ annotation_path.parent.mkdir(exist_ok=True, parents=True)
+ xml = parseString(
+ dicttoxml(
+ {"annotation": {"point": [p.to_dict() for p in self.point_of_interests]}},
+ attr_type=False,
+ root="annotation",
+ item_func=lambda x: x,
+ )
+ .replace(b"<point><point>", b"<point>")
+ .replace(b"</point></point>", b"</point>")
+ ).toprettyxml()
+ annotation_path.write_text(xml)
+
+ @staticmethod
+ def from_directory(directory_path: Path, name: str) -> "LabeledImage":
+ return LabeledImage(
+ image=Image.from_path(directory_path / "image" / f"{name}.jpg"),
+ point_of_interests=PointOfInterest.from_annotation_file(
+ directory_path / "image_annotation" / f"{name}.xml"
+ ),
+ )