From 89a15de3f229a95deeb1718bd809469882508c2b Mon Sep 17 00:00:00 2001
From: Mathieu Beligon <mathieu@feedly.com>
Date: Sat, 28 Mar 2020 22:28:30 -0400
Subject: [PATCH] [runes] (blender) Switch to a class based framework
---
robots-at-runes/research/dataset/__init__.py | 0
.../research/dataset/blend/__init__.py | 0
.../research/dataset/blend/blend.py | 148 ------------------
.../research/dataset/blend/image_blender.py | 82 ++++++++++
.../blend/labeled_image_modifiers/__init__.py | 0
.../labeled_image_modifier_abc.py | 40 +++++
.../research/dataset/labeled_image.py | 16 ++
7 files changed, 138 insertions(+), 148 deletions(-)
create mode 100644 robots-at-runes/research/dataset/__init__.py
create mode 100644 robots-at-runes/research/dataset/blend/__init__.py
delete mode 100644 robots-at-runes/research/dataset/blend/blend.py
create mode 100644 robots-at-runes/research/dataset/blend/image_blender.py
create mode 100644 robots-at-runes/research/dataset/blend/labeled_image_modifiers/__init__.py
create mode 100644 robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_modifier_abc.py
create mode 100644 robots-at-runes/research/dataset/labeled_image.py
diff --git a/robots-at-runes/research/dataset/__init__.py b/robots-at-runes/research/dataset/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/robots-at-runes/research/dataset/blend/__init__.py b/robots-at-runes/research/dataset/blend/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/robots-at-runes/research/dataset/blend/blend.py b/robots-at-runes/research/dataset/blend/blend.py
deleted file mode 100644
index dc16489..0000000
--- a/robots-at-runes/research/dataset/blend/blend.py
+++ /dev/null
@@ -1,148 +0,0 @@
-import random
-import xml.etree.ElementTree as ET
-from pathlib import Path
-from xml.dom.minidom import parseString
-
-import cv2
-import matplotlib.pyplot as plt
-import numpy as np
-from imutils import rotate_bound
-
-EXAMPLES_DIR = Path(__file__).parent / "examples"
-
-# Facteur de redimensionnement. La taille de l'image sera divisée ou multipliée par ce chiffre au maximum
-FACT_RESIZE = 1
-
-# Max angle in degrees for random rotation
-MAX_ANGLE = 20
-
-
-def preprocess_background(path_back):
- background = cv2.imread(path_back)
- return cv2.cvtColor(background, cv2.COLOR_BGR2RGB)
-
-
-def preprocess_sticker(path_item):
- item = cv2.imread(path_item, cv2.IMREAD_UNCHANGED)
- return cv2.cvtColor(item, cv2.COLOR_BGRA2RGBA)
-
-
-def generate_one(
- background, item, rotate=True, scale=True, custom_rotate=None, custom_scale=None, to_print=False, save_name=None
-):
- if rotate:
- if custom_rotate is None:
- percent_rotate = np.round((2 * random.random()) - 1, 3) # entre -1 et 1
- else:
- percent_rotate = custom_rotate
- item = rotate_percentage(item, percent_rotate)
- if scale:
- if custom_scale is None:
- percent_scale = np.round((2 * random.random()) - 1, 3) # entre -1 et 1
- else:
- percent_scale = custom_scale
- item = reshape_percentage(item, percent_scale)
- mask_alpha = item[:, :, 3]
- item = cv2.cvtColor(item, cv2.COLOR_RGBA2RGB)
- hs, ws, _ = item.shape
-
- background_subset, h_start, w_start = get_subset_shapes(background, hs, ws)
- composition_subset = superimpose(background_subset, item, mask_alpha)
-
- composition = background.copy()
- composition[h_start : h_start + hs, w_start : w_start + ws, :] = composition_subset
-
- labels = [h_start, w_start, h_start + hs, w_start + ws]
-
- if not (save_name is None):
- cv2.imwrite(save_name, cv2.cvtColor(composition, cv2.COLOR_RGB2BGR))
-
- if to_print:
- plt.imshow(composition)
- plt.show()
-
- return composition, labels
-
-
-def superimpose(img1, img2, mask):
- """
- Superimpose two pictures based on a mask. The two pictures
- must have the same shape.
- :param img1: first picture
- :param img2: second picture
- :param mask: mask applied on both pictures. Values between 0 and 255
- :return: Composition
- """
- dst_shape = img1.shape
- img1f = img1.astype(np.float)
- img2f = img2.astype(np.float)
- mix = np.zeros(dst_shape, dtype=np.uint8)
- for i in range(dst_shape[2]):
- mix[:, :, i] = ((~mask * img1f[:, :, i] + mask * img2f[:, :, i]) / 255).astype(np.uint8)
- return mix
-
-
-def get_subset_shapes(img_extract, hs, ws):
- he, we, _ = img_extract.shape
- delta_h = he - hs
- delta_w = we - ws
- h_start = int(random.random() * delta_h)
- w_start = int(random.random() * delta_w)
- return img_extract[h_start : h_start + hs, w_start : w_start + ws, :], h_start, w_start
-
-
-def reshape_percentage(img_base, percent):
- intensity = 1 + abs(percent) * FACT_RESIZE
- h, w, _ = img_base.shape
- if percent < 0:
- h_dest, w_dest = int(h / intensity), int(w / intensity)
- else:
- h_dest, w_dest = h * int(intensity), w * int(intensity)
-
- return cv2.resize(img_base, (w_dest, h_dest), interpolation=cv2.INTER_AREA)
-
-
-def rotate_percentage(img_base, percent):
- angle = MAX_ANGLE * percent
- return rotate_bound(img_base, angle)
-
-
-path_background = EXAMPLES_DIR / "back1.jpg"
-path_sticker = EXAMPLES_DIR / "logo.png"
-
-background = preprocess_background(str(path_background))
-sticker = preprocess_sticker(str(path_sticker))
-
-labels = []
-filenames = []
-for i in range(10):
- folder = EXAMPLES_DIR
- filename = "image_" + str(i) + ".jpg"
- filenames.append(filename)
- _, label = generate_one(background, sticker, to_print=False, save_name=str(folder / filename))
- labels.append(label)
-
-
-data = ET.Element("annotations")
-
-for i, [xmin, ymin, xmax, ymax] in enumerate(labels):
- object = ET.SubElement(data, "object")
- sub_name = ET.SubElement(object, "filename")
- sub_xmin = ET.SubElement(object, "xmin")
- sub_ymin = ET.SubElement(object, "ymin")
- sub_xmax = ET.SubElement(object, "xmax")
- sub_ymax = ET.SubElement(object, "ymax")
- sub_name.text = filenames[i]
- sub_xmin.text = str(xmin)
- sub_ymin.text = str(ymin)
- sub_xmax.text = str(xmax)
- sub_ymax.text = str(ymax)
-
-dom = parseString(ET.tostring(data).decode("utf-8"))
-pretty_xml = dom.toprettyxml()
-myfile = EXAMPLES_DIR / "labels.xml"
-myfile.write_text(pretty_xml)
-
-# with open('dataset/labels.txt', 'w') as f:
-# for label in labels:
-# f.write("%s\n" % label)
diff --git a/robots-at-runes/research/dataset/blend/image_blender.py b/robots-at-runes/research/dataset/blend/image_blender.py
new file mode 100644
index 0000000..57452da
--- /dev/null
+++ b/robots-at-runes/research/dataset/blend/image_blender.py
@@ -0,0 +1,82 @@
+from dataclasses import dataclass
+from random import random
+from typing import List, Tuple
+
+import cv2
+import numpy as np
+
+from polystar.common.models.image import Image
+from research.dataset.blend.labeled_image_modifiers.labeled_image_modifier_abc import LabeledImageModifierABC
+from research.dataset.labeled_image import LabeledImage, PointOfInterest
+
+
+@dataclass
+class ImageBlender:
+ modifiers: List[LabeledImageModifierABC]
+
+ def blend(self, background: Image, obj: LabeledImage) -> LabeledImage:
+ obj = self._modify_object(obj)
+ x, y = self._generate_position_of_object(background.shape, obj.image.shape)
+ return LabeledImage(
+ image=self._blend_obj_on_background(background, obj.image, x, y),
+ point_of_interests=[self._translate_poi(poi, x, y) for poi in obj.point_of_interests],
+ )
+
+ def _modify_object(self, obj: LabeledImage) -> LabeledImage:
+ for modifier in self.modifiers:
+ obj = modifier.randomly_modify(obj)
+ return obj
+
+ def _generate_position_of_object(
+ self, background_shape: Tuple[int, int, int], obj_shape: Tuple[int, int, int]
+ ) -> Tuple[int, int]:
+ return (
+ self._generate_position_of_object_amoung_axis(background_shape[1], obj_shape[1]),
+ self._generate_position_of_object_amoung_axis(background_shape[0], obj_shape[0]),
+ )
+
+ @staticmethod
+ def _generate_position_of_object_amoung_axis(background_size: int, obj_size: int) -> int:
+ return int(random() * (background_size - obj_size))
+
+ def _blend_obj_on_background(self, background: Image, obj_img: Image, x: int, y: int) -> Image:
+ background_roi = background[y : y + obj_img.shape[0], x : x + obj_img.shape[1], :]
+ mask = obj_img[:, :, 3]
+ obj_img = cv2.cvtColor(obj_img, cv2.COLOR_RGBA2RGB)
+ background_roi = background_roi.astype(np.float)
+ obj_img = obj_img.astype(np.float)
+ rv = background.copy()
+ for i in range(3):
+ rv[y : y + obj_img.shape[0], x : x + obj_img.shape[1], i] = (
+ (~mask * background_roi[:, :, i] + mask * obj_img[:, :, i]) / 255
+ ).astype(np.uint8)
+ return rv
+
+ @staticmethod
+ def _translate_poi(poi: PointOfInterest, x: int, y: int) -> PointOfInterest:
+ return PointOfInterest(poi.x + x, poi.y + y)
+
+
+if __name__ == "__main__":
+ from pathlib import Path
+
+ import matplotlib.pyplot as plt
+
+ EXAMPLES_DIR = Path(__file__).parent / "examples"
+
+ _obj = LabeledImage(
+ cv2.cvtColor(cv2.imread(str(EXAMPLES_DIR / "logo.png"), cv2.IMREAD_UNCHANGED), cv2.COLOR_BGRA2RGBA),
+ [PointOfInterest(432, 76), PointOfInterest(432, 13)],
+ )
+ _bg = cv2.cvtColor(cv2.imread(str(EXAMPLES_DIR / "back1.jpg")), cv2.COLOR_BGR2RGB)
+
+ _blender = ImageBlender([])
+ for i in range(10):
+ res = _blender.blend(_bg, _obj)
+
+ plt.imshow(res.image)
+ plt.plot([poi.x for poi in res.point_of_interests], [poi.y for poi in res.point_of_interests], "r.")
+ plt.axis("off")
+ plt.tight_layout()
+ plt.savefig(str(EXAMPLES_DIR / f"image_{i}.jpg"))
+ plt.show()
diff --git a/robots-at-runes/research/dataset/blend/labeled_image_modifiers/__init__.py b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_modifier_abc.py b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_modifier_abc.py
new file mode 100644
index 0000000..e3ab2e7
--- /dev/null
+++ b/robots-at-runes/research/dataset/blend/labeled_image_modifiers/labeled_image_modifier_abc.py
@@ -0,0 +1,40 @@
+from abc import ABC, abstractmethod
+from random import random
+
+from polystar.common.models.image import Image
+from research.dataset.labeled_image import LabeledImage, PointOfInterest
+
+
+class LabeledImageModifierABC(ABC):
+ def randomly_modify(self, labeled_image: LabeledImage) -> LabeledImage:
+ return self.modify_from_factor(labeled_image, random() * 2 - 1)
+
+ def modify_from_factor(self, labeled_image: LabeledImage, factor: float) -> LabeledImage:
+ return self.modify_from_value(labeled_image, self._get_value_from_factor(factor))
+
+ def modify_from_value(self, labeled_image: LabeledImage, value: float) -> LabeledImage:
+ new_image = self._generate_modified_image(labeled_image.image, value)
+ return LabeledImage(
+ image=new_image,
+ point_of_interests=[
+ self._generate_modified_poi(poi, labeled_image.image, new_image, value)
+ for poi in labeled_image.point_of_interests
+ ],
+ )
+
+ @abstractmethod
+ def _get_value_from_factor(self, factor: float) -> float:
+ """
+ :param factor: a factor of modification, in range [-1, 1]
+ :return: the value of modification used by other function of this class
+ """
+
+ @abstractmethod
+ def _generate_modified_image(self, image: Image, value: float) -> Image:
+ pass
+
+ @abstractmethod
+ def _generate_modified_poi(
+ self, poi: PointOfInterest, original_image: Image, new_image: Image, value: float
+ ) -> PointOfInterest:
+ pass
diff --git a/robots-at-runes/research/dataset/labeled_image.py b/robots-at-runes/research/dataset/labeled_image.py
new file mode 100644
index 0000000..fec3912
--- /dev/null
+++ b/robots-at-runes/research/dataset/labeled_image.py
@@ -0,0 +1,16 @@
+from dataclasses import dataclass, field
+from typing import List
+
+from polystar.common.models.image import Image
+
+
+@dataclass
+class PointOfInterest:
+ x: int
+ y: int
+
+
+@dataclass
+class LabeledImage:
+ image: Image
+ point_of_interests: List[PointOfInterest] = field(default_factory=list)
--
GitLab