diff --git a/dataset/tf_records/.gitignore b/dataset/tf_records/.gitignore
deleted file mode 100644
index b9045fd730d938c05f2a0a14dcc3b4a035e0a633..0000000000000000000000000000000000000000
--- a/dataset/tf_records/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.record
\ No newline at end of file
diff --git a/dataset/twitch/robots-views/.gitignore b/dataset/twitch/robots-views/.gitignore
deleted file mode 100644
index c96a04f008ee21e260b28f7701595ed59e2839e3..0000000000000000000000000000000000000000
--- a/dataset/twitch/robots-views/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*
-!.gitignore
\ No newline at end of file
diff --git a/dataset/twitch/runes/.gitignore b/dataset/twitch/runes/.gitignore
deleted file mode 100644
index c96a04f008ee21e260b28f7701595ed59e2839e3..0000000000000000000000000000000000000000
--- a/dataset/twitch/runes/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-*
-!.gitignore
\ No newline at end of file
diff --git a/dataset/twitch/aerial-views/.gitignore b/dataset/twitch/v1/runes/.gitignore
similarity index 100%
rename from dataset/twitch/aerial-views/.gitignore
rename to dataset/twitch/v1/runes/.gitignore
diff --git a/src/research/dataset/scripts/move_aerial_views.py b/src/research/dataset/scripts/move_aerial_views.py
deleted file mode 100644
index 5b71f39bf6459e6b48d6ccadf361930ddcfe04b5..0000000000000000000000000000000000000000
--- a/src/research/dataset/scripts/move_aerial_views.py
+++ /dev/null
@@ -1,20 +0,0 @@
-from shutil import move
-
-from skimage import io
-from tqdm import tqdm
-
-from research.common.constants import TWITCH_DSET_DIR, TWITCH_ROBOTS_VIEWS_DIR
-from research.dataset.twitch.aerial_view_detector import aerial_view_detector
-
-AERIAL_VIEWS_DIR = TWITCH_DSET_DIR / "aerial-views"
-
-AERIAL_VIEWS_DIR.mkdir(parents=True, exist_ok=True)
-
-if __name__ == "__main__":
- n = 0
- for file_path in tqdm(list(TWITCH_ROBOTS_VIEWS_DIR.glob("*.jpg")), unit="image", desc="Moving aerial views"):
- if aerial_view_detector.is_matching(io.imread(str(file_path))):
- move(str(file_path), str(AERIAL_VIEWS_DIR / file_path.name))
- n += 1
-
- print(f"Moved {n} images")
diff --git a/src/research/dataset/twitch/README.md b/src/research/dataset/twitch/README.md
index 00976f6f0216dc4c077f4a35a3dbe904253f6343..d1a360b28438a6ba2c3c31efe12a30120d587399 100644
--- a/src/research/dataset/twitch/README.md
+++ b/src/research/dataset/twitch/README.md
@@ -43,8 +43,8 @@ You have 2 options to download the videos:
3. Rename it using the video id on twitch, and place it in [dataset/twitch/videos](../../../../dataset/twitch/videos)
5. Launch the python script [../scripts/extract_robots_views_from_video.py](../scripts/extract_robots_views_from_video.py), with the video id as parameter (In Pycharm, `Run` > `Edit Configurations...`, then in parameters enter the id). You can put multiple video ids by separating them with spaces.
-The frames will appear in the [dataset/twitch/robots-views](../../../../dataset/twitch/robots-views) folder.
+The frames will appear in the [dataset/twitch/robots-views](../../../../dataset/twitch/robots-views-hd-decalees) folder.
## Aerial dataset
-Once you have the robots views in the [dataset/twitch/robots-views](../../../../dataset/twitch/robots-views) folder, run the python script [../scripts/move_aerial_views.py](../scripts/move_aerial_views.py). It will put the aerial views in the [dataset/twitch/aerial-views](../../../../dataset/twitch/aerial-views) directory.
+Once you have the robots views in the [dataset/twitch/robots-views](../../../../dataset/twitch/robots-views-hd-decalees) folder, run the python script [../scripts/move_aerial_views.py](../scripts/move_aerial_and_runes_views.py). It will put the aerial views in the [dataset/twitch/aerial-views](../../../../dataset/twitch/aerial-views) directory.
diff --git a/src/research/dataset/twitch/aerial_view_detector.py b/src/research/dataset/twitch/aerial_view_detector.py
deleted file mode 100644
index f1f0332f34b623eac8566e1c36a133c4843677dd..0000000000000000000000000000000000000000
--- a/src/research/dataset/twitch/aerial_view_detector.py
+++ /dev/null
@@ -1,23 +0,0 @@
-from pathlib import Path
-
-from skimage import io
-
-from research.common.constants import TWITCH_DSET_DIR
-from research.dataset.twitch.mask_detector import MaskDetector
-
-aerial_view_detector = MaskDetector(
- Path(__file__).parent / "mask_aerial.jpg",
- [
- (527, 528, 292, 297, 20),
- (527, 531, 303, 303, 20),
- (532, 537, 286, 287, 20),
- (536, 541, 302, 303, 20),
- (543, 544, 292, 297, 20),
- (535, 535, 292, 297, 20),
- ],
-)
-
-if __name__ == "__main__":
- for file_path in sorted((TWITCH_DSET_DIR / "robots-views").glob("*.jpg")):
- if aerial_view_detector.is_matching(io.imread(str(file_path))):
- print(file_path.name)
diff --git a/src/research/dataset/twitch/mask_aerial_blue_2_hd.jpg b/src/research/dataset/twitch/mask_aerial_blue_2_hd.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..27e14148e00f8bb34ade761c4314ee14813ffc62
Binary files /dev/null and b/src/research/dataset/twitch/mask_aerial_blue_2_hd.jpg differ
diff --git a/src/research/dataset/twitch/mask_aerial_blue_hd.jpg b/src/research/dataset/twitch/mask_aerial_blue_hd.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b208e82ae4a622880a4af3146bb4bc65cbbfd118
Binary files /dev/null and b/src/research/dataset/twitch/mask_aerial_blue_hd.jpg differ
diff --git a/src/research/dataset/twitch/mask_aerial_red_2_hd.jpg b/src/research/dataset/twitch/mask_aerial_red_2_hd.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..0fedc05eb627cbbf82a084c3815930f0740f6b88
Binary files /dev/null and b/src/research/dataset/twitch/mask_aerial_red_2_hd.jpg differ
diff --git a/src/research/dataset/twitch/mask_aerial_red_hd.jpg b/src/research/dataset/twitch/mask_aerial_red_hd.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..fd8e3e99827af2527f68b19837673ed306e031ce
Binary files /dev/null and b/src/research/dataset/twitch/mask_aerial_red_hd.jpg differ
diff --git a/src/research/dataset/twitch/mask_bonus.jpg b/src/research/dataset/twitch/mask_bonus.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..5f5182c80aa14f0a836d88e7c464eb8e95504397
Binary files /dev/null and b/src/research/dataset/twitch/mask_bonus.jpg differ
diff --git a/src/research/dataset/twitch/mask_bonus_2.jpg b/src/research/dataset/twitch/mask_bonus_2.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..b9b90558a8f1026913bb4480f4fe562a24ff7a9e
Binary files /dev/null and b/src/research/dataset/twitch/mask_bonus_2.jpg differ
diff --git a/src/research/dataset/twitch/mask_bonus_3.jpg b/src/research/dataset/twitch/mask_bonus_3.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..9c5d13959b5e6c3d7ca12463f4df1209c1c9365b
Binary files /dev/null and b/src/research/dataset/twitch/mask_bonus_3.jpg differ
diff --git a/src/research/dataset/twitch/mask_detector.py b/src/research/dataset/twitch/mask_detector.py
index 8ffd748dc0fe4468087b6053acb8777385a3ed5d..4d63b526c8dda29406ba90f826450a5440f6b773 100644
--- a/src/research/dataset/twitch/mask_detector.py
+++ b/src/research/dataset/twitch/mask_detector.py
@@ -1,67 +1,51 @@
-import math
from pathlib import Path
-from typing import List, Tuple
-import cv2
import numpy as np
+from polystar.models.image import Image, load_image
-class Zone:
- def __init__(self, x_min, x_max, y_min, y_max, threshold, active_pixels, image_mask):
+DIR_PATH = Path(__file__).parent
- self.pixels = [(x, y) for x, y in active_pixels if (y_min <= y <= y_max) and (x_min <= x <= x_max)]
- self.mean_r = self.get_mean(0, image_mask)
- self.mean_g = self.get_mean(1, image_mask)
- self.mean_b = self.get_mean(2, image_mask)
- self.threshold = threshold
+class Mask:
+ def __init__(self, mask_file: Path, threshold: float):
+ self._threshold = threshold
+ mask_image = load_image(mask_file)
+ self._mask_coordinates = np.where(mask_image.max(axis=-1) > 40)
+ self._mask_values = mask_image[self._mask_coordinates].astype(np.int16)
- def get_mean(self, color, img):
- return sum([img[pix[0], pix[1]][color] for pix in self.pixels]) / len(self.pixels)
+ def match(self, image: Image) -> bool:
+ value = np.abs(self._mask_values - image[self._mask_coordinates]).mean()
+ # print(value)
+ return value <= self._threshold
- def get_means(self, img):
- mr, mg, mb = 0, 0, 0
- for pix in self.pixels:
- p = img[pix[0], pix[1]]
- mr += p[0]
- mg += p[1]
- mb += p[2]
- n_pixels = len(self.pixels)
- return mr / n_pixels, mg / n_pixels, mb / n_pixels
+robot_view_mask_hd = Mask(DIR_PATH / "mask_robot_view_hd.jpg", 20)
+aerial_view_mask_red_hd = Mask(DIR_PATH / "mask_aerial_red_hd.jpg", 15)
+aerial_view_mask_red_2_hd = Mask(DIR_PATH / "mask_aerial_red_2_hd.jpg", 15)
+aerial_view_mask_blue_hd = Mask(DIR_PATH / "mask_aerial_blue_hd.jpg", 15)
+aerial_view_mask_blue_2_hd = Mask(DIR_PATH / "mask_aerial_blue_2_hd.jpg", 15)
+bonus_view_mask_hd = Mask(DIR_PATH / "mask_bonus.jpg", 20)
+bonus_2_view_mask_hd = Mask(DIR_PATH / "mask_bonus_2.jpg", 20)
+bonus_3_view_mask_hd = Mask(DIR_PATH / "mask_bonus_3.jpg", 20)
- def is_matching(self, frame: np.ndarray):
- mean_r, mean_g, mean_b = self.get_means(frame)
- return (
- math.sqrt(pow(mean_r - self.mean_r, 2) + pow(mean_g - self.mean_g, 2) + pow(mean_b - self.mean_b, 2))
- < self.threshold
- )
-
-
-class MaskDetector:
- def __init__(self, image_path: Path, zones_params: List[Tuple[int, int, int, int, int]]):
- image_mask = cv2.imread(str(image_path))
- active_px = [
- (a, b)
- for a in range(0, 720)
- for b in range(0, 1280)
- if (
- image_mask[a, b].any()
- and int(image_mask[a, b][0]) + int(image_mask[a, b][1]) + int(image_mask[a, b][2]) > 50
- )
- ]
- self.zones = [Zone(*zone_params, active_px, image_mask) for zone_params in zones_params]
+def is_aerial_view(image: Image) -> bool:
+ return (
+ aerial_view_mask_red_hd.match(image)
+ or aerial_view_mask_red_2_hd.match(image)
+ or aerial_view_mask_blue_hd.match(image)
+ or aerial_view_mask_blue_2_hd.match(image)
+ )
- def is_matching(self, frame: np.ndarray):
- return all(zone.is_matching(frame) for zone in self.zones)
+def has_bonus_icon(image: Image) -> bool:
+ return bonus_view_mask_hd.match(image) or bonus_2_view_mask_hd.match(image) or bonus_3_view_mask_hd.match(image)
-robot_view_detector = MaskDetector(
- Path(__file__).parent / "mask_robot_view.jpg",
- [(0, 2000, 20, 70, 20), (0, 2000, 270, 370, 20), (0, 2000, 510, 770, 20),],
-)
-
-def is_image_from_robot_view(frame):
- return robot_view_detector.is_matching(frame)
+if __name__ == "__main__":
+ has_bonus_icon(
+ load_image(
+ Path("/Users/cytadel/polystar/cv-code/dataset/twitch/robots-views/470152932/470152932-frame-007460.jpg")
+ )
+ )
diff --git a/src/research/dataset/twitch/mask_robot_view_hd.jpg b/src/research/dataset/twitch/mask_robot_view_hd.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..c788ec54004dcb1ffe416f3c25a0cf972e295e2c
Binary files /dev/null and b/src/research/dataset/twitch/mask_robot_view_hd.jpg differ
diff --git a/src/research/dataset/twitch/resize_mask.py b/src/research/dataset/twitch/resize_mask.py
new file mode 100644
index 0000000000000000000000000000000000000000..238629956b3c4f23043b34287016a14f9b7c5c57
--- /dev/null
+++ b/src/research/dataset/twitch/resize_mask.py
@@ -0,0 +1,7 @@
+from pathlib import Path
+
+from imutils import resize
+
+from polystar.models.image import load_image, save_image
+
+save_image(resize(load_image(Path("mask_aerial.jpg")), 1920, 1080), Path("mask_aerial_red_hd.jpg"))
diff --git a/src/research/dataset/twitch/robots_views_extractor.py b/src/research/dataset/twitch/robots_views_extractor.py
index aa2b306a3da58c6ff5c01ad2a50a29e7b608f280..aa58f27bb3b6b1b29dbf234bfa63ac623c92c40b 100644
--- a/src/research/dataset/twitch/robots_views_extractor.py
+++ b/src/research/dataset/twitch/robots_views_extractor.py
@@ -5,23 +5,27 @@ from tqdm import tqdm
from polystar.frame_generators.fps_video_frame_generator import FPSVideoFrameGenerator
from research.common.constants import TWITCH_DSET_DIR, TWITCH_ROBOTS_VIEWS_DIR
-from research.dataset.twitch.mask_detector import is_image_from_robot_view
+from research.dataset.twitch.mask_detector import has_bonus_icon, robot_view_mask_hd
class RobotsViewExtractor:
FPS = 2
+ OFFSET_SECONDS = 3140 // 2
def __init__(self, video_name: str):
self.video_name: str = video_name
self.video_path = TWITCH_DSET_DIR / "videos" / f"{video_name}.mp4"
- self.frame_generator: FPSVideoFrameGenerator = FPSVideoFrameGenerator(self.video_path, self.FPS)
+ self.frame_generator: FPSVideoFrameGenerator = FPSVideoFrameGenerator(
+ self.video_path, self.OFFSET_SECONDS, self.FPS
+ )
self.count = 0
(TWITCH_ROBOTS_VIEWS_DIR / self.video_name).mkdir(exist_ok=True)
+ self._progress_bar = None
def run(self):
self._progress_bar = tqdm(
- enumerate(self.frame_generator.generate()),
+ enumerate(self.frame_generator.generate(), 1 + self.OFFSET_SECONDS * self.FPS),
total=self._get_number_of_frames(),
desc=f"Extracting robots views from video {self.video_name}.mp4",
unit="frames",
@@ -32,7 +36,7 @@ class RobotsViewExtractor:
print(f"Detected {self.count} robots views")
def _process_frame(self, frame: np.ndarray, frame_number: int):
- if is_image_from_robot_view(frame):
+ if robot_view_mask_hd.match(frame) and not has_bonus_icon(frame):
self._save_frame(frame, frame_number)
self.count += 1
self._progress_bar.set_description(
@@ -40,9 +44,7 @@ class RobotsViewExtractor:
)
def _save_frame(self, frame: np.ndarray, frame_number: int):
- cv2.imwrite(
- f"{TWITCH_ROBOTS_VIEWS_DIR}/{self.video_name}/{self.video_name}-frame-{frame_number + 1:06}.jpg", frame
- )
+ cv2.imwrite(f"{TWITCH_ROBOTS_VIEWS_DIR}/{self.video_name}/{self.video_name}-frame-{frame_number:06}.jpg", frame)
def _get_number_of_frames(self):
return int(ffmpeg.probe(str(self.video_path))["format"]["duration"].split(".")[0]) * self.FPS
diff --git a/src/research/tmp.py b/src/research/tmp.py
deleted file mode 100644
index b30695e9cbcc3862e45409624640cf572662785f..0000000000000000000000000000000000000000
--- a/src/research/tmp.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from research.common.constants import DSET_DIR
-
-print(DSET_DIR)