From b90d02cda1ccf39efc1181a01714abbb9f62463e Mon Sep 17 00:00:00 2001
From: Mathieu Beligon <mathieu@feedly.com>
Date: Fri, 25 Sep 2020 18:51:10 +0200
Subject: [PATCH] [common] (datasets) refactor: drop previous system
---
common/research/common/dataset/improvement/zoom.py | 4 ++--
common/research/common/dataset/tensorflow_record.py | 4 ++--
.../common/{datasets_v3 => datasets}/__init__.py | 0
.../common/{datasets_v3 => datasets}/capped_dataset.py | 2 +-
.../common/{datasets_v3 => datasets}/dataset.py | 2 +-
.../{datasets_v3 => datasets}/dataset_builder.py | 10 +++++-----
.../common/{datasets_v3 => datasets}/filter_dataset.py | 2 +-
.../common/{datasets_v3 => datasets}/image_dataset.py | 4 ++--
.../image_file_dataset_builder.py | 4 ++--
.../{datasets_v3 => datasets}/iterator_dataset.py | 2 +-
.../common/{datasets_v3 => datasets}/lazy_dataset.py | 0
.../common/{datasets_v3 => datasets}/roco/__init__.py | 0
.../{datasets_v3 => datasets}/roco/roco_annotation.py | 0
.../{datasets_v3 => datasets}/roco/roco_dataset.py | 6 +++---
.../roco/roco_dataset_builder.py | 4 ++--
.../roco/roco_dataset_descriptor.py | 4 ++--
.../{datasets_v3 => datasets}/roco/roco_datasets.py | 6 +++---
.../{datasets_v3 => datasets}/roco/zoo/__init__.py | 0
.../common/{datasets_v3 => datasets}/roco/zoo/dji.py | 4 ++--
.../{datasets_v3 => datasets}/roco/zoo/dji_zoomed.py | 4 ++--
.../roco/zoo/roco_dataset_zoo.py | 8 ++++----
.../{datasets_v3 => datasets}/roco/zoo/twitch.py | 4 ++--
.../{datasets_v3 => datasets}/transform_dataset.py | 4 ++--
.../common/{datasets_v3 => datasets}/union_dataset.py | 4 ++--
.../construct_twith_datasets_from_manual_annotation.py | 4 ++--
.../common/scripts/create_tensorflow_records.py | 8 ++++----
.../research/common/scripts/improve_roco_by_zooming.py | 4 ++--
common/research/common/scripts/visualize_dataset.py | 4 ++--
.../datasets/roco/test_directory_dataset_zoo.py | 2 +-
.../datasets_v3/roco/test_directory_dataset_zoo.py | 4 ++--
.../armor_color/armor_color_dataset.py | 2 +-
.../armor_color_pipeline_reporter_factory.py | 2 +-
.../armor_color/baseline_experiments.py | 2 +-
.../armor_digit/armor_digit_dataset.py | 2 +-
.../armor_digit_pipeline_reporter_factory.py | 2 +-
.../armor_digit/baseline_experiments.py | 2 +-
.../robots_at_robots/dataset/armor_dataset_factory.py | 8 ++++----
.../dataset/armor_value_dataset_cache.py | 6 +++---
.../dataset/armor_value_dataset_generator.py | 10 +++++-----
.../dataset/armor_value_target_factory.py | 2 +-
.../research/robots_at_robots/demos/demo_infer.py | 2 +-
.../research/robots_at_robots/demos/demo_pipeline.py | 2 +-
.../evaluation/image_pipeline_evaluation_reporter.py | 4 ++--
.../evaluation/image_pipeline_evaluator.py | 4 ++--
44 files changed, 79 insertions(+), 79 deletions(-)
rename common/research/common/{datasets_v3 => datasets}/__init__.py (100%)
rename common/research/common/{datasets_v3 => datasets}/capped_dataset.py (81%)
rename common/research/common/{datasets_v3 => datasets}/dataset.py (85%)
rename common/research/common/{datasets_v3 => datasets}/dataset_builder.py (86%)
rename common/research/common/{datasets_v3 => datasets}/filter_dataset.py (90%)
rename common/research/common/{datasets_v3 => datasets}/image_dataset.py (66%)
rename common/research/common/{datasets_v3 => datasets}/image_file_dataset_builder.py (89%)
rename common/research/common/{datasets_v3 => datasets}/iterator_dataset.py (80%)
rename common/research/common/{datasets_v3 => datasets}/lazy_dataset.py (100%)
rename common/research/common/{datasets_v3 => datasets}/roco/__init__.py (100%)
rename common/research/common/{datasets_v3 => datasets}/roco/roco_annotation.py (100%)
rename common/research/common/{datasets_v3 => datasets}/roco/roco_dataset.py (58%)
rename common/research/common/{datasets_v3 => datasets}/roco/roco_dataset_builder.py (76%)
rename common/research/common/{datasets_v3 => datasets}/roco/roco_dataset_descriptor.py (94%)
rename common/research/common/{datasets_v3 => datasets}/roco/roco_datasets.py (87%)
rename common/research/common/{datasets_v3 => datasets}/roco/zoo/__init__.py (100%)
rename common/research/common/{datasets_v3 => datasets}/roco/zoo/dji.py (80%)
rename common/research/common/{datasets_v3 => datasets}/roco/zoo/dji_zoomed.py (77%)
rename common/research/common/{datasets_v3 => datasets}/roco/zoo/roco_dataset_zoo.py (56%)
rename common/research/common/{datasets_v3 => datasets}/roco/zoo/twitch.py (80%)
rename common/research/common/{datasets_v3 => datasets}/transform_dataset.py (82%)
rename common/research/common/{datasets_v3 => datasets}/union_dataset.py (88%)
diff --git a/common/research/common/dataset/improvement/zoom.py b/common/research/common/dataset/improvement/zoom.py
index 803a759..f233890 100644
--- a/common/research/common/dataset/improvement/zoom.py
+++ b/common/research/common/dataset/improvement/zoom.py
@@ -7,8 +7,8 @@ from polystar.common.models.box import Box
from polystar.common.models.image import Image
from polystar.common.target_pipeline.objects_validators.in_box_validator import InBoxValidator
from polystar.common.view.plt_results_viewer import PltResultViewer
-from research.common.datasets_v3.roco.roco_annotation import ROCOAnnotation
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
def crop_image_annotation(
diff --git a/common/research/common/dataset/tensorflow_record.py b/common/research/common/dataset/tensorflow_record.py
index e4e8154..0f5f707 100644
--- a/common/research/common/dataset/tensorflow_record.py
+++ b/common/research/common/dataset/tensorflow_record.py
@@ -9,8 +9,8 @@ from tensorflow_core.python.lib.io import python_io
from polystar.common.models.label_map import label_map
from polystar.common.utils.tqdm import smart_tqdm
from research.common.constants import TENSORFLOW_RECORDS_DIR
-from research.common.datasets_v3.roco.roco_annotation import ROCOAnnotation
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
class TensorflowRecordFactory:
diff --git a/common/research/common/datasets_v3/__init__.py b/common/research/common/datasets/__init__.py
similarity index 100%
rename from common/research/common/datasets_v3/__init__.py
rename to common/research/common/datasets/__init__.py
diff --git a/common/research/common/datasets_v3/capped_dataset.py b/common/research/common/datasets/capped_dataset.py
similarity index 81%
rename from common/research/common/datasets_v3/capped_dataset.py
rename to common/research/common/datasets/capped_dataset.py
index 9cbbef5..2580b21 100644
--- a/common/research/common/datasets_v3/capped_dataset.py
+++ b/common/research/common/datasets/capped_dataset.py
@@ -1,7 +1,7 @@
from itertools import islice
from typing import Iterator, Tuple
-from research.common.datasets_v3.lazy_dataset import ExampleT, LazyDataset, TargetT
+from research.common.datasets.lazy_dataset import ExampleT, LazyDataset, TargetT
class CappedDataset(LazyDataset):
diff --git a/common/research/common/datasets_v3/dataset.py b/common/research/common/datasets/dataset.py
similarity index 85%
rename from common/research/common/datasets_v3/dataset.py
rename to common/research/common/datasets/dataset.py
index 4fd2cb8..1a6cb57 100644
--- a/common/research/common/datasets_v3/dataset.py
+++ b/common/research/common/datasets/dataset.py
@@ -1,6 +1,6 @@
from typing import Iterator, List, Tuple
-from research.common.datasets_v3.lazy_dataset import ExampleT, LazyDataset, TargetT
+from research.common.datasets.lazy_dataset import ExampleT, LazyDataset, TargetT
class Dataset(LazyDataset[ExampleT, TargetT]):
diff --git a/common/research/common/datasets_v3/dataset_builder.py b/common/research/common/datasets/dataset_builder.py
similarity index 86%
rename from common/research/common/datasets_v3/dataset_builder.py
rename to common/research/common/datasets/dataset_builder.py
index fc9806b..e3454bd 100644
--- a/common/research/common/datasets_v3/dataset_builder.py
+++ b/common/research/common/datasets/dataset_builder.py
@@ -3,11 +3,11 @@ from typing import Callable, Generic, Iterable, Iterator, Tuple
from polystar.common.filters.filter_abc import FilterABC
from polystar.common.filters.pass_through_filter import PassThroughFilter
from polystar.common.utils.misc import identity
-from research.common.datasets_v3.capped_dataset import CappedDataset
-from research.common.datasets_v3.dataset import Dataset
-from research.common.datasets_v3.filter_dataset import ExampleU, FilterDataset, TargetU
-from research.common.datasets_v3.lazy_dataset import ExampleT, LazyDataset, TargetT
-from research.common.datasets_v3.transform_dataset import TransformDataset
+from research.common.datasets.capped_dataset import CappedDataset
+from research.common.datasets.dataset import Dataset
+from research.common.datasets.filter_dataset import ExampleU, FilterDataset, TargetU
+from research.common.datasets.lazy_dataset import ExampleT, LazyDataset, TargetT
+from research.common.datasets.transform_dataset import TransformDataset
class DatasetBuilder(Generic[ExampleT, TargetT], Iterable[Tuple[ExampleT, TargetT, str]]):
diff --git a/common/research/common/datasets_v3/filter_dataset.py b/common/research/common/datasets/filter_dataset.py
similarity index 90%
rename from common/research/common/datasets_v3/filter_dataset.py
rename to common/research/common/datasets/filter_dataset.py
index 415b3cb..6af2935 100644
--- a/common/research/common/datasets_v3/filter_dataset.py
+++ b/common/research/common/datasets/filter_dataset.py
@@ -1,7 +1,7 @@
from typing import Iterator, Tuple, TypeVar
from polystar.common.filters.filter_abc import FilterABC
-from research.common.datasets_v3.lazy_dataset import ExampleT, LazyDataset, TargetT
+from research.common.datasets.lazy_dataset import ExampleT, LazyDataset, TargetT
ExampleU = TypeVar("ExampleU")
TargetU = TypeVar("TargetU")
diff --git a/common/research/common/datasets_v3/image_dataset.py b/common/research/common/datasets/image_dataset.py
similarity index 66%
rename from common/research/common/datasets_v3/image_dataset.py
rename to common/research/common/datasets/image_dataset.py
index b7bf27e..c7edb38 100644
--- a/common/research/common/datasets_v3/image_dataset.py
+++ b/common/research/common/datasets/image_dataset.py
@@ -1,8 +1,8 @@
from pathlib import Path
from polystar.common.models.image import Image
-from research.common.datasets_v3.dataset import Dataset
-from research.common.datasets_v3.lazy_dataset import TargetT
+from research.common.datasets.dataset import Dataset
+from research.common.datasets.lazy_dataset import TargetT
LazyFileDataset = Dataset[Path, TargetT]
FileDataset = Dataset[Path, TargetT]
diff --git a/common/research/common/datasets_v3/image_file_dataset_builder.py b/common/research/common/datasets/image_file_dataset_builder.py
similarity index 89%
rename from common/research/common/datasets_v3/image_file_dataset_builder.py
rename to common/research/common/datasets/image_file_dataset_builder.py
index dbd5b18..b4e0784 100644
--- a/common/research/common/datasets_v3/image_file_dataset_builder.py
+++ b/common/research/common/datasets/image_file_dataset_builder.py
@@ -2,8 +2,8 @@ from pathlib import Path
from typing import Callable, Generic, Iterable, Iterator, Tuple
from polystar.common.models.image import Image, load_image
-from research.common.datasets_v3.dataset_builder import DatasetBuilder
-from research.common.datasets_v3.lazy_dataset import LazyDataset, TargetT
+from research.common.datasets.dataset_builder import DatasetBuilder
+from research.common.datasets.lazy_dataset import LazyDataset, TargetT
class LazyFileDataset(LazyDataset[Path, TargetT]):
diff --git a/common/research/common/datasets_v3/iterator_dataset.py b/common/research/common/datasets/iterator_dataset.py
similarity index 80%
rename from common/research/common/datasets_v3/iterator_dataset.py
rename to common/research/common/datasets/iterator_dataset.py
index 7b583de..b493b91 100644
--- a/common/research/common/datasets_v3/iterator_dataset.py
+++ b/common/research/common/datasets/iterator_dataset.py
@@ -1,6 +1,6 @@
from typing import Iterator, Tuple
-from research.common.datasets_v3.lazy_dataset import ExampleT, LazyDataset, TargetT
+from research.common.datasets.lazy_dataset import ExampleT, LazyDataset, TargetT
class IteratorDataset(LazyDataset[ExampleT, TargetT]):
diff --git a/common/research/common/datasets_v3/lazy_dataset.py b/common/research/common/datasets/lazy_dataset.py
similarity index 100%
rename from common/research/common/datasets_v3/lazy_dataset.py
rename to common/research/common/datasets/lazy_dataset.py
diff --git a/common/research/common/datasets_v3/roco/__init__.py b/common/research/common/datasets/roco/__init__.py
similarity index 100%
rename from common/research/common/datasets_v3/roco/__init__.py
rename to common/research/common/datasets/roco/__init__.py
diff --git a/common/research/common/datasets_v3/roco/roco_annotation.py b/common/research/common/datasets/roco/roco_annotation.py
similarity index 100%
rename from common/research/common/datasets_v3/roco/roco_annotation.py
rename to common/research/common/datasets/roco/roco_annotation.py
diff --git a/common/research/common/datasets_v3/roco/roco_dataset.py b/common/research/common/datasets/roco/roco_dataset.py
similarity index 58%
rename from common/research/common/datasets_v3/roco/roco_dataset.py
rename to common/research/common/datasets/roco/roco_dataset.py
index 646c6c5..18015dc 100644
--- a/common/research/common/datasets_v3/roco/roco_dataset.py
+++ b/common/research/common/datasets/roco/roco_dataset.py
@@ -1,9 +1,9 @@
from pathlib import Path
from polystar.common.models.image import Image
-from research.common.datasets_v3.dataset import Dataset
-from research.common.datasets_v3.lazy_dataset import LazyDataset
-from research.common.datasets_v3.roco.roco_annotation import ROCOAnnotation
+from research.common.datasets.dataset import Dataset
+from research.common.datasets.lazy_dataset import LazyDataset
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation
LazyROCOFileDataset = LazyDataset[Path, ROCOAnnotation]
ROCOFileDataset = Dataset[Path, ROCOAnnotation]
diff --git a/common/research/common/datasets_v3/roco/roco_dataset_builder.py b/common/research/common/datasets/roco/roco_dataset_builder.py
similarity index 76%
rename from common/research/common/datasets_v3/roco/roco_dataset_builder.py
rename to common/research/common/datasets/roco/roco_dataset_builder.py
index 4bd4207..bb40222 100644
--- a/common/research/common/datasets_v3/roco/roco_dataset_builder.py
+++ b/common/research/common/datasets/roco/roco_dataset_builder.py
@@ -1,7 +1,7 @@
from pathlib import Path
-from research.common.datasets_v3.image_file_dataset_builder import DirectoryDatasetBuilder
-from research.common.datasets_v3.roco.roco_annotation import ROCOAnnotation
+from research.common.datasets.image_file_dataset_builder import DirectoryDatasetBuilder
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation
class ROCODatasetBuilder(DirectoryDatasetBuilder[ROCOAnnotation]):
diff --git a/common/research/common/datasets_v3/roco/roco_dataset_descriptor.py b/common/research/common/datasets/roco/roco_dataset_descriptor.py
similarity index 94%
rename from common/research/common/datasets_v3/roco/roco_dataset_descriptor.py
rename to common/research/common/datasets/roco/roco_dataset_descriptor.py
index f50925f..070bc9f 100644
--- a/common/research/common/datasets_v3/roco/roco_dataset_descriptor.py
+++ b/common/research/common/datasets/roco/roco_dataset_descriptor.py
@@ -7,8 +7,8 @@ from pandas import DataFrame
from polystar.common.models.object import Armor, ObjectType
from polystar.common.utils.markdown import MarkdownFile
from polystar.common.utils.tqdm import smart_tqdm
-from research.common.datasets_v3.roco.roco_dataset import LazyROCOFileDataset
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.roco_dataset import LazyROCOFileDataset
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
@dataclass
diff --git a/common/research/common/datasets_v3/roco/roco_datasets.py b/common/research/common/datasets/roco/roco_datasets.py
similarity index 87%
rename from common/research/common/datasets_v3/roco/roco_datasets.py
rename to common/research/common/datasets/roco/roco_datasets.py
index 993fd40..d824c69 100644
--- a/common/research/common/datasets_v3/roco/roco_datasets.py
+++ b/common/research/common/datasets/roco/roco_datasets.py
@@ -2,9 +2,9 @@ from abc import abstractmethod
from pathlib import Path
from typing import ClassVar, Iterator, Set
-from research.common.datasets_v3.roco.roco_annotation import ROCOAnnotation
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
-from research.common.datasets_v3.union_dataset import UnionLazyDataset
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.union_dataset import UnionLazyDataset
class ROCODatasetsMeta(type):
diff --git a/common/research/common/datasets_v3/roco/zoo/__init__.py b/common/research/common/datasets/roco/zoo/__init__.py
similarity index 100%
rename from common/research/common/datasets_v3/roco/zoo/__init__.py
rename to common/research/common/datasets/roco/zoo/__init__.py
diff --git a/common/research/common/datasets_v3/roco/zoo/dji.py b/common/research/common/datasets/roco/zoo/dji.py
similarity index 80%
rename from common/research/common/datasets_v3/roco/zoo/dji.py
rename to common/research/common/datasets/roco/zoo/dji.py
index ac5af1c..3ba6151 100644
--- a/common/research/common/datasets_v3/roco/zoo/dji.py
+++ b/common/research/common/datasets/roco/zoo/dji.py
@@ -1,6 +1,6 @@
from research.common.constants import DJI_ROCO_DSET_DIR
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
-from research.common.datasets_v3.roco.roco_datasets import ROCODatasets
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_datasets import ROCODatasets
class DJIROCODatasets(ROCODatasets):
diff --git a/common/research/common/datasets_v3/roco/zoo/dji_zoomed.py b/common/research/common/datasets/roco/zoo/dji_zoomed.py
similarity index 77%
rename from common/research/common/datasets_v3/roco/zoo/dji_zoomed.py
rename to common/research/common/datasets/roco/zoo/dji_zoomed.py
index 4d10b33..5bad2bc 100644
--- a/common/research/common/datasets_v3/roco/zoo/dji_zoomed.py
+++ b/common/research/common/datasets/roco/zoo/dji_zoomed.py
@@ -1,7 +1,7 @@
from polystar.common.utils.str_utils import snake2camel
from research.common.constants import DJI_ROCO_ZOOMED_DSET_DIR
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
-from research.common.datasets_v3.roco.roco_datasets import ROCODatasets
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_datasets import ROCODatasets
class DJIROCOZoomedDatasets(ROCODatasets):
diff --git a/common/research/common/datasets_v3/roco/zoo/roco_dataset_zoo.py b/common/research/common/datasets/roco/zoo/roco_dataset_zoo.py
similarity index 56%
rename from common/research/common/datasets_v3/roco/zoo/roco_dataset_zoo.py
rename to common/research/common/datasets/roco/zoo/roco_dataset_zoo.py
index ba571b9..12a284c 100644
--- a/common/research/common/datasets_v3/roco/zoo/roco_dataset_zoo.py
+++ b/common/research/common/datasets/roco/zoo/roco_dataset_zoo.py
@@ -1,9 +1,9 @@
from typing import Iterable, Iterator, Type
-from research.common.datasets_v3.roco.roco_datasets import ROCODatasets
-from research.common.datasets_v3.roco.zoo.dji import DJIROCODatasets
-from research.common.datasets_v3.roco.zoo.dji_zoomed import DJIROCOZoomedDatasets
-from research.common.datasets_v3.roco.zoo.twitch import TwitchROCODatasets
+from research.common.datasets.roco.roco_datasets import ROCODatasets
+from research.common.datasets.roco.zoo.dji import DJIROCODatasets
+from research.common.datasets.roco.zoo.dji_zoomed import DJIROCOZoomedDatasets
+from research.common.datasets.roco.zoo.twitch import TwitchROCODatasets
class ROCODatasetsZoo(Iterable[Type[ROCODatasets]]):
diff --git a/common/research/common/datasets_v3/roco/zoo/twitch.py b/common/research/common/datasets/roco/zoo/twitch.py
similarity index 80%
rename from common/research/common/datasets_v3/roco/zoo/twitch.py
rename to common/research/common/datasets/roco/zoo/twitch.py
index 7cfecb3..5a4c420 100644
--- a/common/research/common/datasets_v3/roco/zoo/twitch.py
+++ b/common/research/common/datasets/roco/zoo/twitch.py
@@ -1,6 +1,6 @@
from research.common.constants import TWITCH_DSET_DIR
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
-from research.common.datasets_v3.roco.roco_datasets import ROCODatasets
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_datasets import ROCODatasets
class TwitchROCODatasets(ROCODatasets):
diff --git a/common/research/common/datasets_v3/transform_dataset.py b/common/research/common/datasets/transform_dataset.py
similarity index 82%
rename from common/research/common/datasets_v3/transform_dataset.py
rename to common/research/common/datasets/transform_dataset.py
index 8b5fcb8..54251be 100644
--- a/common/research/common/datasets_v3/transform_dataset.py
+++ b/common/research/common/datasets/transform_dataset.py
@@ -1,7 +1,7 @@
from typing import Callable, Iterator, Tuple
-from research.common.datasets_v3.dataset_builder import ExampleU, TargetU
-from research.common.datasets_v3.lazy_dataset import ExampleT, LazyDataset, TargetT
+from research.common.datasets.dataset_builder import ExampleU, TargetU
+from research.common.datasets.lazy_dataset import ExampleT, LazyDataset, TargetT
class TransformDataset(LazyDataset[ExampleU, TargetU]):
diff --git a/common/research/common/datasets_v3/union_dataset.py b/common/research/common/datasets/union_dataset.py
similarity index 88%
rename from common/research/common/datasets_v3/union_dataset.py
rename to common/research/common/datasets/union_dataset.py
index ddbd437..4a381a6 100644
--- a/common/research/common/datasets_v3/union_dataset.py
+++ b/common/research/common/datasets/union_dataset.py
@@ -1,7 +1,7 @@
from typing import Iterable, Iterator, List, Tuple
-from research.common.datasets_v3.dataset import Dataset
-from research.common.datasets_v3.lazy_dataset import ExampleT, LazyDataset, TargetT
+from research.common.datasets.dataset import Dataset
+from research.common.datasets.lazy_dataset import ExampleT, LazyDataset, TargetT
class UnionLazyDataset(LazyDataset[ExampleT, TargetT]):
diff --git a/common/research/common/scripts/construct_twith_datasets_from_manual_annotation.py b/common/research/common/scripts/construct_twith_datasets_from_manual_annotation.py
index 0cbabc4..d7333bd 100644
--- a/common/research/common/scripts/construct_twith_datasets_from_manual_annotation.py
+++ b/common/research/common/scripts/construct_twith_datasets_from_manual_annotation.py
@@ -2,8 +2,8 @@ from os import remove
from shutil import copy, make_archive, move, rmtree
from research.common.constants import TWITCH_DSET_DIR, TWITCH_DSET_ROBOTS_VIEWS_DIR, TWITCH_ROBOTS_VIEWS_DIR
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
-from research.common.datasets_v3.roco.roco_dataset_descriptor import make_markdown_dataset_report
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_dataset_descriptor import make_markdown_dataset_report
from research.common.scripts.construct_dataset_from_manual_annotation import construct_dataset_from_manual_annotations
from research.common.scripts.correct_annotations import AnnotationFileCorrector
diff --git a/common/research/common/scripts/create_tensorflow_records.py b/common/research/common/scripts/create_tensorflow_records.py
index 9456688..ad1d625 100644
--- a/common/research/common/scripts/create_tensorflow_records.py
+++ b/common/research/common/scripts/create_tensorflow_records.py
@@ -1,8 +1,8 @@
from research.common.dataset.tensorflow_record import TensorflowRecordFactory
-from research.common.datasets_v3.roco.zoo.dji import DJIROCODatasets
-from research.common.datasets_v3.roco.zoo.dji_zoomed import DJIROCOZoomedDatasets
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
-from research.common.datasets_v3.roco.zoo.twitch import TwitchROCODatasets
+from research.common.datasets.roco.zoo.dji import DJIROCODatasets
+from research.common.datasets.roco.zoo.dji_zoomed import DJIROCOZoomedDatasets
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.zoo.twitch import TwitchROCODatasets
def create_one_record_per_roco_dset():
diff --git a/common/research/common/scripts/improve_roco_by_zooming.py b/common/research/common/scripts/improve_roco_by_zooming.py
index eee88b9..6cc8de9 100644
--- a/common/research/common/scripts/improve_roco_by_zooming.py
+++ b/common/research/common/scripts/improve_roco_by_zooming.py
@@ -10,8 +10,8 @@ from research.common.dataset.perturbations.image_modifiers.brightness import Bri
from research.common.dataset.perturbations.image_modifiers.contrast import ContrastModifier
from research.common.dataset.perturbations.image_modifiers.saturation import SaturationModifier
from research.common.dataset.perturbations.perturbator import ImagePerturbator
-from research.common.datasets_v3.roco.roco_dataset import LazyROCODataset
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.roco_dataset import LazyROCODataset
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
def improve_dji_roco_dataset_by_zooming_and_perturbating(
diff --git a/common/research/common/scripts/visualize_dataset.py b/common/research/common/scripts/visualize_dataset.py
index 636ddd5..6c57ec3 100644
--- a/common/research/common/scripts/visualize_dataset.py
+++ b/common/research/common/scripts/visualize_dataset.py
@@ -1,6 +1,6 @@
from polystar.common.view.plt_results_viewer import PltResultViewer
-from research.common.datasets_v3.roco.roco_dataset import LazyROCODataset
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.roco_dataset import LazyROCODataset
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
def visualize_dataset(dataset: LazyROCODataset, n_images: int):
diff --git a/common/tests/common/unittests/datasets/roco/test_directory_dataset_zoo.py b/common/tests/common/unittests/datasets/roco/test_directory_dataset_zoo.py
index b82b017..60e5b65 100644
--- a/common/tests/common/unittests/datasets/roco/test_directory_dataset_zoo.py
+++ b/common/tests/common/unittests/datasets/roco/test_directory_dataset_zoo.py
@@ -7,7 +7,7 @@ from numpy.testing import assert_array_almost_equal
from polystar.common.models.image import save_image
from research.common.datasets.roco.directory_roco_dataset import DirectoryROCODataset
-from research.common.datasets_v3.roco.roco_annotation import ROCOAnnotation
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation
class TestDirectoryROCODataset(TestCase):
diff --git a/common/tests/common/unittests/datasets_v3/roco/test_directory_dataset_zoo.py b/common/tests/common/unittests/datasets_v3/roco/test_directory_dataset_zoo.py
index 90dabb9..f2c839f 100644
--- a/common/tests/common/unittests/datasets_v3/roco/test_directory_dataset_zoo.py
+++ b/common/tests/common/unittests/datasets_v3/roco/test_directory_dataset_zoo.py
@@ -7,8 +7,8 @@ from numpy import asarray, float32
from numpy.testing import assert_array_almost_equal
from polystar.common.models.image import save_image
-from research.common.datasets_v3.roco.roco_annotation import ROCOAnnotation
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
class TestDirectoryROCODataset(TestCase):
diff --git a/robots-at-robots/research/robots_at_robots/armor_color/armor_color_dataset.py b/robots-at-robots/research/robots_at_robots/armor_color/armor_color_dataset.py
index 2aff0a8..548f0cf 100644
--- a/robots-at-robots/research/robots_at_robots/armor_color/armor_color_dataset.py
+++ b/robots-at-robots/research/robots_at_robots/armor_color/armor_color_dataset.py
@@ -1,7 +1,7 @@
from itertools import islice
from polystar.common.models.object import Armor
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
from research.robots_at_robots.dataset.armor_value_dataset_generator import ArmorValueDatasetGenerator
from research.robots_at_robots.dataset.armor_value_target_factory import ArmorValueTargetFactory
diff --git a/robots-at-robots/research/robots_at_robots/armor_color/armor_color_pipeline_reporter_factory.py b/robots-at-robots/research/robots_at_robots/armor_color/armor_color_pipeline_reporter_factory.py
index 6ddd9cd..b1f976d 100644
--- a/robots-at-robots/research/robots_at_robots/armor_color/armor_color_pipeline_reporter_factory.py
+++ b/robots-at-robots/research/robots_at_robots/armor_color/armor_color_pipeline_reporter_factory.py
@@ -1,6 +1,6 @@
from typing import List
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
from research.robots_at_robots.armor_color.armor_color_dataset import make_armor_color_dataset_generator
from research.robots_at_robots.evaluation.image_pipeline_evaluation_reporter import ImagePipelineEvaluationReporter
from research.robots_at_robots.evaluation.image_pipeline_evaluator import ImagePipelineEvaluator
diff --git a/robots-at-robots/research/robots_at_robots/armor_color/baseline_experiments.py b/robots-at-robots/research/robots_at_robots/armor_color/baseline_experiments.py
index 06f6354..c0ab8de 100644
--- a/robots-at-robots/research/robots_at_robots/armor_color/baseline_experiments.py
+++ b/robots-at-robots/research/robots_at_robots/armor_color/baseline_experiments.py
@@ -4,7 +4,7 @@ from polystar.common.image_pipeline.classifier_image_pipeline import ClassifierI
from polystar.common.image_pipeline.image_featurizer.mean_rgb_channels_featurizer import MeanChannelsFeaturizer
from polystar.common.image_pipeline.models.random_model import RandomModel
from polystar.common.image_pipeline.models.red_blue_channels_comparison_model import RedBlueComparisonModel
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
from research.robots_at_robots.armor_color.armor_color_pipeline_reporter_factory import (
ArmorColorPipelineReporterFactory,
)
diff --git a/robots-at-robots/research/robots_at_robots/armor_digit/armor_digit_dataset.py b/robots-at-robots/research/robots_at_robots/armor_digit/armor_digit_dataset.py
index 38b9e63..9635d53 100644
--- a/robots-at-robots/research/robots_at_robots/armor_digit/armor_digit_dataset.py
+++ b/robots-at-robots/research/robots_at_robots/armor_digit/armor_digit_dataset.py
@@ -3,7 +3,7 @@ from typing import Iterable
from polystar.common.filters.keep_filter import KeepFilter
from polystar.common.models.object import Armor
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
from research.robots_at_robots.dataset.armor_value_dataset_generator import ArmorValueDatasetGenerator
from research.robots_at_robots.dataset.armor_value_target_factory import ArmorValueTargetFactory
diff --git a/robots-at-robots/research/robots_at_robots/armor_digit/armor_digit_pipeline_reporter_factory.py b/robots-at-robots/research/robots_at_robots/armor_digit/armor_digit_pipeline_reporter_factory.py
index 9fcdc40..64eceb7 100644
--- a/robots-at-robots/research/robots_at_robots/armor_digit/armor_digit_pipeline_reporter_factory.py
+++ b/robots-at-robots/research/robots_at_robots/armor_digit/armor_digit_pipeline_reporter_factory.py
@@ -1,6 +1,6 @@
from typing import Iterable, List
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
from research.robots_at_robots.armor_digit.armor_digit_dataset import make_armor_digit_dataset_generator
from research.robots_at_robots.evaluation.image_pipeline_evaluation_reporter import ImagePipelineEvaluationReporter
from research.robots_at_robots.evaluation.image_pipeline_evaluator import ImagePipelineEvaluator
diff --git a/robots-at-robots/research/robots_at_robots/armor_digit/baseline_experiments.py b/robots-at-robots/research/robots_at_robots/armor_digit/baseline_experiments.py
index 9925a7b..5088065 100644
--- a/robots-at-robots/research/robots_at_robots/armor_digit/baseline_experiments.py
+++ b/robots-at-robots/research/robots_at_robots/armor_digit/baseline_experiments.py
@@ -2,7 +2,7 @@ import logging
from polystar.common.image_pipeline.classifier_image_pipeline import ClassifierImagePipeline
from polystar.common.image_pipeline.models.random_model import RandomModel
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
from research.robots_at_robots.armor_digit.armor_digit_pipeline_reporter_factory import (
ArmorDigitPipelineReporterFactory,
)
diff --git a/robots-at-robots/research/robots_at_robots/dataset/armor_dataset_factory.py b/robots-at-robots/research/robots_at_robots/dataset/armor_dataset_factory.py
index 7b98b40..6a800ae 100644
--- a/robots-at-robots/research/robots_at_robots/dataset/armor_dataset_factory.py
+++ b/robots-at-robots/research/robots_at_robots/dataset/armor_dataset_factory.py
@@ -6,10 +6,10 @@ import matplotlib.pyplot as plt
from polystar.common.models.image import Image
from polystar.common.models.object import Armor, ObjectType
from polystar.common.target_pipeline.objects_validators.type_object_validator import TypeObjectValidator
-from research.common.datasets_v3.lazy_dataset import LazyDataset
-from research.common.datasets_v3.roco.roco_annotation import ROCOAnnotation
-from research.common.datasets_v3.roco.roco_dataset import LazyROCODataset
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.lazy_dataset import LazyDataset
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation
+from research.common.datasets.roco.roco_dataset import LazyROCODataset
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
class ArmorDataset(LazyDataset[Image, Armor]):
diff --git a/robots-at-robots/research/robots_at_robots/dataset/armor_value_dataset_cache.py b/robots-at-robots/research/robots_at_robots/dataset/armor_value_dataset_cache.py
index a7b7c78..68f1b15 100644
--- a/robots-at-robots/research/robots_at_robots/dataset/armor_value_dataset_cache.py
+++ b/robots-at-robots/research/robots_at_robots/dataset/armor_value_dataset_cache.py
@@ -7,9 +7,9 @@ from polystar.common.models.image import Image, save_image
from polystar.common.utils.misc import identity
from polystar.common.utils.time import create_time_id
from polystar.common.utils.tqdm import smart_tqdm
-from research.common.datasets_v3.lazy_dataset import LazyDataset, TargetT
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
-from research.common.datasets_v3.transform_dataset import TransformDataset
+from research.common.datasets.lazy_dataset import LazyDataset, TargetT
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.transform_dataset import TransformDataset
from research.robots_at_robots.dataset.armor_dataset_factory import ArmorDataset
from research.robots_at_robots.dataset.armor_value_target_factory import ArmorValueTargetFactory
diff --git a/robots-at-robots/research/robots_at_robots/dataset/armor_value_dataset_generator.py b/robots-at-robots/research/robots_at_robots/dataset/armor_value_dataset_generator.py
index f95138f..befa1a1 100644
--- a/robots-at-robots/research/robots_at_robots/dataset/armor_value_dataset_generator.py
+++ b/robots-at-robots/research/robots_at_robots/dataset/armor_value_dataset_generator.py
@@ -3,11 +3,11 @@ from typing import Generic, List
from polystar.common.filters.filter_abc import FilterABC
from polystar.common.filters.pass_through_filter import PassThroughFilter
-from research.common.datasets_v3.dataset import Dataset
-from research.common.datasets_v3.image_file_dataset_builder import DirectoryDatasetBuilder
-from research.common.datasets_v3.lazy_dataset import TargetT
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
-from research.common.datasets_v3.union_dataset import UnionDataset
+from research.common.datasets.dataset import Dataset
+from research.common.datasets.image_file_dataset_builder import DirectoryDatasetBuilder
+from research.common.datasets.lazy_dataset import TargetT
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.union_dataset import UnionDataset
from research.robots_at_robots.dataset.armor_value_dataset_cache import ArmorValueDatasetCache
from research.robots_at_robots.dataset.armor_value_target_factory import ArmorValueTargetFactory
diff --git a/robots-at-robots/research/robots_at_robots/dataset/armor_value_target_factory.py b/robots-at-robots/research/robots_at_robots/dataset/armor_value_target_factory.py
index 2d5da79..a6a0326 100644
--- a/robots-at-robots/research/robots_at_robots/dataset/armor_value_target_factory.py
+++ b/robots-at-robots/research/robots_at_robots/dataset/armor_value_target_factory.py
@@ -3,7 +3,7 @@ from pathlib import Path
from typing import Generic
from polystar.common.models.object import Armor
-from research.common.datasets_v3.lazy_dataset import TargetT
+from research.common.datasets.lazy_dataset import TargetT
class ArmorValueTargetFactory(Generic[TargetT]):
diff --git a/robots-at-robots/research/robots_at_robots/demos/demo_infer.py b/robots-at-robots/research/robots_at_robots/demos/demo_infer.py
index c5d8edd..40c567f 100644
--- a/robots-at-robots/research/robots_at_robots/demos/demo_infer.py
+++ b/robots-at-robots/research/robots_at_robots/demos/demo_infer.py
@@ -5,7 +5,7 @@ from polystar.common.target_pipeline.objects_validators.confidence_object_valida
from polystar.common.utils.tensorflow import patch_tf_v2
from polystar.common.view.plt_results_viewer import PltResultViewer
from polystar.robots_at_robots.dependency_injection import make_injector
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
from research.robots_at_robots.demos.utils import load_tf_model
if __name__ == "__main__":
diff --git a/robots-at-robots/research/robots_at_robots/demos/demo_pipeline.py b/robots-at-robots/research/robots_at_robots/demos/demo_pipeline.py
index 9149086..5d86192 100644
--- a/robots-at-robots/research/robots_at_robots/demos/demo_pipeline.py
+++ b/robots-at-robots/research/robots_at_robots/demos/demo_pipeline.py
@@ -18,7 +18,7 @@ from polystar.common.target_pipeline.target_pipeline import NoTargetFoundExcepti
from polystar.common.utils.tensorflow import patch_tf_v2
from polystar.common.view.plt_results_viewer import PltResultViewer
from polystar.robots_at_robots.dependency_injection import make_injector
-from research.common.datasets_v3.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
+from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
from research.robots_at_robots.demos.utils import load_tf_model
if __name__ == "__main__":
diff --git a/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluation_reporter.py b/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluation_reporter.py
index 226758b..5607a3a 100644
--- a/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluation_reporter.py
+++ b/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluation_reporter.py
@@ -15,8 +15,8 @@ from polystar.common.utils.dataframe import Format, format_df_column, format_df_
from polystar.common.utils.markdown import MarkdownFile
from polystar.common.utils.time import create_time_id
from research.common.constants import DSET_DIR, EVALUATION_DIR
-from research.common.datasets_v3.lazy_dataset import TargetT
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.lazy_dataset import TargetT
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
from research.robots_at_robots.evaluation.image_pipeline_evaluator import (
ClassificationResults,
ImagePipelineEvaluator,
diff --git a/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluator.py b/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluator.py
index aab7352..97dac63 100644
--- a/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluator.py
+++ b/robots-at-robots/research/robots_at_robots/evaluation/image_pipeline_evaluator.py
@@ -10,8 +10,8 @@ from sklearn.metrics import classification_report, confusion_matrix
from polystar.common.image_pipeline.image_pipeline import ImagePipeline
from polystar.common.models.image import Image, load_images
-from research.common.datasets_v3.lazy_dataset import TargetT
-from research.common.datasets_v3.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.lazy_dataset import TargetT
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
from research.robots_at_robots.dataset.armor_value_dataset_generator import ArmorValueDatasetGenerator
--
GitLab