diff --git a/dataset/.gitignore b/dataset/.gitignore
index 0a353edc8d563e6ec535234db257ab8a9e7770e5..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644
--- a/dataset/.gitignore
+++ b/dataset/.gitignore
@@ -1 +0,0 @@
-**/*.md
\ No newline at end of file
diff --git a/dataset/dji_roco/dset_DJI_report.md b/dataset/dji_roco/dset_DJI_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..bc28a7e867f2d3239a0604fa871a45d285c7b002
--- /dev/null
+++ b/dataset/dji_roco/dset_DJI_report.md
@@ -0,0 +1,23 @@
+# Dataset DJI
+
+10369 images, with:
+
+ - 5249 bases
+ - 9313 watchers
+ - 0 runes
+ - 57547 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 2020 | 168 | 1896 | 4084 |
+| 1 | 13511 | 3125 | 14493 | 31129 |
+| 2 | 11487 | 177 | 10694 | 22358 |
+| 3 | 12031 | 2949 | 11316 | 26296 |
+| 4 | 10739 | 2897 | 11446 | 25082 |
+| 5 | 3046 | 595 | 2940 | 6581 |
+| 6 | 7630 | 109 | 8686 | 16425 |
+| 7 | 1892 | 1974 | 3968 | 7834 |
+| 8 | 2881 | 72 | 3065 | 6018 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 65237 | 12066 | 68504 | 0 |
+
diff --git a/dataset/dji_roco/report.md b/dataset/dji_roco/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..03b1926fb74c2528c1aebb5486e42d75593a7ed5
--- /dev/null
+++ b/dataset/dji_roco/report.md
@@ -0,0 +1,32 @@
+# Global report
+
+10369 images, with:
+
+ - 10369 images
+ - 57547 robots
+ - 9313 watchers
+ - 5249 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 2020 | 168 | 1896 | 4084 |
+| 1 | 13511 | 3125 | 14493 | 31129 |
+| 2 | 11487 | 177 | 10694 | 22358 |
+| 3 | 12031 | 2949 | 11316 | 26296 |
+| 4 | 10739 | 2897 | 11446 | 25082 |
+| 5 | 3046 | 595 | 2940 | 6581 |
+| 6 | 7630 | 109 | 8686 | 16425 |
+| 7 | 1892 | 1974 | 3968 | 7834 |
+| 8 | 2881 | 72 | 3065 | 6018 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 65237 | 12066 | 68504 | 0 |
+
+# Repartition
+
+| | CENTRAL_CHINA | FINAL | NORTH_CHINA | SOUTH_CHINA |
+|:---------|:----------------|:--------------|:--------------|:--------------|
+| images | 2655 (25.6%) | 2685 (25.9%) | 2474 (23.9%) | 2555 (24.6%) |
+| robots | 15112 (26.3%) | 17091 (29.7%) | 13000 (22.6%) | 12344 (21.5%) |
+| watchers | 2373 (25.5%) | 2572 (27.6%) | 2497 (26.8%) | 1871 (20.1%) |
+| bases | 1228 (23.4%) | 1583 (30.2%) | 1265 (24.1%) | 1173 (22.3%) |
+
diff --git a/dataset/dji_roco/robomaster_Central China Regional Competition/dset_CENTRAL_CHINA_report.md b/dataset/dji_roco/robomaster_Central China Regional Competition/dset_CENTRAL_CHINA_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..e1ff71a3ae9e820dd0efea3156bd61b313c26b18
--- /dev/null
+++ b/dataset/dji_roco/robomaster_Central China Regional Competition/dset_CENTRAL_CHINA_report.md
@@ -0,0 +1,23 @@
+# Dataset CENTRAL_CHINA
+
+2655 images, with:
+
+ - 1228 bases
+ - 2373 watchers
+ - 0 runes
+ - 15112 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 48 | 0 | 61 | 109 |
+| 1 | 3864 | 716 | 4389 | 8969 |
+| 2 | 2973 | 55 | 3395 | 6423 |
+| 3 | 3586 | 632 | 3874 | 8092 |
+| 4 | 2974 | 615 | 3731 | 7320 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1368 | 109 | 2331 | 3808 |
+| 7 | 536 | 479 | 1233 | 2248 |
+| 8 | 762 | 3 | 955 | 1720 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 16111 | 2609 | 19969 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_Central China Regional Competition/dset_CentralChina_report.md b/dataset/dji_roco/robomaster_Central China Regional Competition/dset_CentralChina_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..24b0d23f6824e92249f9ddacee7262d8fb036746
--- /dev/null
+++ b/dataset/dji_roco/robomaster_Central China Regional Competition/dset_CentralChina_report.md
@@ -0,0 +1,23 @@
+# Dataset CentralChina
+
+2655 images, with:
+
+ - 1228 bases
+ - 2373 watchers
+ - 0 runes
+ - 15112 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 48 | 0 | 61 | 109 |
+| 1 | 3864 | 716 | 4389 | 8969 |
+| 2 | 2973 | 55 | 3395 | 6423 |
+| 3 | 3586 | 632 | 3874 | 8092 |
+| 4 | 2974 | 615 | 3731 | 7320 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1368 | 109 | 2331 | 3808 |
+| 7 | 536 | 479 | 1233 | 2248 |
+| 8 | 762 | 3 | 955 | 1720 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 16111 | 2609 | 19969 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_Central China Regional Competition/report.md b/dataset/dji_roco/robomaster_Central China Regional Competition/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..91da9702a388f227d09f7519ab0771a0673edba9
--- /dev/null
+++ b/dataset/dji_roco/robomaster_Central China Regional Competition/report.md
@@ -0,0 +1,23 @@
+# Dataset CENTRAL_CHINA
+
+2655 images, with:
+
+ - 2655 images
+ - 15112 robots
+ - 2373 watchers
+ - 1228 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 48 | 0 | 61 | 109 |
+| 1 | 3864 | 716 | 4389 | 8969 |
+| 2 | 2973 | 55 | 3395 | 6423 |
+| 3 | 3586 | 632 | 3874 | 8092 |
+| 4 | 2974 | 615 | 3731 | 7320 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1368 | 109 | 2331 | 3808 |
+| 7 | 536 | 479 | 1233 | 2248 |
+| 8 | 762 | 3 | 955 | 1720 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 16111 | 2609 | 19969 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_Final Tournament/dset_Final_report.md b/dataset/dji_roco/robomaster_Final Tournament/dset_Final_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..e4c69916fa566bfcfbfb02dadbc7f1b534bab055
--- /dev/null
+++ b/dataset/dji_roco/robomaster_Final Tournament/dset_Final_report.md
@@ -0,0 +1,23 @@
+# Dataset FINAL
+
+2685 images, with:
+
+ - 1583 bases
+ - 2572 watchers
+ - 0 runes
+ - 17091 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 597 | 125 | 695 | 1417 |
+| 1 | 3573 | 642 | 3652 | 7867 |
+| 2 | 2931 | 94 | 2442 | 5467 |
+| 3 | 2924 | 802 | 2664 | 6390 |
+| 4 | 2488 | 840 | 2496 | 5824 |
+| 5 | 3046 | 595 | 2940 | 6581 |
+| 6 | 2461 | 0 | 2481 | 4942 |
+| 7 | 443 | 578 | 839 | 1860 |
+| 8 | 1177 | 51 | 553 | 1781 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 19640 | 3727 | 18762 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_Final Tournament/report.md b/dataset/dji_roco/robomaster_Final Tournament/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..11b1c8e81ba9f2ab299a9cee93cd0b418f1cd9ff
--- /dev/null
+++ b/dataset/dji_roco/robomaster_Final Tournament/report.md
@@ -0,0 +1,23 @@
+# Dataset FINAL
+
+2685 images, with:
+
+ - 2685 images
+ - 17091 robots
+ - 2572 watchers
+ - 1583 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 597 | 125 | 695 | 1417 |
+| 1 | 3573 | 642 | 3652 | 7867 |
+| 2 | 2931 | 94 | 2442 | 5467 |
+| 3 | 2924 | 802 | 2664 | 6390 |
+| 4 | 2488 | 840 | 2496 | 5824 |
+| 5 | 3046 | 595 | 2940 | 6581 |
+| 6 | 2461 | 0 | 2481 | 4942 |
+| 7 | 443 | 578 | 839 | 1860 |
+| 8 | 1177 | 51 | 553 | 1781 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 19640 | 3727 | 18762 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_North China Regional Competition/dset_NORTH_CHINA_report.md b/dataset/dji_roco/robomaster_North China Regional Competition/dset_NORTH_CHINA_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..12d4c857e8de3ca6f265e062faf81b490eb17018
--- /dev/null
+++ b/dataset/dji_roco/robomaster_North China Regional Competition/dset_NORTH_CHINA_report.md
@@ -0,0 +1,23 @@
+# Dataset NORTH_CHINA
+
+2474 images, with:
+
+ - 1265 bases
+ - 2497 watchers
+ - 0 runes
+ - 13000 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1294 | 15 | 1024 | 2333 |
+| 1 | 2961 | 516 | 2844 | 6321 |
+| 2 | 2870 | 3 | 2517 | 5390 |
+| 3 | 2807 | 877 | 2242 | 5926 |
+| 4 | 2388 | 311 | 2418 | 5117 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1882 | 0 | 1834 | 3716 |
+| 7 | 391 | 341 | 1216 | 1948 |
+| 8 | 750 | 13 | 436 | 1199 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 15343 | 2076 | 14531 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_North China Regional Competition/dset_NorthChina_report.md b/dataset/dji_roco/robomaster_North China Regional Competition/dset_NorthChina_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..ff5bcdbb7150c81a7d307f3cbbd3569041781e36
--- /dev/null
+++ b/dataset/dji_roco/robomaster_North China Regional Competition/dset_NorthChina_report.md
@@ -0,0 +1,23 @@
+# Dataset NorthChina
+
+2474 images, with:
+
+ - 1265 bases
+ - 2497 watchers
+ - 0 runes
+ - 13000 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1294 | 15 | 1024 | 2333 |
+| 1 | 2961 | 516 | 2844 | 6321 |
+| 2 | 2870 | 3 | 2517 | 5390 |
+| 3 | 2807 | 877 | 2242 | 5926 |
+| 4 | 2388 | 311 | 2418 | 5117 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1882 | 0 | 1834 | 3716 |
+| 7 | 391 | 341 | 1216 | 1948 |
+| 8 | 750 | 13 | 436 | 1199 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 15343 | 2076 | 14531 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_North China Regional Competition/report.md b/dataset/dji_roco/robomaster_North China Regional Competition/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..5302feae3cf7e599be37789bde656df7915605e8
--- /dev/null
+++ b/dataset/dji_roco/robomaster_North China Regional Competition/report.md
@@ -0,0 +1,23 @@
+# Dataset NORTH_CHINA
+
+2474 images, with:
+
+ - 2474 images
+ - 13000 robots
+ - 2497 watchers
+ - 1265 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1294 | 15 | 1024 | 2333 |
+| 1 | 2961 | 516 | 2844 | 6321 |
+| 2 | 2870 | 3 | 2517 | 5390 |
+| 3 | 2807 | 877 | 2242 | 5926 |
+| 4 | 2388 | 311 | 2418 | 5117 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1882 | 0 | 1834 | 3716 |
+| 7 | 391 | 341 | 1216 | 1948 |
+| 8 | 750 | 13 | 436 | 1199 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 15343 | 2076 | 14531 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_South China Regional Competition/dset_SOUTH_CHINA_report.md b/dataset/dji_roco/robomaster_South China Regional Competition/dset_SOUTH_CHINA_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..b22291eaeeb489ccdaf3fd61e5f8e56723db6cd5
--- /dev/null
+++ b/dataset/dji_roco/robomaster_South China Regional Competition/dset_SOUTH_CHINA_report.md
@@ -0,0 +1,23 @@
+# Dataset SOUTH_CHINA
+
+2555 images, with:
+
+ - 1173 bases
+ - 1871 watchers
+ - 0 runes
+ - 12344 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 81 | 28 | 116 | 225 |
+| 1 | 3113 | 1251 | 3608 | 7972 |
+| 2 | 2713 | 25 | 2340 | 5078 |
+| 3 | 2714 | 638 | 2536 | 5888 |
+| 4 | 2889 | 1131 | 2801 | 6821 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1919 | 0 | 2040 | 3959 |
+| 7 | 522 | 576 | 680 | 1778 |
+| 8 | 192 | 5 | 1121 | 1318 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 14143 | 3654 | 15242 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_South China Regional Competition/dset_SouthChina_report.md b/dataset/dji_roco/robomaster_South China Regional Competition/dset_SouthChina_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..419905004b9fb2a67ffac71830138bb1400fe7c6
--- /dev/null
+++ b/dataset/dji_roco/robomaster_South China Regional Competition/dset_SouthChina_report.md
@@ -0,0 +1,23 @@
+# Dataset SouthChina
+
+2555 images, with:
+
+ - 1173 bases
+ - 1871 watchers
+ - 0 runes
+ - 12344 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 81 | 28 | 116 | 225 |
+| 1 | 3113 | 1251 | 3608 | 7972 |
+| 2 | 2713 | 25 | 2340 | 5078 |
+| 3 | 2714 | 638 | 2536 | 5888 |
+| 4 | 2889 | 1131 | 2801 | 6821 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1919 | 0 | 2040 | 3959 |
+| 7 | 522 | 576 | 680 | 1778 |
+| 8 | 192 | 5 | 1121 | 1318 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 14143 | 3654 | 15242 | 0 |
+
diff --git a/dataset/dji_roco/robomaster_South China Regional Competition/report.md b/dataset/dji_roco/robomaster_South China Regional Competition/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..151f17c72aaa806123609ba084b7784c5b353b99
--- /dev/null
+++ b/dataset/dji_roco/robomaster_South China Regional Competition/report.md
@@ -0,0 +1,23 @@
+# Dataset SOUTH_CHINA
+
+2555 images, with:
+
+ - 2555 images
+ - 12344 robots
+ - 1871 watchers
+ - 1173 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 81 | 28 | 116 | 225 |
+| 1 | 3113 | 1251 | 3608 | 7972 |
+| 2 | 2713 | 25 | 2340 | 5078 |
+| 3 | 2714 | 638 | 2536 | 5888 |
+| 4 | 2889 | 1131 | 2801 | 6821 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1919 | 0 | 2040 | 3959 |
+| 7 | 522 | 576 | 680 | 1778 |
+| 8 | 192 | 5 | 1121 | 1318 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 14143 | 3654 | 15242 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/central_china/dset_CentralChinaZoomedV2_report.md b/dataset/dji_roco_zoomed_v2/central_china/dset_CentralChinaZoomedV2_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..171b538594052a575cee413a51759f96eb885d80
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/central_china/dset_CentralChinaZoomedV2_report.md
@@ -0,0 +1,23 @@
+# Dataset CentralChinaZoomedV2
+
+5307 images, with:
+
+ - 1132 bases
+ - 2120 watchers
+ - 0 runes
+ - 14341 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 35 | 0 | 54 | 89 |
+| 1 | 3801 | 696 | 4216 | 8713 |
+| 2 | 3001 | 56 | 3411 | 6468 |
+| 3 | 3358 | 611 | 3629 | 7598 |
+| 4 | 2714 | 523 | 3544 | 6781 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1367 | 109 | 2180 | 3656 |
+| 7 | 510 | 461 | 1088 | 2059 |
+| 8 | 762 | 3 | 907 | 1672 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 15548 | 2459 | 19029 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/central_china/dset_CentralChinaZoomed_report.md b/dataset/dji_roco_zoomed_v2/central_china/dset_CentralChinaZoomed_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..e917eb9c57c00967f6a33a99c59859bb4865eb9c
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/central_china/dset_CentralChinaZoomed_report.md
@@ -0,0 +1,23 @@
+# Dataset CentralChinaZoomed
+
+5307 images, with:
+
+ - 1132 bases
+ - 2120 watchers
+ - 0 runes
+ - 14341 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 35 | 0 | 54 | 89 |
+| 1 | 3801 | 696 | 4216 | 8713 |
+| 2 | 3001 | 56 | 3411 | 6468 |
+| 3 | 3358 | 611 | 3629 | 7598 |
+| 4 | 2714 | 523 | 3544 | 6781 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1367 | 109 | 2180 | 3656 |
+| 7 | 510 | 461 | 1088 | 2059 |
+| 8 | 762 | 3 | 907 | 1672 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 15548 | 2459 | 19029 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/central_china/report.md b/dataset/dji_roco_zoomed_v2/central_china/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..21d0a4a6a149aa6c2fe8d7079458257f34853fce
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/central_china/report.md
@@ -0,0 +1,23 @@
+# Dataset CentralChinaZoomed
+
+5307 images, with:
+
+ - 5307 images
+ - 14341 robots
+ - 2120 watchers
+ - 1132 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 35 | 0 | 54 | 89 |
+| 1 | 3801 | 696 | 4216 | 8713 |
+| 2 | 3001 | 56 | 3411 | 6468 |
+| 3 | 3358 | 611 | 3629 | 7598 |
+| 4 | 2714 | 523 | 3544 | 6781 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1367 | 109 | 2180 | 3656 |
+| 7 | 510 | 461 | 1088 | 2059 |
+| 8 | 762 | 3 | 907 | 1672 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 15548 | 2459 | 19029 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/dset_DJIZoomed_report.md b/dataset/dji_roco_zoomed_v2/dset_DJIZoomed_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..40f2ba2f8c8d0b8e473a0f66f454387e3d123b62
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/dset_DJIZoomed_report.md
@@ -0,0 +1,23 @@
+# Dataset DJIZoomed
+
+21313 images, with:
+
+ - 4807 bases
+ - 8291 watchers
+ - 0 runes
+ - 52684 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1877 | 153 | 1710 | 3740 |
+| 1 | 12951 | 2987 | 13687 | 29625 |
+| 2 | 11367 | 166 | 10577 | 22110 |
+| 3 | 11024 | 2612 | 10249 | 23885 |
+| 4 | 9848 | 2598 | 10464 | 22910 |
+| 5 | 2748 | 558 | 2620 | 5926 |
+| 6 | 7626 | 109 | 8158 | 15893 |
+| 7 | 1758 | 1846 | 3536 | 7140 |
+| 8 | 2875 | 72 | 2971 | 5918 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 62074 | 11101 | 63972 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/final/dset_FinalZoomedV2_report.md b/dataset/dji_roco_zoomed_v2/final/dset_FinalZoomedV2_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..81908a194addbc9ae7e9ced7b64706e3d03c2bf1
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/final/dset_FinalZoomedV2_report.md
@@ -0,0 +1,23 @@
+# Dataset FinalZoomedV2
+
+5260 images, with:
+
+ - 1496 bases
+ - 2368 watchers
+ - 0 runes
+ - 15314 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 540 | 115 | 609 | 1264 |
+| 1 | 3336 | 550 | 3381 | 7267 |
+| 2 | 2846 | 86 | 2380 | 5312 |
+| 3 | 2531 | 698 | 2312 | 5541 |
+| 4 | 2313 | 787 | 2325 | 5425 |
+| 5 | 2748 | 558 | 2620 | 5926 |
+| 6 | 2461 | 0 | 2365 | 4826 |
+| 7 | 436 | 537 | 789 | 1762 |
+| 8 | 1176 | 50 | 546 | 1772 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 18387 | 3381 | 17327 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/final/dset_FinalZoomed_report.md b/dataset/dji_roco_zoomed_v2/final/dset_FinalZoomed_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..d49d8167a83d26b46aa47eeb37714a60720dfde1
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/final/dset_FinalZoomed_report.md
@@ -0,0 +1,23 @@
+# Dataset FinalZoomed
+
+5260 images, with:
+
+ - 1496 bases
+ - 2368 watchers
+ - 0 runes
+ - 15314 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 540 | 115 | 609 | 1264 |
+| 1 | 3336 | 550 | 3381 | 7267 |
+| 2 | 2846 | 86 | 2380 | 5312 |
+| 3 | 2531 | 698 | 2312 | 5541 |
+| 4 | 2313 | 787 | 2325 | 5425 |
+| 5 | 2748 | 558 | 2620 | 5926 |
+| 6 | 2461 | 0 | 2365 | 4826 |
+| 7 | 436 | 537 | 789 | 1762 |
+| 8 | 1176 | 50 | 546 | 1772 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 18387 | 3381 | 17327 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/final/report.md b/dataset/dji_roco_zoomed_v2/final/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..595416424d4b14bbcd9833c65f106708f9ed58e5
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/final/report.md
@@ -0,0 +1,23 @@
+# Dataset FinalZoomed
+
+5260 images, with:
+
+ - 5260 images
+ - 15314 robots
+ - 2368 watchers
+ - 1496 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 540 | 115 | 609 | 1264 |
+| 1 | 3336 | 550 | 3381 | 7267 |
+| 2 | 2846 | 86 | 2380 | 5312 |
+| 3 | 2531 | 698 | 2312 | 5541 |
+| 4 | 2313 | 787 | 2325 | 5425 |
+| 5 | 2748 | 558 | 2620 | 5926 |
+| 6 | 2461 | 0 | 2365 | 4826 |
+| 7 | 436 | 537 | 789 | 1762 |
+| 8 | 1176 | 50 | 546 | 1772 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 18387 | 3381 | 17327 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/north_china/dset_NorthChinaZoomedV2_report.md b/dataset/dji_roco_zoomed_v2/north_china/dset_NorthChinaZoomedV2_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..a5c09b374b73b9985e3a2ce75f1d048514dc82a0
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/north_china/dset_NorthChinaZoomedV2_report.md
@@ -0,0 +1,23 @@
+# Dataset NorthChinaZoomedV2
+
+5474 images, with:
+
+ - 1042 bases
+ - 2265 watchers
+ - 0 runes
+ - 11962 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1226 | 15 | 946 | 2187 |
+| 1 | 2886 | 405 | 2665 | 5956 |
+| 2 | 2896 | 5 | 2497 | 5398 |
+| 3 | 2599 | 759 | 2139 | 5497 |
+| 4 | 2234 | 267 | 2148 | 4649 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1859 | 0 | 1585 | 3444 |
+| 7 | 341 | 357 | 1078 | 1776 |
+| 8 | 745 | 14 | 397 | 1156 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 14786 | 1822 | 13455 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/north_china/dset_NorthChinaZoomed_report.md b/dataset/dji_roco_zoomed_v2/north_china/dset_NorthChinaZoomed_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..dfd712a9711a3ef7f6c7df0bbec7b0bdd06fea74
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/north_china/dset_NorthChinaZoomed_report.md
@@ -0,0 +1,23 @@
+# Dataset NorthChinaZoomed
+
+5474 images, with:
+
+ - 1042 bases
+ - 2265 watchers
+ - 0 runes
+ - 11962 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1226 | 15 | 946 | 2187 |
+| 1 | 2886 | 405 | 2665 | 5956 |
+| 2 | 2896 | 5 | 2497 | 5398 |
+| 3 | 2599 | 759 | 2139 | 5497 |
+| 4 | 2234 | 267 | 2148 | 4649 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1859 | 0 | 1585 | 3444 |
+| 7 | 341 | 357 | 1078 | 1776 |
+| 8 | 745 | 14 | 397 | 1156 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 14786 | 1822 | 13455 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/north_china/report.md b/dataset/dji_roco_zoomed_v2/north_china/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..1d7fa156aee00423a8866965d9b25e6fc09f0767
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/north_china/report.md
@@ -0,0 +1,23 @@
+# Dataset NorthChinaZoomed
+
+5474 images, with:
+
+ - 5474 images
+ - 11962 robots
+ - 2265 watchers
+ - 1042 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1226 | 15 | 946 | 2187 |
+| 1 | 2886 | 405 | 2665 | 5956 |
+| 2 | 2896 | 5 | 2497 | 5398 |
+| 3 | 2599 | 759 | 2139 | 5497 |
+| 4 | 2234 | 267 | 2148 | 4649 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1859 | 0 | 1585 | 3444 |
+| 7 | 341 | 357 | 1078 | 1776 |
+| 8 | 745 | 14 | 397 | 1156 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 14786 | 1822 | 13455 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/report.md b/dataset/dji_roco_zoomed_v2/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..03789a112d620be6f37c48eca6a4ce7860592190
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/report.md
@@ -0,0 +1,32 @@
+# Global report
+
+21313 images, with:
+
+ - 21313 images
+ - 52684 robots
+ - 8291 watchers
+ - 4807 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1877 | 153 | 1710 | 3740 |
+| 1 | 12951 | 2987 | 13687 | 29625 |
+| 2 | 11367 | 166 | 10577 | 22110 |
+| 3 | 11024 | 2612 | 10249 | 23885 |
+| 4 | 9848 | 2598 | 10464 | 22910 |
+| 5 | 2748 | 558 | 2620 | 5926 |
+| 6 | 7626 | 109 | 8158 | 15893 |
+| 7 | 1758 | 1846 | 3536 | 7140 |
+| 8 | 2875 | 72 | 2971 | 5918 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 62074 | 11101 | 63972 | 0 |
+
+# Repartition
+
+| | CentralChinaZoomed | FinalZoomed | NorthChinaZoomed | SouthChinaZoomed |
+|:---------|:---------------------|:--------------|:-------------------|:-------------------|
+| images | 5307 (24.9%) | 5260 (24.7%) | 5474 (25.7%) | 5272 (24.7%) |
+| robots | 14341 (27.2%) | 15314 (29.1%) | 11962 (22.7%) | 11067 (21.0%) |
+| watchers | 2120 (25.6%) | 2368 (28.6%) | 2265 (27.3%) | 1538 (18.6%) |
+| bases | 1132 (23.5%) | 1496 (31.1%) | 1042 (21.7%) | 1137 (23.7%) |
+
diff --git a/dataset/dji_roco_zoomed_v2/south_china/dset_SouthChinaZoomedV2_report.md b/dataset/dji_roco_zoomed_v2/south_china/dset_SouthChinaZoomedV2_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..d8da5d582f78a83cea4e4c7405ca016a76f72d06
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/south_china/dset_SouthChinaZoomedV2_report.md
@@ -0,0 +1,23 @@
+# Dataset SouthChinaZoomedV2
+
+5272 images, with:
+
+ - 1137 bases
+ - 1538 watchers
+ - 0 runes
+ - 11067 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 76 | 23 | 101 | 200 |
+| 1 | 2928 | 1336 | 3425 | 7689 |
+| 2 | 2624 | 19 | 2289 | 4932 |
+| 3 | 2536 | 544 | 2169 | 5249 |
+| 4 | 2587 | 1021 | 2447 | 6055 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1939 | 0 | 2028 | 3967 |
+| 7 | 471 | 491 | 581 | 1543 |
+| 8 | 192 | 5 | 1121 | 1318 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 13353 | 3439 | 14161 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/south_china/dset_SouthChinaZoomed_report.md b/dataset/dji_roco_zoomed_v2/south_china/dset_SouthChinaZoomed_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..89c8e1d836c9a5c6a702c5976f196e2ac9c27b2c
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/south_china/dset_SouthChinaZoomed_report.md
@@ -0,0 +1,23 @@
+# Dataset SouthChinaZoomed
+
+5272 images, with:
+
+ - 1137 bases
+ - 1538 watchers
+ - 0 runes
+ - 11067 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 76 | 23 | 101 | 200 |
+| 1 | 2928 | 1336 | 3425 | 7689 |
+| 2 | 2624 | 19 | 2289 | 4932 |
+| 3 | 2536 | 544 | 2169 | 5249 |
+| 4 | 2587 | 1021 | 2447 | 6055 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1939 | 0 | 2028 | 3967 |
+| 7 | 471 | 491 | 581 | 1543 |
+| 8 | 192 | 5 | 1121 | 1318 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 13353 | 3439 | 14161 | 0 |
+
diff --git a/dataset/dji_roco_zoomed_v2/south_china/report.md b/dataset/dji_roco_zoomed_v2/south_china/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..d7a1a4a906029ef1d72a45dd7aee68f1b52cc6aa
--- /dev/null
+++ b/dataset/dji_roco_zoomed_v2/south_china/report.md
@@ -0,0 +1,23 @@
+# Dataset SouthChinaZoomed
+
+5272 images, with:
+
+ - 5272 images
+ - 11067 robots
+ - 1538 watchers
+ - 1137 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 76 | 23 | 101 | 200 |
+| 1 | 2928 | 1336 | 3425 | 7689 |
+| 2 | 2624 | 19 | 2289 | 4932 |
+| 3 | 2536 | 544 | 2169 | 5249 |
+| 4 | 2587 | 1021 | 2447 | 6055 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 1939 | 0 | 2028 | 3967 |
+| 7 | 471 | 491 | 581 | 1543 |
+| 8 | 192 | 5 | 1121 | 1318 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 13353 | 3439 | 14161 | 0 |
+
diff --git a/dataset/twitch/v2/.gitignore b/dataset/twitch/v2/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..571c0679ccbe25620e5829b2b84c0bda05638e79
--- /dev/null
+++ b/dataset/twitch/v2/.gitignore
@@ -0,0 +1,6 @@
+**/*.jpg
+**/*.png
+**/*.jpeg
+**/*.lock
+**/*.xml
+temp.txt
\ No newline at end of file
diff --git a/dataset/twitch/v2/470149066/report.md b/dataset/twitch/v2/470149066/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..0656ab00f81f1d403ea87b5b97111a4220fbd715
--- /dev/null
+++ b/dataset/twitch/v2/470149066/report.md
@@ -0,0 +1,23 @@
+# Dataset T470149066
+
+822 images, with:
+
+ - 822 images
+ - 785 robots
+ - 321 watchers
+ - 488 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 7 | 0 | 37 | 44 |
+| 1 | 104 | 13 | 133 | 250 |
+| 2 | 76 | 14 | 152 | 242 |
+| 3 | 138 | 7 | 69 | 214 |
+| 4 | 120 | 45 | 116 | 281 |
+| 5 | 29 | 11 | 121 | 161 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 149 | 52 | 60 | 261 |
+| 8 | 446 | 0 | 154 | 600 |
+| 9 | 4 | 0 | 0 | 4 |
+| total | 1073 | 142 | 842 | 0 |
+
diff --git a/dataset/twitch/v2/470149568/digits/.changes b/dataset/twitch/v2/470149568/digits/.changes
new file mode 100644
index 0000000000000000000000000000000000000000..0822a8dd305f383a7335e7cc75cb6d742f3ef5c1
Binary files /dev/null and b/dataset/twitch/v2/470149568/digits/.changes differ
diff --git a/dataset/twitch/v2/470149568/report.md b/dataset/twitch/v2/470149568/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..8b3c8b6d166ea8e1a78c3f972dca17f6dd753e3d
--- /dev/null
+++ b/dataset/twitch/v2/470149568/report.md
@@ -0,0 +1,23 @@
+# Dataset T470149568
+
+389 images, with:
+
+ - 389 images
+ - 332 robots
+ - 196 watchers
+ - 253 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 8 | 0 | 7 | 15 |
+| 1 | 55 | 14 | 35 | 104 |
+| 2 | 64 | 2 | 22 | 88 |
+| 3 | 13 | 11 | 26 | 50 |
+| 4 | 44 | 11 | 36 | 91 |
+| 5 | 100 | 3 | 34 | 137 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 48 | 66 | 71 | 185 |
+| 8 | 103 | 1 | 185 | 289 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 435 | 108 | 416 | 0 |
+
diff --git a/dataset/twitch/v2/470150052/digits/.changes b/dataset/twitch/v2/470150052/digits/.changes
new file mode 100644
index 0000000000000000000000000000000000000000..94b33fda17ee396e0e1430c343981b673fd2e690
Binary files /dev/null and b/dataset/twitch/v2/470150052/digits/.changes differ
diff --git a/dataset/twitch/v2/470150052/report.md b/dataset/twitch/v2/470150052/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..9f659467b9f7a25a0069953217ba893979ada1cf
--- /dev/null
+++ b/dataset/twitch/v2/470150052/report.md
@@ -0,0 +1,23 @@
+# Dataset T470150052
+
+154 images, with:
+
+ - 154 images
+ - 81 robots
+ - 89 watchers
+ - 135 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 5 | 0 | 0 | 5 |
+| 1 | 19 | 0 | 0 | 19 |
+| 2 | 1 | 0 | 4 | 5 |
+| 3 | 0 | 0 | 1 | 1 |
+| 4 | 22 | 0 | 0 | 22 |
+| 5 | 32 | 2 | 22 | 56 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 32 | 17 | 35 | 84 |
+| 8 | 117 | 0 | 48 | 165 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 228 | 19 | 110 | 0 |
+
diff --git a/dataset/twitch/v2/470151286/digits/.changes b/dataset/twitch/v2/470151286/digits/.changes
new file mode 100644
index 0000000000000000000000000000000000000000..5c3c97e716ee5fa341f7c4fea59435a671570997
Binary files /dev/null and b/dataset/twitch/v2/470151286/digits/.changes differ
diff --git a/dataset/twitch/v2/470151286/report.md b/dataset/twitch/v2/470151286/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..7f43764ba867426a9e0afa196581844cf31e26f1
--- /dev/null
+++ b/dataset/twitch/v2/470151286/report.md
@@ -0,0 +1,23 @@
+# Dataset T470151286
+
+454 images, with:
+
+ - 454 images
+ - 295 robots
+ - 189 watchers
+ - 337 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 10 | 0 | 19 | 29 |
+| 1 | 42 | 10 | 45 | 97 |
+| 2 | 28 | 0 | 22 | 50 |
+| 3 | 17 | 7 | 17 | 41 |
+| 4 | 29 | 3 | 39 | 71 |
+| 5 | 43 | 4 | 31 | 78 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 80 | 44 | 49 | 173 |
+| 8 | 161 | 7 | 199 | 367 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 410 | 75 | 421 | 0 |
+
diff --git a/dataset/twitch/v2/470152289/digits/.changes b/dataset/twitch/v2/470152289/digits/.changes
new file mode 100644
index 0000000000000000000000000000000000000000..6cca4879d214de53c2cf4b3a9d0e95355d4d3339
Binary files /dev/null and b/dataset/twitch/v2/470152289/digits/.changes differ
diff --git a/dataset/twitch/v2/470152289/report.md b/dataset/twitch/v2/470152289/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..792fbec651971f1ea76f9076d6ac8b44502cb7fc
--- /dev/null
+++ b/dataset/twitch/v2/470152289/report.md
@@ -0,0 +1,23 @@
+# Dataset T470152289
+
+271 images, with:
+
+ - 271 images
+ - 171 robots
+ - 145 watchers
+ - 219 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 4 | 0 | 1 | 5 |
+| 1 | 22 | 0 | 14 | 36 |
+| 2 | 29 | 0 | 7 | 36 |
+| 3 | 40 | 2 | 16 | 58 |
+| 4 | 5 | 6 | 10 | 21 |
+| 5 | 12 | 5 | 6 | 23 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 57 | 36 | 30 | 123 |
+| 8 | 141 | 1 | 77 | 219 |
+| 9 | 0 | 0 | 1 | 1 |
+| total | 310 | 50 | 162 | 0 |
+
diff --git a/dataset/twitch/v2/470152730/digits/.changes b/dataset/twitch/v2/470152730/digits/.changes
new file mode 100644
index 0000000000000000000000000000000000000000..531ffd9c3d7cebe4b3101c451d869cdfdbb90531
Binary files /dev/null and b/dataset/twitch/v2/470152730/digits/.changes differ
diff --git a/dataset/twitch/v2/470152730/report.md b/dataset/twitch/v2/470152730/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..afdd6860c195eff6ba5ee058db3b0c6c370d6aeb
--- /dev/null
+++ b/dataset/twitch/v2/470152730/report.md
@@ -0,0 +1,23 @@
+# Dataset T470152730
+
+116 images, with:
+
+ - 116 images
+ - 76 robots
+ - 59 watchers
+ - 108 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 4 | 0 | 1 | 5 |
+| 1 | 3 | 12 | 0 | 15 |
+| 2 | 9 | 0 | 10 | 19 |
+| 3 | 0 | 6 | 4 | 10 |
+| 4 | 6 | 10 | 12 | 28 |
+| 5 | 15 | 0 | 1 | 16 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 25 | 15 | 12 | 52 |
+| 8 | 80 | 1 | 38 | 119 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 142 | 44 | 78 | 0 |
+
diff --git a/dataset/twitch/v2/470152838/digits/.changes b/dataset/twitch/v2/470152838/digits/.changes
new file mode 100644
index 0000000000000000000000000000000000000000..009cbf4f286ae0e94a87a9d691114508bcafc600
Binary files /dev/null and b/dataset/twitch/v2/470152838/digits/.changes differ
diff --git a/dataset/twitch/v2/470152838/report.md b/dataset/twitch/v2/470152838/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..1232e0d0e2969ce36251789af043244a56579f48
--- /dev/null
+++ b/dataset/twitch/v2/470152838/report.md
@@ -0,0 +1,23 @@
+# Dataset T470152838
+
+116 images, with:
+
+ - 116 images
+ - 46 robots
+ - 56 watchers
+ - 89 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 2 | 0 | 0 | 2 |
+| 1 | 0 | 0 | 2 | 2 |
+| 2 | 20 | 1 | 14 | 35 |
+| 3 | 0 | 0 | 0 | 0 |
+| 4 | 5 | 0 | 0 | 5 |
+| 5 | 8 | 0 | 0 | 8 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 21 | 13 | 22 | 56 |
+| 8 | 52 | 2 | 37 | 91 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 108 | 16 | 75 | 0 |
+
diff --git a/dataset/twitch/v2/470152932/report.md b/dataset/twitch/v2/470152932/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..5e23dcd8b5133954a51ba063096896e3b9064e08
--- /dev/null
+++ b/dataset/twitch/v2/470152932/report.md
@@ -0,0 +1,23 @@
+# Dataset T470152932
+
+88 images, with:
+
+ - 88 images
+ - 9 robots
+ - 17 watchers
+ - 77 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 0 | 0 | 0 | 0 |
+| 1 | 0 | 0 | 0 | 0 |
+| 2 | 0 | 0 | 0 | 0 |
+| 3 | 1 | 0 | 2 | 3 |
+| 4 | 10 | 0 | 0 | 10 |
+| 5 | 0 | 0 | 0 | 0 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 0 | 1 | 16 | 17 |
+| 8 | 26 | 0 | 47 | 73 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 37 | 1 | 65 | 0 |
+
diff --git a/dataset/twitch/v2/470153081/digits/.changes b/dataset/twitch/v2/470153081/digits/.changes
new file mode 100644
index 0000000000000000000000000000000000000000..98e6839eae3d039c86cafc6754dd168d89e8ad5d
Binary files /dev/null and b/dataset/twitch/v2/470153081/digits/.changes differ
diff --git a/dataset/twitch/v2/470153081/report.md b/dataset/twitch/v2/470153081/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..120559cfa90307e4977febed7dbfcb563f052658
--- /dev/null
+++ b/dataset/twitch/v2/470153081/report.md
@@ -0,0 +1,23 @@
+# Dataset T470153081
+
+76 images, with:
+
+ - 76 images
+ - 29 robots
+ - 34 watchers
+ - 41 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 1 | 0 | 1 | 2 |
+| 1 | 6 | 0 | 4 | 10 |
+| 2 | 4 | 0 | 2 | 6 |
+| 3 | 3 | 0 | 0 | 3 |
+| 4 | 0 | 0 | 2 | 2 |
+| 5 | 2 | 0 | 3 | 5 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 13 | 3 | 16 | 32 |
+| 8 | 17 | 0 | 25 | 42 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 46 | 3 | 53 | 0 |
+
diff --git a/dataset/twitch/v2/470158483/digits/.changes b/dataset/twitch/v2/470158483/digits/.changes
new file mode 100644
index 0000000000000000000000000000000000000000..22eec15bcfb12e76fe31b9b3e70f3fd11bf06e3e
Binary files /dev/null and b/dataset/twitch/v2/470158483/digits/.changes differ
diff --git a/dataset/twitch/v2/470158483/report.md b/dataset/twitch/v2/470158483/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..5e8e0dbf5ea2f87da3bb684b66689f80992ff5dc
--- /dev/null
+++ b/dataset/twitch/v2/470158483/report.md
@@ -0,0 +1,23 @@
+# Dataset T470158483
+
+452 images, with:
+
+ - 452 images
+ - 309 robots
+ - 239 watchers
+ - 339 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 13 | 0 | 2 | 15 |
+| 1 | 51 | 0 | 54 | 105 |
+| 2 | 46 | 2 | 54 | 102 |
+| 3 | 27 | 0 | 34 | 61 |
+| 4 | 42 | 3 | 23 | 68 |
+| 5 | 60 | 0 | 21 | 81 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 66 | 9 | 153 | 228 |
+| 8 | 97 | 0 | 220 | 317 |
+| 9 | 0 | 0 | 0 | 0 |
+| total | 402 | 14 | 561 | 0 |
+
diff --git a/dataset/twitch/v2/dset_Twitch_report.md b/dataset/twitch/v2/dset_Twitch_report.md
new file mode 100644
index 0000000000000000000000000000000000000000..4668c2f14a932dccdbe133329aa0eb7980ab266a
--- /dev/null
+++ b/dataset/twitch/v2/dset_Twitch_report.md
@@ -0,0 +1,23 @@
+# Dataset Twitch
+
+2938 images, with:
+
+ - 2086 bases
+ - 1345 watchers
+ - 0 runes
+ - 2133 robots
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 54 | 0 | 68 | 122 |
+| 1 | 302 | 49 | 287 | 638 |
+| 2 | 277 | 19 | 287 | 583 |
+| 3 | 239 | 33 | 169 | 441 |
+| 4 | 283 | 78 | 238 | 599 |
+| 5 | 301 | 25 | 239 | 565 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 491 | 256 | 464 | 1211 |
+| 8 | 1240 | 12 | 1030 | 2282 |
+| 9 | 4 | 0 | 1 | 5 |
+| total | 3191 | 472 | 2783 | 0 |
+
diff --git a/dataset/twitch/v2/report.md b/dataset/twitch/v2/report.md
new file mode 100644
index 0000000000000000000000000000000000000000..34addd77c8d14306b4da1d46445cc646b4de66c2
--- /dev/null
+++ b/dataset/twitch/v2/report.md
@@ -0,0 +1,32 @@
+# Global report
+
+2938 images, with:
+
+ - 2938 images
+ - 2133 robots
+ - 1345 watchers
+ - 2086 bases
+
+| | red | grey | blue | total |
+|:------|------:|-------:|-------:|--------:|
+| 0 | 54 | 0 | 68 | 122 |
+| 1 | 302 | 49 | 287 | 638 |
+| 2 | 277 | 19 | 287 | 583 |
+| 3 | 239 | 33 | 169 | 441 |
+| 4 | 283 | 78 | 238 | 599 |
+| 5 | 301 | 25 | 239 | 565 |
+| 6 | 0 | 0 | 0 | 0 |
+| 7 | 491 | 256 | 464 | 1211 |
+| 8 | 1240 | 12 | 1030 | 2282 |
+| 9 | 4 | 0 | 1 | 5 |
+| total | 3191 | 472 | 2783 | 0 |
+
+# Repartition
+
+| | T470149066 | T470149568 | T470150052 | T470151286 | T470152289 | T470152730 | T470152838 | T470152932 | T470153081 | T470158483 |
+|:---------|:-------------|:-------------|:-------------|:-------------|:-------------|:-------------|:-------------|:-------------|:-------------|:-------------|
+| images | 822 (28.0%) | 389 (13.2%) | 154 (5.2%) | 454 (15.5%) | 271 (9.2%) | 116 (3.9%) | 116 (3.9%) | 88 (3.0%) | 76 (2.6%) | 452 (15.4%) |
+| robots | 785 (36.8%) | 332 (15.6%) | 81 (3.8%) | 295 (13.8%) | 171 (8.0%) | 76 (3.6%) | 46 (2.2%) | 9 (0.4%) | 29 (1.4%) | 309 (14.5%) |
+| watchers | 321 (23.9%) | 196 (14.6%) | 89 (6.6%) | 189 (14.1%) | 145 (10.8%) | 59 (4.4%) | 56 (4.2%) | 17 (1.3%) | 34 (2.5%) | 239 (17.8%) |
+| bases | 488 (23.4%) | 253 (12.1%) | 135 (6.5%) | 337 (16.2%) | 219 (10.5%) | 108 (5.2%) | 89 (4.3%) | 77 (3.7%) | 41 (2.0%) | 339 (16.3%) |
+
diff --git a/src/polystar/models/roco_object.py b/src/polystar/models/roco_object.py
index 844ebd5f4bd9ec5079d6f69bb5aa5e1e46e6a2bc..0c9cc596296d37fdbd724d4509fecacaaf1e6cc8 100644
--- a/src/polystar/models/roco_object.py
+++ b/src/polystar/models/roco_object.py
@@ -95,27 +95,32 @@ class Armor(ROCOObject):
class ROCOObjectFactory:
- @staticmethod
- def from_json(json: Json) -> ROCOObject:
+ def __init__(self, image_w: int, image_h: int):
+ self.image_w = image_w
+ self.image_h = image_h
+
+ def from_json(self, json: Json) -> ROCOObject:
t: ObjectType = ObjectType(json["name"])
- x, y, w, h = (
+ x1, y1, x2, y2 = (
int(float(json["bndbox"]["xmin"])),
int(float(json["bndbox"]["ymin"])),
- int(float(json["bndbox"]["xmax"])) - int(float(json["bndbox"]["xmin"])),
- int(float(json["bndbox"]["ymax"])) - int(float(json["bndbox"]["ymin"])),
+ int(float(json["bndbox"]["xmax"])),
+ int(float(json["bndbox"]["ymax"])),
)
- x, y = max(0, x), max(0, y)
+ x1, y1 = max(0, x1), max(0, y1)
+ x2, y2 = min(x2, self.image_w - 1), min(y2, self.image_h - 1)
+ box = Box.from_positions(x1, y1, x2, y2)
if t is not ObjectType.ARMOR:
- return ROCOObject(type=t, box=Box.from_size(x, y, w, h=h))
+ return ROCOObject(type=t, box=box)
armor_number = int(json["armor_class"]) if json["armor_class"] != "none" else 0
return Armor(
type=t,
- box=Box.from_size(x, y, w, h=h),
+ box=box,
number=armor_number,
digit=ArmorDigit.from_number(armor_number),
color=ArmorColor(json["armor_color"]),
diff --git a/src/polystar/utils/dataframe.py b/src/polystar/utils/dataframe.py
index 814915c9804998cd5cadcc31b7bb64bd4cfbbae2..e020875b53a3ba4999062cc4d269e2850ea2cef9 100644
--- a/src/polystar/utils/dataframe.py
+++ b/src/polystar/utils/dataframe.py
@@ -27,3 +27,7 @@ def make_formater(fmt: Format) -> Callable:
if isinstance(fmt, str):
return fmt.format
return fmt
+
+
+def add_percentages_to_df(df: DataFrame, axis: int) -> DataFrame:
+ return df.applymap(str) + df.div(df.sum(axis=axis), axis=(1 - axis)).applymap(" ({:.1%})".format)
diff --git a/src/polystar/utils/iterable_utils.py b/src/polystar/utils/iterable_utils.py
index 1efac7228ee1aea8d161613ade1422b74525ad13..3467c9fe02d7490afbdf62d2559dbfb13ec91a57 100644
--- a/src/polystar/utils/iterable_utils.py
+++ b/src/polystar/utils/iterable_utils.py
@@ -1,6 +1,6 @@
from collections import defaultdict
from itertools import chain
-from typing import Callable, Dict, Iterable, List, TypeVar
+from typing import Any, Callable, Dict, Iterable, List, TypeVar
from more_itertools import ilen
@@ -38,3 +38,8 @@ def chunk(it: Iterable[T], batch_size: float) -> Iterable[List[T]]:
batch = []
if batch:
yield batch
+
+
+def apply(f: Callable[[T], Any], it: Iterable[T]):
+ for el in it:
+ f(el)
diff --git a/src/polystar/utils/markdown.py b/src/polystar/utils/markdown.py
index c44ed58b175873a0d6c6098dc382aa88b25da050..36f54c59a733fefe6f02f2629ad11eed88a29c69 100644
--- a/src/polystar/utils/markdown.py
+++ b/src/polystar/utils/markdown.py
@@ -12,7 +12,9 @@ from polystar.utils.working_directory import working_directory
class MarkdownFile:
- def __init__(self, markdown_path: Path):
+ def __init__(self, markdown_path: Path, name: str = "report"):
+ if markdown_path.suffix != ".md":
+ markdown_path /= f"{name}.md"
self.markdown_path = markdown_path
def __enter__(self):
@@ -47,11 +49,14 @@ class MarkdownFile:
close(figure)
return self.image(name, alt)
- def table(self, data: DataFrame) -> "MarkdownFile":
- self.file.write(tabulate(data, tablefmt="pipe", headers="keys").replace(".0 ", " "))
+ def table(self, data: Any) -> "MarkdownFile":
+ self.file.write(tabulate(DataFrame(data), tablefmt="pipe", headers="keys").replace(".0 ", " "))
self.file.write("\n\n")
return self
+ def __repr__(self):
+ return f"file://{self.markdown_path}"
+
def markdown_to_pdf(markdown_path: Path):
html_text = markdown(markdown_path.read_text(), output_format="html", extensions=["markdown.extensions.tables"])
diff --git a/src/polystar/utils/path.py b/src/polystar/utils/path.py
index be35166b13f8908889afc944ce7808a79f84ecf0..72e8a997d5d2a10d07dd94ddefa7949be1d8b1c5 100644
--- a/src/polystar/utils/path.py
+++ b/src/polystar/utils/path.py
@@ -1,11 +1,15 @@
+from os import remove
from pathlib import Path
from shutil import copy, make_archive, move
from typing import Iterable
-def move_file(source: Path, destination_directory: Path):
+def move_file(source: Path, destination_directory: Path) -> Path:
destination_directory.mkdir(exist_ok=True, parents=True)
- move(str(source), str(destination_directory))
+ destination = destination_directory / source.name
+ if destination.exists():
+ remove(destination)
+ return Path(move(str(source), str(destination_directory)))
def move_files(sources: Iterable[Path], destination_directory: Path):
@@ -13,9 +17,9 @@ def move_files(sources: Iterable[Path], destination_directory: Path):
move_file(source, destination_directory)
-def copy_file(source: Path, destination_directory: Path):
+def copy_file(source: Path, destination_directory: Path) -> Path:
destination_directory.mkdir(exist_ok=True, parents=True)
- copy(str(source), str(destination_directory))
+ return Path(copy(str(source), str(destination_directory)))
def archive_directory(directory:Path):
diff --git a/src/research/common/datasets/image_file_dataset_builder.py b/src/research/common/datasets/image_file_dataset_builder.py
index 07acd80c91212078a6dfd5b63f623f3f5211f471..8ef9a1b67e68817e25345253ab460b983e0faca3 100644
--- a/src/research/common/datasets/image_file_dataset_builder.py
+++ b/src/research/common/datasets/image_file_dataset_builder.py
@@ -5,6 +5,8 @@ from polystar.models.image import FileImage, Image, load_image
from research.common.datasets.dataset_builder import DatasetBuilder
from research.common.datasets.lazy_dataset import LazyDataset, TargetT
+CORRUPTED_FILES = {"EvolutionVsåŽå—虎_BO3_2_381.jpg"} # TODO: cleaner way to do that
+
class LazyFileDataset(LazyDataset[Path, TargetT]):
def __init__(self, files: Iterable[Path], target_from_file: Callable[[Path], TargetT], name: str):
@@ -14,7 +16,8 @@ class LazyFileDataset(LazyDataset[Path, TargetT]):
def __iter__(self) -> Iterator[Tuple[Path, TargetT, str]]:
for file in self.files:
- yield file, self.target_from_file(file), file.stem
+ if file.name not in CORRUPTED_FILES:
+ yield file, self.target_from_file(file), file.stem
def __len__(self):
return len(self.files)
diff --git a/src/research/common/datasets/roco/roco_annotation.py b/src/research/common/datasets/roco/roco_annotation.py
index fdcd57d55544dee7e2f773e34987c8da0278ad8a..8c262f0f82a9ffcad64fc123e0436de20e91d9cc 100644
--- a/src/research/common/datasets/roco/roco_annotation.py
+++ b/src/research/common/datasets/roco/roco_annotation.py
@@ -29,21 +29,23 @@ class ROCOAnnotation:
try:
return ROCOAnnotation.from_xml_dict(xmltodict.parse(xml_file.read_text())["annotation"])
except Exception as e:
- logging.exception(f"Error parsing annotation file {xml_file}")
+ logging.exception(f"Error parsing annotation file file://{xml_file}")
raise e
@staticmethod
def from_xml_dict(xml_dict: Dict) -> "ROCOAnnotation":
+ image_h = int(xml_dict["size"]["height"])
+ image_w = int(xml_dict["size"]["width"])
+
json_objects = xml_dict.get("object", []) or []
json_objects = json_objects if isinstance(json_objects, list) else [json_objects]
roco_json_objects = [obj_json for obj_json in json_objects if not obj_json["name"].startswith("rune")]
- objects = [ROCOObjectFactory.from_json(obj_json) for obj_json in roco_json_objects]
+ objects = [
+ ROCOObjectFactory(image_w=image_w, image_h=image_h).from_json(obj_json) for obj_json in roco_json_objects
+ ]
return ROCOAnnotation(
- objects=objects,
- has_rune=len(roco_json_objects) != len(json_objects),
- w=int(xml_dict["size"]["width"]),
- h=int(xml_dict["size"]["height"]),
+ objects=objects, has_rune=len(roco_json_objects) != len(json_objects), w=image_w, h=image_h,
)
def save_in_directory(self, directory: Path, name: str):
@@ -71,7 +73,9 @@ class ROCOAnnotation:
).toprettyxml()
-def move_image_and_annotation(source_dataset_directory: Path, destination_dataset_directory: Path, name: str):
+def move_image_and_annotation_from_directory(
+ source_dataset_directory: Path, destination_dataset_directory: Path, name: str
+):
move_file((source_dataset_directory / "image" / name).with_suffix(".jpg"), destination_dataset_directory / "image")
move_file(
(source_dataset_directory / "image_annotation" / name).with_suffix(".xml"),
diff --git a/src/research/common/datasets/roco/roco_dataset_descriptor.py b/src/research/common/datasets/roco/roco_dataset_descriptor.py
index 2ea388e699bccb2fee2d2f63b9fc9ffba7ae3fcd..2a80192a1068f3b0cd64f3aa5b78f7c1d84c7c2a 100644
--- a/src/research/common/datasets/roco/roco_dataset_descriptor.py
+++ b/src/research/common/datasets/roco/roco_dataset_descriptor.py
@@ -1,13 +1,16 @@
from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Dict
+from typing import Dict, Type
from pandas import DataFrame
from polystar.models.roco_object import Armor, ObjectType
+from polystar.utils.dataframe import add_percentages_to_df
+from polystar.utils.iterable_utils import apply
from polystar.utils.markdown import MarkdownFile
from polystar.utils.tqdm import smart_tqdm
from research.common.datasets.roco.roco_dataset import LazyROCOFileDataset
+from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
+from research.common.datasets.roco.roco_datasets import ROCODatasets
from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
@@ -20,15 +23,25 @@ class ROCODatasetStats:
n_watchers: int = 0
n_bases: int = 0
- armors_color2num2count: Dict[str, Dict[int, int]] = field(default_factory=dict)
+ armors_df: DataFrame = field(default_factory=DataFrame)
+
+ @property
+ def numbers(self) -> Dict[str, int]:
+ return {
+ "images": self.n_images,
+ # "runes": self.n_runes,
+ "robots": self.n_robots,
+ "watchers": self.n_watchers,
+ "bases": self.n_bases,
+ }
@staticmethod
def from_dataset(dataset: LazyROCOFileDataset) -> "ROCODatasetStats":
rv = ROCODatasetStats()
colors = ["red", "grey", "blue", "total"]
- rv.armors_color2num2count = {c: {n: 0 for n in range(10)} for c in colors}
+ armors_color2num2count = {c: {n: 0 for n in range(10)} for c in colors}
for c in colors:
- rv.armors_color2num2count[c]["total"] = 0
+ armors_color2num2count[c]["total"] = 0
for (_, annotation, _) in smart_tqdm(dataset, desc=dataset.name, unit="frame"):
rv.n_images += 1
rv.n_runes += annotation.has_rune
@@ -40,35 +53,56 @@ class ROCODatasetStats:
elif obj.type == ObjectType.WATCHER:
rv.n_watchers += 1
elif isinstance(obj, Armor):
- rv.armors_color2num2count[obj.color.name.lower()][obj.number] += 1
- rv.armors_color2num2count[obj.color.name.lower()]["total"] += 1
- rv.armors_color2num2count["total"][obj.number] += 1
+ armors_color2num2count[obj.color.name.lower()][obj.number] += 1
+ armors_color2num2count[obj.color.name.lower()]["total"] += 1
+ armors_color2num2count["total"][obj.number] += 1
+ rv.armors_df = DataFrame(armors_color2num2count)
return rv
+ def __add__(self, other: "ROCODatasetStats") -> "ROCODatasetStats":
+ if self.armors_df.empty:
+ armors_df = other.armors_df.copy()
+ else:
+ armors_df = self.armors_df + other.armors_df
+ return ROCODatasetStats(
+ n_robots=self.n_robots + other.n_robots,
+ n_images=self.n_images + other.n_images,
+ n_watchers=self.n_watchers + other.n_watchers,
+ n_runes=self.n_runes + other.n_runes,
+ n_bases=self.n_bases + other.n_bases,
+ armors_df=armors_df,
+ )
-def make_markdown_dataset_report(dataset: LazyROCOFileDataset, report_dir: Path):
- report_path = report_dir / f"dset_{dataset.name}_report.md"
- stats = ROCODatasetStats.from_dataset(dataset)
+def make_dataset_report(builder: ROCODatasetBuilder) -> ROCODatasetStats:
+ stats = ROCODatasetStats.from_dataset(builder.build_lazy())
- with MarkdownFile(report_path) as mf:
- mf.title(f"Dataset {dataset.name}")
+ with MarkdownFile(builder.main_dir) as mf:
+ mf.title(f"Dataset {builder.name}")
+ _report_single_stat(mf, stats)
- mf.paragraph(f"{stats.n_images} images, with:")
- mf.list(
- [
- f"{stats.n_bases} bases",
- f"{stats.n_watchers} watchers",
- f"{stats.n_runes} runes",
- f"{stats.n_robots} robots",
- ]
- )
- mf.table(DataFrame(stats.armors_color2num2count))
- print(f"file://{report_path}")
+ return stats
+
+
+def make_datasets_report(datasets: Type[ROCODatasets]):
+ name2stats = {builder.name: make_dataset_report(builder) for builder in datasets}
+ global_stat = sum(name2stats.values(), ROCODatasetStats())
+
+ with MarkdownFile(datasets.main_dir / "report.md") as mf:
+ mf.title("Global report")
+ _report_single_stat(mf, global_stat)
+
+ mf.title("Repartition")
+ mf.table(add_percentages_to_df(DataFrame({name: stat.numbers for name, stat in name2stats.items()}), axis=1))
+
+ print(mf)
+
+
+def _report_single_stat(mf: MarkdownFile, stats: ROCODatasetStats):
+ mf.paragraph(f"{stats.n_images} images, with:")
+ mf.list([f"{number} {name}" for name, number in stats.numbers.items()])
+ mf.table(DataFrame(stats.armors_df))
if __name__ == "__main__":
- for datasets in ROCODatasetsZoo:
- make_markdown_dataset_report(datasets.union(), datasets.main_dir)
- for dset in datasets:
- make_markdown_dataset_report(dset.build_lazy(), dset.main_dir)
+ apply(make_datasets_report, ROCODatasetsZoo)
diff --git a/src/research/common/datasets/roco/roco_datasets.py b/src/research/common/datasets/roco/roco_datasets.py
index dbecdcdec9add9f5939986cdcc3e4d8955f19e1d..b7765e94746ea8fadd1a20ada6c71d72c1e0fb2d 100644
--- a/src/research/common/datasets/roco/roco_datasets.py
+++ b/src/research/common/datasets/roco/roco_datasets.py
@@ -7,6 +7,7 @@ from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilde
from research.common.datasets.union_dataset import UnionLazyDataset
+# FIXME : we should be able to access a builder 2 times
class ROCODatasetsMeta(type):
def __init__(cls, name: str, bases, dct):
super().__init__(name, bases, dct)
diff --git a/src/research/common/datasets/roco/zoo/roco_dataset_zoo.py b/src/research/common/datasets/roco/zoo/roco_dataset_zoo.py
index 83cd116fcef5b23fbc16faea47a4d361b6ceb4b0..65a982acf125e98c6726df18f28eb8fb285b7788 100644
--- a/src/research/common/datasets/roco/zoo/roco_dataset_zoo.py
+++ b/src/research/common/datasets/roco/zoo/roco_dataset_zoo.py
@@ -11,9 +11,16 @@ class ROCODatasetsZoo(Iterable[Type[ROCODatasets]]):
DJI = DJIROCODatasets
TWITCH = TwitchROCODatasets
- DEFAULT_TEST_DATASETS = [TWITCH.T470152838, TWITCH.T470151286]
- DEFAULT_VALIDATION_DATASETS = [TWITCH.T470152289, TWITCH.T470149568]
- TWITCH_TRAIN_DATASETS = [TWITCH.T470150052, TWITCH.T470152730, TWITCH.T470153081, TWITCH.T470158483]
+ TWITCH_TRAIN_DATASETS = [
+ TWITCH.T470149066,
+ TWITCH.T470150052,
+ TWITCH.T470152289,
+ TWITCH.T470153081,
+ TWITCH.T470158483,
+ ]
+ TWITCH_VALIDATION_DATASETS = [TWITCH.T470152932, TWITCH.T470149568]
+ TWITCH_TEST_DATASETS = [TWITCH.T470152838, TWITCH.T470151286]
+
DJI_TRAIN_DATASETS = [DJI.FINAL, DJI.CENTRAL_CHINA, DJI.NORTH_CHINA, DJI.SOUTH_CHINA]
DJI_ZOOMED_TRAIN_DATASETS = [
DJI_ZOOMED.FINAL,
@@ -21,7 +28,13 @@ class ROCODatasetsZoo(Iterable[Type[ROCODatasets]]):
DJI_ZOOMED.NORTH_CHINA,
DJI_ZOOMED.SOUTH_CHINA,
]
- DEFAULT_TRAIN_DATASETS = TWITCH_TRAIN_DATASETS + DJI_TRAIN_DATASETS
+
+ TWITCH_DJI_TRAIN_DATASETS = TWITCH_TRAIN_DATASETS + DJI_TRAIN_DATASETS
+ TWITCH_DJI_ZOOMED_TRAIN_DATASETS = TWITCH_TRAIN_DATASETS + DJI_TRAIN_DATASETS
+
+ DEFAULT_TEST_DATASETS = TWITCH_TEST_DATASETS
+ DEFAULT_VALIDATION_DATASETS = TWITCH_VALIDATION_DATASETS
+ DEFAULT_TRAIN_DATASETS = TWITCH_DJI_TRAIN_DATASETS
def __iter__(self) -> Iterator[Type[ROCODatasets]]:
return iter((self.DJI, self.DJI_ZOOMED, self.TWITCH))
diff --git a/src/research/common/datasets/roco/zoo/twitch.py b/src/research/common/datasets/roco/zoo/twitch.py
index 5a4c4206692fe56226e2e9977236bd9a33b48459..e8dbd4d51f3ff3063ef20c1dbe90c5a25b5ae1e8 100644
--- a/src/research/common/datasets/roco/zoo/twitch.py
+++ b/src/research/common/datasets/roco/zoo/twitch.py
@@ -4,13 +4,15 @@ from research.common.datasets.roco.roco_datasets import ROCODatasets
class TwitchROCODatasets(ROCODatasets):
- main_dir = TWITCH_DSET_DIR / "v1"
+ main_dir = TWITCH_DSET_DIR / "v2"
+ T470149066: ROCODatasetBuilder = ()
T470149568: ROCODatasetBuilder = ()
T470150052: ROCODatasetBuilder = ()
T470151286: ROCODatasetBuilder = ()
T470152289: ROCODatasetBuilder = ()
T470152730: ROCODatasetBuilder = ()
+ T470152932: ROCODatasetBuilder = ()
T470152838: ROCODatasetBuilder = ()
T470153081: ROCODatasetBuilder = ()
T470158483: ROCODatasetBuilder = ()
diff --git a/src/research/dataset/scripts/add_annotated_images_to_twitch_dataset.py b/src/research/dataset/scripts/add_annotated_images_to_twitch_dataset.py
new file mode 100644
index 0000000000000000000000000000000000000000..fae248ee326c60401c7539d17dbb42f3213e4054
--- /dev/null
+++ b/src/research/dataset/scripts/add_annotated_images_to_twitch_dataset.py
@@ -0,0 +1,28 @@
+from pathlib import Path
+
+from tqdm import tqdm
+
+from polystar.utils.path import copy_file
+from research.common.constants import TWITCH_DSET_DIR
+from research.common.datasets.roco.roco_annotation import ROCOAnnotation, move_image_and_annotation_from_directory
+from research.dataset.twitch.annotation_file_corrector import AnnotationFileCorrector
+
+
+def add_annotated_images_to_dataset(images_dir: Path, annotations_dir: Path, dataset_dir: Path):
+ corrector = AnnotationFileCorrector(save_before=False)
+ for annotation_file in tqdm(list(annotations_dir.glob("**/*.xml"))):
+ sub_dataset_dir = dataset_dir / annotation_file.stem.split("-")[0]
+ image_file = (images_dir / annotation_file.relative_to(annotations_dir)).with_suffix(".jpg")
+
+ copy_file(image_file, sub_dataset_dir / "image")
+ annotation_file = copy_file(annotation_file, sub_dataset_dir / "image_annotation")
+ corrector.correct_annotation_file(annotation_file)
+
+ if ROCOAnnotation.from_xml_file(annotation_file).has_rune:
+ move_image_and_annotation_from_directory(sub_dataset_dir, dataset_dir / "runes", annotation_file.stem)
+
+
+if __name__ == "__main__":
+ add_annotated_images_to_dataset(
+ TWITCH_DSET_DIR / "chunks-to-annotate", TWITCH_DSET_DIR / "chunks-annotations", TWITCH_DSET_DIR / "v2"
+ )
diff --git a/src/research/dataset/scripts/construct_dataset_from_manual_annotation.py b/src/research/dataset/scripts/construct_dataset_from_manual_annotation.py
deleted file mode 100644
index 3178c9139ee0bba87e820e4859d74d3382e27306..0000000000000000000000000000000000000000
--- a/src/research/dataset/scripts/construct_dataset_from_manual_annotation.py
+++ /dev/null
@@ -1,29 +0,0 @@
-from os import remove
-from pathlib import Path
-from shutil import move, rmtree
-from zipfile import ZipFile
-
-
-def construct_dataset_from_manual_annotations(
- source_images_directory: Path, source_annotations_directory: Path, destination_directory: Path
-):
- destination_images_directory = destination_directory / "image"
- destination_annotations_directory = destination_directory / "image_annotation"
- _unzip_all_in_directory(source_images_directory, destination_images_directory, "jpg")
- _unzip_all_in_directory(source_annotations_directory, destination_annotations_directory, "xml")
-
- names_of_annotated = {annotation_file.stem for annotation_file in destination_annotations_directory.glob("*.xml")}
- for image_file in destination_images_directory.glob("*.jpg"):
- if image_file.stem not in names_of_annotated:
- remove(str(image_file))
-
-
-def _unzip_all_in_directory(source_directory: Path, destination_directory: Path, extension: str):
- for zip_path in source_directory.glob("*.zip"):
- with ZipFile(zip_path, "r") as zip_ref:
- zip_ref.extractall(destination_directory)
- for file_path in destination_directory.glob(f"**/*.{extension}"):
- move(str(file_path), str(destination_directory / file_path.name))
- for directory in destination_directory.glob("*"):
- if directory.is_dir():
- rmtree(str(directory))
diff --git a/src/research/dataset/scripts/construct_twith_datasets_from_manual_annotation.py b/src/research/dataset/scripts/construct_twith_datasets_from_manual_annotation.py
deleted file mode 100644
index b4ef92792e35a482adfea3e4338572dd9530f1be..0000000000000000000000000000000000000000
--- a/src/research/dataset/scripts/construct_twith_datasets_from_manual_annotation.py
+++ /dev/null
@@ -1,78 +0,0 @@
-from os import remove
-from shutil import copy, make_archive, move, rmtree
-
-from research.common.constants import TWITCH_DSET_DIR, TWITCH_DSET_ROBOTS_VIEWS_DIR, TWITCH_ROBOTS_VIEWS_DIR
-from research.common.datasets.roco.roco_dataset_builder import ROCODatasetBuilder
-from research.common.datasets.roco.roco_dataset_descriptor import make_markdown_dataset_report
-from research.dataset.scripts.construct_dataset_from_manual_annotation import construct_dataset_from_manual_annotations
-from research.dataset.scripts.correct_annotations import AnnotationFileCorrector
-
-
-def _construct_mixed_twitch_dset():
- construct_dataset_from_manual_annotations(
- TWITCH_ROBOTS_VIEWS_DIR, TWITCH_DSET_DIR / "reviewed-robots-views-annotations", TWITCH_DSET_ROBOTS_VIEWS_DIR,
- )
-
-
-def _correct_manual_annotations():
- corrector = AnnotationFileCorrector(save_before=False)
- corrector.correct_annotations_in_directory(TWITCH_DSET_ROBOTS_VIEWS_DIR / "image_annotation")
-
-
-def _extract_runes_images():
- all_twitch_dataset = _get_mixed_dataset_builder()
- for image_file, annotation, _ in all_twitch_dataset:
- if annotation.has_rune:
- copy(str(image_file), str(TWITCH_DSET_DIR / "runes" / image_file.name))
-
-
-def _separate_twitch_videos():
- all_twitch_dataset_builder = _get_mixed_dataset_builder()
- for image_file, annotation, _ in all_twitch_dataset_builder:
- video_name = image_file.name.split("-")[0]
- dset_path = TWITCH_DSET_ROBOTS_VIEWS_DIR / video_name
- images_path = dset_path / "image"
- annotations_path = dset_path / "image_annotation"
- images_path.mkdir(exist_ok=True, parents=True)
- annotations_path.mkdir(exist_ok=True, parents=True)
- move(str(image_file), str(images_path / image_file.name))
- xml_name = f"{image_file.stem}.xml"
- move(str(all_twitch_dataset_builder.annotations_dir / xml_name), str(annotations_path / xml_name))
- if list((TWITCH_DSET_ROBOTS_VIEWS_DIR / "image").glob("*")):
- raise Exception(f"Some images remains unmoved")
- for remaining_file in (TWITCH_DSET_ROBOTS_VIEWS_DIR / "image_annotation").glob("*"):
- if remaining_file.name != ".DS_Store":
- raise Exception(f"Some annotations remains unmoved")
- rmtree(str(TWITCH_DSET_ROBOTS_VIEWS_DIR / "image"))
- rmtree(str(TWITCH_DSET_ROBOTS_VIEWS_DIR / "image_annotation"))
-
-
-def _make_global_report():
- all_twitch_dataset_builder = _get_mixed_dataset_builder()
- make_markdown_dataset_report(all_twitch_dataset_builder.build_lazy(), all_twitch_dataset_builder.main_dir)
-
-
-def _get_mixed_dataset_builder() -> ROCODatasetBuilder:
- return ROCODatasetBuilder(TWITCH_DSET_ROBOTS_VIEWS_DIR, "Twitch")
-
-
-def _make_separate_reports():
- for video_dset_path in TWITCH_DSET_ROBOTS_VIEWS_DIR.glob("*"):
- if video_dset_path.is_dir():
- twitch_dset = ROCODatasetBuilder(video_dset_path, f"TWITCH_{video_dset_path.name}")
- make_markdown_dataset_report(twitch_dset.build_lazy(), twitch_dset.main_dir)
-
-
-if __name__ == "__main__":
-
- for zip_file in (TWITCH_DSET_DIR / "reviewed-robots-views-annotations").glob("*.zip"):
- remove(str(zip_file))
- for chunk_dir in (TWITCH_DSET_DIR / "reviewed-robots-views-annotations").glob("chunk_*"):
- make_archive(chunk_dir, "zip", chunk_dir)
-
- _construct_mixed_twitch_dset()
- _correct_manual_annotations()
- _extract_runes_images()
- _make_global_report()
- _separate_twitch_videos()
- _make_separate_reports()
diff --git a/src/research/dataset/scripts/create_tensorflow_records.py b/src/research/dataset/scripts/create_tensorflow_records.py
index 3921e81c7e2ba9020b5a2ff14127e533ae738b9c..21486eb7956ce97906552c061ad545e4200bcc77 100644
--- a/src/research/dataset/scripts/create_tensorflow_records.py
+++ b/src/research/dataset/scripts/create_tensorflow_records.py
@@ -1,16 +1,14 @@
from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
from research.dataset.tensorflow_record import TensorflowRecordFactory
-
-def create_one_record_per_roco_dset():
- for datasets in ROCODatasetsZoo:
- for dataset in datasets:
- TensorflowRecordFactory.from_dataset(dataset)
-
-
if __name__ == "__main__":
- TensorflowRecordFactory.from_datasets(ROCODatasetsZoo.DEFAULT_TEST_DATASETS, "Twitch_Test_")
- # TensorflowRecordFactory.from_datasets(ROCODatasetsZoo.TWITCH_TRAIN_DATASETS, "Twitch_Train_")
+ _factory = TensorflowRecordFactory()
+ # _factory.from_datasets(ROCODatasetsZoo.TWITCH_TRAIN_DATASETS, "Twitch2_Train_")
+ # _factory.from_datasets(ROCODatasetsZoo.TWITCH_VALIDATION_DATASETS, "Twitch2_Val_")
+ # _factory.from_datasets(ROCODatasetsZoo.TWITCH_TEST_DATASETS, "Twitch2_Test_")
+ _factory.from_datasets(ROCODatasetsZoo.TWITCH_DJI_TRAIN_DATASETS, "Twitch2_Dji_Train_")
+ # _factory.from_datasets(ROCODatasetsZoo.TWITCH_DJI_ZOOMED_TRAIN_DATASETS, "Twitch2_DjiZoomed2_Train_")
+
# TensorflowRecordFactory.from_datasets(ROCODatasetsZoo.DJI_TRAIN_DATASETS, "DJI_Train_")
# TensorflowRecordFactory.from_datasets(ROCODatasetsZoo.DJI_ZOOMED_TRAIN_DATASETS, "DJIZoomedV2_Train_")
# TensorflowRecordFactory.from_datasets(
diff --git a/src/research/dataset/scripts/visualize_dataset.py b/src/research/dataset/scripts/visualize_dataset.py
index 6dc7e5017755567bb42d4ae098257f4629d5b6f4..9b88b931c7b7be1d6c721755862e8e8fb4834a37 100644
--- a/src/research/dataset/scripts/visualize_dataset.py
+++ b/src/research/dataset/scripts/visualize_dataset.py
@@ -3,16 +3,13 @@ from research.common.datasets.roco.roco_dataset import LazyROCODataset
from research.common.datasets.roco.zoo.roco_dataset_zoo import ROCODatasetsZoo
-def visualize_dataset(dataset: LazyROCODataset, n_images: int):
+def visualize_dataset(dataset: LazyROCODataset):
viewer = PltResultViewer(dataset.name)
- for i, (image, annotation, name) in enumerate(dataset, 1):
+ for image, annotation, _ in dataset:
viewer.display_image_with_objects(image, annotation.objects)
- if i == n_images:
- return
-
if __name__ == "__main__":
for builder in ROCODatasetsZoo.TWITCH:
- visualize_dataset(builder.to_images().build_lazy(), 20)
+ visualize_dataset(builder.cap(5).to_images().build_lazy())
diff --git a/src/research/dataset/tensorflow_record.py b/src/research/dataset/tensorflow_record.py
index 615b23551adc4d256db0bdeae8d66c022243967c..5c2c80407f2e0f5638fb6b9695a4b33f40ceb1c3 100644
--- a/src/research/dataset/tensorflow_record.py
+++ b/src/research/dataset/tensorflow_record.py
@@ -47,10 +47,16 @@ class TensorflowRecordFactory:
xmin, ymin, xmax, ymax, classes, classes_text = [], [], [], [], [], []
for obj in self.objects_filter.filter(annotation.objects):
- xmin.append(float(obj.box.x1) / width)
- ymin.append(float(obj.box.y1) / height)
- xmax.append(float(obj.box.x2) / width)
- ymax.append(float(obj.box.y2) / height)
+ x1 = max(0.0, obj.box.x1 / width)
+ y1 = max(0.0, obj.box.y1 / height)
+ x2 = min(1.0, obj.box.x2 / width)
+ y2 = min(1.0, obj.box.y2 / height)
+ if x1 >= x2 or y1 >= y2:
+ continue
+ xmin.append(x1)
+ ymin.append(y1)
+ xmax.append(x2)
+ ymax.append(y2)
classes_text.append(obj.type.name.lower().encode("utf8"))
classes.append(label_map.id_of(obj.type.name.lower()))
diff --git a/src/research/dataset/scripts/correct_annotations.py b/src/research/dataset/twitch/annotation_file_corrector.py
similarity index 90%
rename from src/research/dataset/scripts/correct_annotations.py
rename to src/research/dataset/twitch/annotation_file_corrector.py
index 5b45d96fcc2167c732484a7cc4dd8808ff2af4a1..c0e059b85e49579aaa719ef63795b7d061ecb192 100644
--- a/src/research/dataset/scripts/correct_annotations.py
+++ b/src/research/dataset/twitch/annotation_file_corrector.py
@@ -1,5 +1,7 @@
+import logging
import re
from pathlib import Path
+from typing.re import Match
class AnnotationFileCorrector:
@@ -10,7 +12,7 @@ class AnnotationFileCorrector:
ABV_RUNES_PATTERN = re.compile(r"<name>r(?P<color>\w)</name>", re.IGNORECASE)
ABV_BASE_PATTERN = re.compile(r"<name>b</name>", re.IGNORECASE)
ABV_WATCHER_PATTERN = re.compile(r"<name>w</name>", re.IGNORECASE)
- ABV_CAR_PATTERN = re.compile(r"<name>(c|x|robot)</name>", re.IGNORECASE)
+ ABV_CAR_PATTERN = re.compile(r"<name>(c|x|o|robot)</name>", re.IGNORECASE)
COLORS_MAP = {
"r": "red",
@@ -36,7 +38,7 @@ class AnnotationFileCorrector:
text = self._correct_annotation_text(text)
annotation_file.write_text(text)
except Exception as e:
- print(f"Error processing annotation {annotation_file}")
+ logging.exception(f"Error processing annotation file://{annotation_file}")
raise e
def _correct_annotation_text(self, text: str) -> str:
@@ -64,9 +66,11 @@ class AnnotationFileCorrector:
def _correct_armor_format(self, text: str) -> str:
return self.FINAL_ARMOR_NAME_PATTERN.sub(self._armor_pattern_mapping, text)
- def _armor_pattern_mapping(self, match: re.Match):
+ def _armor_pattern_mapping(self, match: Match):
color = self.COLORS_MAP[match.group("color")]
num = match.group("num")
+ if num == "p":
+ num = 4
return f"<name>armor</name> <armor_class>{num}</armor_class> <armor_color>{color}</armor_color>"
def _correct_armor_abbreviation(self, text: str) -> str:
@@ -82,7 +86,7 @@ class AnnotationFileCorrector:
return self.ABV_WATCHER_PATTERN.sub("<name>watcher</name>", text)
def _correct_runes_abbreviation(self, text: str) -> str:
- def runes_abbreviation_mapping(match: re.Match):
+ def runes_abbreviation_mapping(match: Match):
color = self.COLORS_MAP[match.group("color")]
return f"<name>rune-{color}</name>"