├── MU_ScottGS_ML_Fusion_Tutorial.pdf
├── README.md
├── datafiles
├── cross_val
│ ├── xval.densenet.5fold.csv
│ ├── xval.inception.5fold.csv
│ └── xval.res50.5fold.csv
├── pred
│ ├── pred.densenet.5fold::A.csv
│ ├── pred.inception.5fold::A.csv
│ └── pred.res50.5fold::A.csv
├── xval-0.test.csv
├── xval-0.train.csv
├── xval-1.test.csv
├── xval-1.train.csv
├── xval-2.test.csv
├── xval-2.train.csv
├── xval-3.test.csv
├── xval-3.train.csv
├── xval-4.test.csv
├── xval-4.train.csv
├── xval_cls.densenet.5fold.csv
├── xval_cls.inception.5fold.csv
└── xval_cls.res50.5fold.csv
├── notebooks
├── FuzzyFusion.ipynb
├── IntroductionToPyTorch.ipynb
├── PyTorchResNet50.ipynb
├── Results6411.csv
├── images
│ ├── DCNN_Fusion_Framework_Vert.png
│ ├── RESISC45_p1.PNG
│ ├── RESISC45_p2.PNG
│ ├── convfilters_giphy.gif
│ ├── densenet_towarddatascience.png
│ ├── inceptionv3_kaggle.png
│ ├── resnet50_kaggle.png
│ └── training.png
├── label.csv
├── samples0.csv
├── samples1.csv
└── samples2.csv
└── src
├── ChI.py
├── ChoquetIntegral.py
├── compute_sugeno.py
├── fuse_nets_data_driven.py
├── fuse_nets_sugeno.py
├── fusion_csv.sql
├── metadata-resisc45.py
├── metadata-ucm.py
├── tables.sql
├── train-resisc45.py
└── train-ucm.py
/MU_ScottGS_ML_Fusion_Tutorial.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/MU_ScottGS_ML_Fusion_Tutorial.pdf
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ### FuzzyFusion_DeepLearning_Tutorial
2 |
3 | # Fuzzy Fusion of Decisions from Heterogeneous Deep Machine Learning Models
4 |
5 | This tutorial will teach participants three key skills that are critical to
6 | advancing research in computational intelligence:
7 |
8 | 1) use of Keras and TensorFlow;
9 | 2) deep learning models and transfer learning techniques; and
10 | 3) fuzzy machine learning model fusion.
11 |
12 | The tutorial session will be broken into these three portions,
13 | each of which culminates in code examples that can be immediately
14 | migrated from the tutorial to the participants own research thrusts
15 | (theories and applications). The session will conclude with a
16 | case-study demonstrating how the components have been tied together
17 | for scientific studies in remote sensing.
18 |
19 | ## Code Content
20 | Tutorial code content develop by Alex Yang and Bryce Murray
21 | Department of Electrical Engineering and Computer Science
22 | University of Missouri - Columbia
23 |
24 | Please see the Jupyter Notebooks in [notebooks](notebooks/)
25 | * [IntroductionToPyTorch](notebooks/IntroductionToPyTorch.ipynb)
26 | * [PyTorchResNet50](notebooks/PyTorchResNet50.ipynb)
27 | * [FuzzyFusion](notebooks/FuzzyFusion.ipynb)
28 |
29 | ## Docker container image for running tutorial codes
30 |
31 | https://hub.docker.com/r/muiidsa/singleuser-pytorch-cpu/
32 |
33 | ## Tutorial Organizer
34 | Grant Scott
35 | Department of Electrical Engineering and Computer Science
36 | University of Missouri - Columbia
37 | &
38 | Director, Data Science and Analytics
39 | University of Missouri
40 |
41 |
42 | # Sponsors / Support
43 | The tutorial was developed thanks to support of:
44 | * **Mizzou High Performance Data-Intensive Computing Systems Laboratory**
45 | * **Mizzou INformation and Data FUsion Laboratory (MINDFUL)**
46 |
47 |
48 | # Deliveries
49 | ### [FUZZ-IEEE Tutorial](https://attend.ieee.org/fuzzieee-2019/tutorials/#ffodfhdmlm)
50 | * 2019, June 23, 1400 - 1600
51 | * IEEE International Conference on Fuzzy Systems (FUZZ-IEEE),
52 | * New Orleans, LA, USA
53 | * June 23-26, 2019
54 | * The tutorial is delivered with support of the University of Missouri's **_Data Science and Analytics Program_**
55 |
56 |
57 |
58 |
--------------------------------------------------------------------------------
/datafiles/cross_val/xval.densenet.5fold.csv:
--------------------------------------------------------------------------------
1 | fold_name,accuracy
2 | A,0.923619844444444
3 | B,0.924509688888889
4 | C,0.931278844444444
5 | D,0.912228355555556
6 | E,0.916558622222222
7 |
--------------------------------------------------------------------------------
/datafiles/cross_val/xval.inception.5fold.csv:
--------------------------------------------------------------------------------
1 | fold_name,accuracy
2 | A,0.614331911111111
3 | B,0.500792133333334
4 | C,0.659007822222222
5 | D,0.586910444444445
6 | E,0.670219688888889
7 |
--------------------------------------------------------------------------------
/datafiles/cross_val/xval.res50.5fold.csv:
--------------------------------------------------------------------------------
1 | fold_name,accuracy
2 | A,0.888224644444445
3 | B,0.904369333333333
4 | C,0.877219688888889
5 | D,0.881196933333333
6 | E,0.902779711111111
7 |
--------------------------------------------------------------------------------
/datafiles/xval-0.test.csv:
--------------------------------------------------------------------------------
1 | ,image,class_label
2 | 0,basketball_court/basketball_court_095.jpg,3
3 | 1,basketball_court/basketball_court_156.jpg,3
4 | 2,basketball_court/basketball_court_314.jpg,3
5 | 3,basketball_court/basketball_court_579.jpg,3
6 | 4,basketball_court/basketball_court_213.jpg,3
7 | 5,basketball_court/basketball_court_087.jpg,3
8 | 6,basketball_court/basketball_court_288.jpg,3
9 | 7,basketball_court/basketball_court_339.jpg,3
10 | 8,basketball_court/basketball_court_162.jpg,3
11 | 9,basketball_court/basketball_court_557.jpg,3
12 | 10,basketball_court/basketball_court_494.jpg,3
13 | 11,basketball_court/basketball_court_284.jpg,3
14 | 12,basketball_court/basketball_court_110.jpg,3
15 | 13,basketball_court/basketball_court_035.jpg,3
16 | 14,basketball_court/basketball_court_631.jpg,3
17 | 15,basketball_court/basketball_court_334.jpg,3
18 | 16,basketball_court/basketball_court_305.jpg,3
19 | 17,basketball_court/basketball_court_468.jpg,3
20 | 18,basketball_court/basketball_court_657.jpg,3
21 | 19,basketball_court/basketball_court_014.jpg,3
22 | 20,basketball_court/basketball_court_239.jpg,3
23 | 21,basketball_court/basketball_court_661.jpg,3
24 | 22,basketball_court/basketball_court_046.jpg,3
25 | 23,basketball_court/basketball_court_257.jpg,3
26 | 24,basketball_court/basketball_court_655.jpg,3
27 | 25,basketball_court/basketball_court_433.jpg,3
28 | 26,basketball_court/basketball_court_372.jpg,3
29 | 27,basketball_court/basketball_court_169.jpg,3
30 | 28,basketball_court/basketball_court_295.jpg,3
31 | 29,basketball_court/basketball_court_177.jpg,3
32 | 30,basketball_court/basketball_court_567.jpg,3
33 | 31,basketball_court/basketball_court_586.jpg,3
34 | 32,basketball_court/basketball_court_044.jpg,3
35 | 33,basketball_court/basketball_court_458.jpg,3
36 | 34,basketball_court/basketball_court_108.jpg,3
37 | 35,basketball_court/basketball_court_489.jpg,3
38 | 36,basketball_court/basketball_court_614.jpg,3
39 | 37,basketball_court/basketball_court_268.jpg,3
40 | 38,basketball_court/basketball_court_481.jpg,3
41 | 39,basketball_court/basketball_court_684.jpg,3
42 | 40,circular_farmland/circular_farmland_643.jpg,8
43 | 41,circular_farmland/circular_farmland_587.jpg,8
44 | 42,circular_farmland/circular_farmland_310.jpg,8
45 | 43,circular_farmland/circular_farmland_485.jpg,8
46 | 44,circular_farmland/circular_farmland_591.jpg,8
47 | 45,circular_farmland/circular_farmland_522.jpg,8
48 | 46,circular_farmland/circular_farmland_169.jpg,8
49 | 47,circular_farmland/circular_farmland_549.jpg,8
50 | 48,circular_farmland/circular_farmland_323.jpg,8
51 | 49,circular_farmland/circular_farmland_387.jpg,8
52 | 50,circular_farmland/circular_farmland_141.jpg,8
53 | 51,circular_farmland/circular_farmland_679.jpg,8
54 | 52,circular_farmland/circular_farmland_529.jpg,8
55 | 53,circular_farmland/circular_farmland_585.jpg,8
56 | 54,circular_farmland/circular_farmland_127.jpg,8
57 | 55,circular_farmland/circular_farmland_677.jpg,8
58 | 56,circular_farmland/circular_farmland_304.jpg,8
59 | 57,circular_farmland/circular_farmland_259.jpg,8
60 | 58,circular_farmland/circular_farmland_322.jpg,8
61 | 59,circular_farmland/circular_farmland_544.jpg,8
62 | 60,circular_farmland/circular_farmland_046.jpg,8
63 | 61,circular_farmland/circular_farmland_654.jpg,8
64 | 62,circular_farmland/circular_farmland_174.jpg,8
65 | 63,circular_farmland/circular_farmland_106.jpg,8
66 | 64,circular_farmland/circular_farmland_278.jpg,8
67 | 65,circular_farmland/circular_farmland_007.jpg,8
68 | 66,circular_farmland/circular_farmland_186.jpg,8
69 | 67,circular_farmland/circular_farmland_650.jpg,8
70 | 68,circular_farmland/circular_farmland_498.jpg,8
71 | 69,circular_farmland/circular_farmland_687.jpg,8
72 | 70,circular_farmland/circular_farmland_256.jpg,8
73 | 71,circular_farmland/circular_farmland_684.jpg,8
74 | 72,circular_farmland/circular_farmland_487.jpg,8
75 | 73,circular_farmland/circular_farmland_683.jpg,8
76 | 74,circular_farmland/circular_farmland_452.jpg,8
77 | 75,circular_farmland/circular_farmland_144.jpg,8
78 | 76,circular_farmland/circular_farmland_652.jpg,8
79 | 77,circular_farmland/circular_farmland_189.jpg,8
80 | 78,circular_farmland/circular_farmland_397.jpg,8
81 | 79,circular_farmland/circular_farmland_281.jpg,8
82 | 80,roundabout/roundabout_378.jpg,33
83 | 81,roundabout/roundabout_185.jpg,33
84 | 82,roundabout/roundabout_697.jpg,33
85 | 83,roundabout/roundabout_403.jpg,33
86 | 84,roundabout/roundabout_417.jpg,33
87 | 85,roundabout/roundabout_326.jpg,33
88 | 86,roundabout/roundabout_113.jpg,33
89 | 87,roundabout/roundabout_667.jpg,33
90 | 88,roundabout/roundabout_131.jpg,33
91 | 89,roundabout/roundabout_170.jpg,33
92 | 90,roundabout/roundabout_203.jpg,33
93 | 91,roundabout/roundabout_470.jpg,33
94 | 92,roundabout/roundabout_358.jpg,33
95 | 93,roundabout/roundabout_514.jpg,33
96 | 94,roundabout/roundabout_640.jpg,33
97 | 95,roundabout/roundabout_634.jpg,33
98 | 96,roundabout/roundabout_087.jpg,33
99 | 97,roundabout/roundabout_423.jpg,33
100 | 98,roundabout/roundabout_416.jpg,33
101 | 99,roundabout/roundabout_105.jpg,33
102 | 100,roundabout/roundabout_123.jpg,33
103 | 101,roundabout/roundabout_376.jpg,33
104 | 102,roundabout/roundabout_367.jpg,33
105 | 103,roundabout/roundabout_164.jpg,33
106 | 104,roundabout/roundabout_555.jpg,33
107 | 105,roundabout/roundabout_168.jpg,33
108 | 106,roundabout/roundabout_611.jpg,33
109 | 107,roundabout/roundabout_003.jpg,33
110 | 108,roundabout/roundabout_600.jpg,33
111 | 109,roundabout/roundabout_443.jpg,33
112 | 110,roundabout/roundabout_092.jpg,33
113 | 111,roundabout/roundabout_568.jpg,33
114 | 112,roundabout/roundabout_433.jpg,33
115 | 113,roundabout/roundabout_678.jpg,33
116 | 114,roundabout/roundabout_434.jpg,33
117 | 115,roundabout/roundabout_153.jpg,33
118 | 116,roundabout/roundabout_648.jpg,33
119 | 117,roundabout/roundabout_570.jpg,33
120 | 118,roundabout/roundabout_497.jpg,33
121 | 119,roundabout/roundabout_390.jpg,33
122 |
--------------------------------------------------------------------------------
/datafiles/xval-0.train.csv:
--------------------------------------------------------------------------------
1 | ,image,class_label
2 | 0,basketball_court/basketball_court_112.jpg,3
3 | 1,basketball_court/basketball_court_228.jpg,3
4 | 2,basketball_court/basketball_court_147.jpg,3
5 | 3,basketball_court/basketball_court_152.jpg,3
6 | 4,basketball_court/basketball_court_366.jpg,3
7 | 5,basketball_court/basketball_court_363.jpg,3
8 | 6,basketball_court/basketball_court_233.jpg,3
9 | 7,basketball_court/basketball_court_623.jpg,3
10 | 8,basketball_court/basketball_court_365.jpg,3
11 | 9,basketball_court/basketball_court_472.jpg,3
12 | 10,basketball_court/basketball_court_392.jpg,3
13 | 11,basketball_court/basketball_court_496.jpg,3
14 | 12,basketball_court/basketball_court_301.jpg,3
15 | 13,basketball_court/basketball_court_036.jpg,3
16 | 14,basketball_court/basketball_court_432.jpg,3
17 | 15,basketball_court/basketball_court_599.jpg,3
18 | 16,basketball_court/basketball_court_369.jpg,3
19 | 17,basketball_court/basketball_court_530.jpg,3
20 | 18,basketball_court/basketball_court_229.jpg,3
21 | 19,basketball_court/basketball_court_155.jpg,3
22 | 20,basketball_court/basketball_court_076.jpg,3
23 | 21,basketball_court/basketball_court_541.jpg,3
24 | 22,basketball_court/basketball_court_271.jpg,3
25 | 23,basketball_court/basketball_court_650.jpg,3
26 | 24,basketball_court/basketball_court_451.jpg,3
27 | 25,basketball_court/basketball_court_634.jpg,3
28 | 26,basketball_court/basketball_court_166.jpg,3
29 | 27,basketball_court/basketball_court_038.jpg,3
30 | 28,basketball_court/basketball_court_114.jpg,3
31 | 29,basketball_court/basketball_court_037.jpg,3
32 | 30,basketball_court/basketball_court_571.jpg,3
33 | 31,basketball_court/basketball_court_066.jpg,3
34 | 32,basketball_court/basketball_court_424.jpg,3
35 | 33,basketball_court/basketball_court_217.jpg,3
36 | 34,basketball_court/basketball_court_628.jpg,3
37 | 35,basketball_court/basketball_court_460.jpg,3
38 | 36,basketball_court/basketball_court_554.jpg,3
39 | 37,basketball_court/basketball_court_205.jpg,3
40 | 38,basketball_court/basketball_court_528.jpg,3
41 | 39,basketball_court/basketball_court_643.jpg,3
42 | 40,basketball_court/basketball_court_492.jpg,3
43 | 41,basketball_court/basketball_court_674.jpg,3
44 | 42,basketball_court/basketball_court_566.jpg,3
45 | 43,basketball_court/basketball_court_354.jpg,3
46 | 44,basketball_court/basketball_court_679.jpg,3
47 | 45,basketball_court/basketball_court_064.jpg,3
48 | 46,basketball_court/basketball_court_593.jpg,3
49 | 47,basketball_court/basketball_court_051.jpg,3
50 | 48,basketball_court/basketball_court_289.jpg,3
51 | 49,basketball_court/basketball_court_648.jpg,3
52 | 50,basketball_court/basketball_court_437.jpg,3
53 | 51,basketball_court/basketball_court_335.jpg,3
54 | 52,basketball_court/basketball_court_690.jpg,3
55 | 53,basketball_court/basketball_court_322.jpg,3
56 | 54,basketball_court/basketball_court_193.jpg,3
57 | 55,basketball_court/basketball_court_030.jpg,3
58 | 56,basketball_court/basketball_court_218.jpg,3
59 | 57,basketball_court/basketball_court_133.jpg,3
60 | 58,basketball_court/basketball_court_248.jpg,3
61 | 59,basketball_court/basketball_court_438.jpg,3
62 | 60,basketball_court/basketball_court_371.jpg,3
63 | 61,basketball_court/basketball_court_476.jpg,3
64 | 62,basketball_court/basketball_court_475.jpg,3
65 | 63,basketball_court/basketball_court_433.jpg,3
66 | 64,basketball_court/basketball_court_296.jpg,3
67 | 65,basketball_court/basketball_court_411.jpg,3
68 | 66,basketball_court/basketball_court_611.jpg,3
69 | 67,basketball_court/basketball_court_559.jpg,3
70 | 68,basketball_court/basketball_court_141.jpg,3
71 | 69,basketball_court/basketball_court_313.jpg,3
72 | 70,basketball_court/basketball_court_268.jpg,3
73 | 71,basketball_court/basketball_court_171.jpg,3
74 | 72,basketball_court/basketball_court_239.jpg,3
75 | 73,basketball_court/basketball_court_558.jpg,3
76 | 74,basketball_court/basketball_court_606.jpg,3
77 | 75,basketball_court/basketball_court_132.jpg,3
78 | 76,basketball_court/basketball_court_044.jpg,3
79 | 77,basketball_court/basketball_court_058.jpg,3
80 | 78,basketball_court/basketball_court_204.jpg,3
81 | 79,basketball_court/basketball_court_162.jpg,3
82 | 80,basketball_court/basketball_court_321.jpg,3
83 | 81,basketball_court/basketball_court_385.jpg,3
84 | 82,basketball_court/basketball_court_577.jpg,3
85 | 83,basketball_court/basketball_court_487.jpg,3
86 | 84,basketball_court/basketball_court_251.jpg,3
87 | 85,basketball_court/basketball_court_497.jpg,3
88 | 86,basketball_court/basketball_court_035.jpg,3
89 | 87,basketball_court/basketball_court_521.jpg,3
90 | 88,basketball_court/basketball_court_651.jpg,3
91 | 89,basketball_court/basketball_court_294.jpg,3
92 | 90,basketball_court/basketball_court_264.jpg,3
93 | 91,basketball_court/basketball_court_267.jpg,3
94 | 92,basketball_court/basketball_court_040.jpg,3
95 | 93,basketball_court/basketball_court_407.jpg,3
96 | 94,basketball_court/basketball_court_499.jpg,3
97 | 95,basketball_court/basketball_court_332.jpg,3
98 | 96,basketball_court/basketball_court_434.jpg,3
99 | 97,basketball_court/basketball_court_348.jpg,3
100 | 98,basketball_court/basketball_court_506.jpg,3
101 | 99,basketball_court/basketball_court_345.jpg,3
102 | 100,basketball_court/basketball_court_654.jpg,3
103 | 101,basketball_court/basketball_court_458.jpg,3
104 | 102,basketball_court/basketball_court_617.jpg,3
105 | 103,basketball_court/basketball_court_620.jpg,3
106 | 104,basketball_court/basketball_court_488.jpg,3
107 | 105,basketball_court/basketball_court_464.jpg,3
108 | 106,basketball_court/basketball_court_357.jpg,3
109 | 107,basketball_court/basketball_court_178.jpg,3
110 | 108,basketball_court/basketball_court_397.jpg,3
111 | 109,basketball_court/basketball_court_652.jpg,3
112 | 110,basketball_court/basketball_court_337.jpg,3
113 | 111,basketball_court/basketball_court_602.jpg,3
114 | 112,basketball_court/basketball_court_655.jpg,3
115 | 113,basketball_court/basketball_court_106.jpg,3
116 | 114,basketball_court/basketball_court_062.jpg,3
117 | 115,basketball_court/basketball_court_483.jpg,3
118 | 116,basketball_court/basketball_court_449.jpg,3
119 | 117,basketball_court/basketball_court_500.jpg,3
120 | 118,basketball_court/basketball_court_686.jpg,3
121 | 119,basketball_court/basketball_court_565.jpg,3
122 | 120,basketball_court/basketball_court_362.jpg,3
123 | 121,basketball_court/basketball_court_537.jpg,3
124 | 122,basketball_court/basketball_court_668.jpg,3
125 | 123,basketball_court/basketball_court_013.jpg,3
126 | 124,basketball_court/basketball_court_448.jpg,3
127 | 125,basketball_court/basketball_court_053.jpg,3
128 | 126,basketball_court/basketball_court_573.jpg,3
129 | 127,basketball_court/basketball_court_353.jpg,3
130 | 128,basketball_court/basketball_court_148.jpg,3
131 | 129,basketball_court/basketball_court_364.jpg,3
132 | 130,basketball_court/basketball_court_494.jpg,3
133 | 131,basketball_court/basketball_court_063.jpg,3
134 | 132,basketball_court/basketball_court_245.jpg,3
135 | 133,basketball_court/basketball_court_086.jpg,3
136 | 134,basketball_court/basketball_court_316.jpg,3
137 | 135,basketball_court/basketball_court_452.jpg,3
138 | 136,basketball_court/basketball_court_002.jpg,3
139 | 137,basketball_court/basketball_court_319.jpg,3
140 | 138,basketball_court/basketball_court_163.jpg,3
141 | 139,basketball_court/basketball_court_143.jpg,3
142 | 140,basketball_court/basketball_court_610.jpg,3
143 | 141,basketball_court/basketball_court_015.jpg,3
144 | 142,basketball_court/basketball_court_083.jpg,3
145 | 143,basketball_court/basketball_court_309.jpg,3
146 | 144,basketball_court/basketball_court_420.jpg,3
147 | 145,basketball_court/basketball_court_174.jpg,3
148 | 146,basketball_court/basketball_court_689.jpg,3
149 | 147,basketball_court/basketball_court_441.jpg,3
150 | 148,basketball_court/basketball_court_045.jpg,3
151 | 149,basketball_court/basketball_court_518.jpg,3
152 | 150,basketball_court/basketball_court_671.jpg,3
153 | 151,basketball_court/basketball_court_346.jpg,3
154 | 152,basketball_court/basketball_court_185.jpg,3
155 | 153,basketball_court/basketball_court_421.jpg,3
156 | 154,basketball_court/basketball_court_653.jpg,3
157 | 155,basketball_court/basketball_court_549.jpg,3
158 | 156,basketball_court/basketball_court_640.jpg,3
159 | 157,basketball_court/basketball_court_195.jpg,3
160 | 158,basketball_court/basketball_court_068.jpg,3
161 | 159,basketball_court/basketball_court_697.jpg,3
162 | 160,circular_farmland/circular_farmland_499.jpg,8
163 | 161,circular_farmland/circular_farmland_286.jpg,8
164 | 162,circular_farmland/circular_farmland_332.jpg,8
165 | 163,circular_farmland/circular_farmland_676.jpg,8
166 | 164,circular_farmland/circular_farmland_428.jpg,8
167 | 165,circular_farmland/circular_farmland_105.jpg,8
168 | 166,circular_farmland/circular_farmland_111.jpg,8
169 | 167,circular_farmland/circular_farmland_353.jpg,8
170 | 168,circular_farmland/circular_farmland_468.jpg,8
171 | 169,circular_farmland/circular_farmland_621.jpg,8
172 | 170,circular_farmland/circular_farmland_402.jpg,8
173 | 171,circular_farmland/circular_farmland_162.jpg,8
174 | 172,circular_farmland/circular_farmland_389.jpg,8
175 | 173,circular_farmland/circular_farmland_378.jpg,8
176 | 174,circular_farmland/circular_farmland_572.jpg,8
177 | 175,circular_farmland/circular_farmland_417.jpg,8
178 | 176,circular_farmland/circular_farmland_136.jpg,8
179 | 177,circular_farmland/circular_farmland_478.jpg,8
180 | 178,circular_farmland/circular_farmland_326.jpg,8
181 | 179,circular_farmland/circular_farmland_115.jpg,8
182 | 180,circular_farmland/circular_farmland_565.jpg,8
183 | 181,circular_farmland/circular_farmland_031.jpg,8
184 | 182,circular_farmland/circular_farmland_433.jpg,8
185 | 183,circular_farmland/circular_farmland_665.jpg,8
186 | 184,circular_farmland/circular_farmland_560.jpg,8
187 | 185,circular_farmland/circular_farmland_261.jpg,8
188 | 186,circular_farmland/circular_farmland_168.jpg,8
189 | 187,circular_farmland/circular_farmland_109.jpg,8
190 | 188,circular_farmland/circular_farmland_384.jpg,8
191 | 189,circular_farmland/circular_farmland_699.jpg,8
192 | 190,circular_farmland/circular_farmland_247.jpg,8
193 | 191,circular_farmland/circular_farmland_682.jpg,8
194 | 192,circular_farmland/circular_farmland_007.jpg,8
195 | 193,circular_farmland/circular_farmland_046.jpg,8
196 | 194,circular_farmland/circular_farmland_404.jpg,8
197 | 195,circular_farmland/circular_farmland_346.jpg,8
198 | 196,circular_farmland/circular_farmland_573.jpg,8
199 | 197,circular_farmland/circular_farmland_394.jpg,8
200 | 198,circular_farmland/circular_farmland_064.jpg,8
201 | 199,circular_farmland/circular_farmland_047.jpg,8
202 | 200,circular_farmland/circular_farmland_552.jpg,8
203 | 201,circular_farmland/circular_farmland_100.jpg,8
204 | 202,circular_farmland/circular_farmland_315.jpg,8
205 | 203,circular_farmland/circular_farmland_486.jpg,8
206 | 204,circular_farmland/circular_farmland_214.jpg,8
207 | 205,circular_farmland/circular_farmland_490.jpg,8
208 | 206,circular_farmland/circular_farmland_145.jpg,8
209 | 207,circular_farmland/circular_farmland_429.jpg,8
210 | 208,circular_farmland/circular_farmland_014.jpg,8
211 | 209,circular_farmland/circular_farmland_660.jpg,8
212 | 210,circular_farmland/circular_farmland_493.jpg,8
213 | 211,circular_farmland/circular_farmland_496.jpg,8
214 | 212,circular_farmland/circular_farmland_577.jpg,8
215 | 213,circular_farmland/circular_farmland_460.jpg,8
216 | 214,circular_farmland/circular_farmland_150.jpg,8
217 | 215,circular_farmland/circular_farmland_246.jpg,8
218 | 216,circular_farmland/circular_farmland_197.jpg,8
219 | 217,circular_farmland/circular_farmland_392.jpg,8
220 | 218,circular_farmland/circular_farmland_196.jpg,8
221 | 219,circular_farmland/circular_farmland_673.jpg,8
222 | 220,circular_farmland/circular_farmland_331.jpg,8
223 | 221,circular_farmland/circular_farmland_622.jpg,8
224 | 222,circular_farmland/circular_farmland_355.jpg,8
225 | 223,circular_farmland/circular_farmland_672.jpg,8
226 | 224,circular_farmland/circular_farmland_321.jpg,8
227 | 225,circular_farmland/circular_farmland_349.jpg,8
228 | 226,circular_farmland/circular_farmland_360.jpg,8
229 | 227,circular_farmland/circular_farmland_623.jpg,8
230 | 228,circular_farmland/circular_farmland_530.jpg,8
231 | 229,circular_farmland/circular_farmland_093.jpg,8
232 | 230,circular_farmland/circular_farmland_274.jpg,8
233 | 231,circular_farmland/circular_farmland_275.jpg,8
234 | 232,circular_farmland/circular_farmland_566.jpg,8
235 | 233,circular_farmland/circular_farmland_088.jpg,8
236 | 234,circular_farmland/circular_farmland_260.jpg,8
237 | 235,circular_farmland/circular_farmland_110.jpg,8
238 | 236,circular_farmland/circular_farmland_615.jpg,8
239 | 237,circular_farmland/circular_farmland_079.jpg,8
240 | 238,circular_farmland/circular_farmland_432.jpg,8
241 | 239,circular_farmland/circular_farmland_553.jpg,8
242 | 240,circular_farmland/circular_farmland_311.jpg,8
243 | 241,circular_farmland/circular_farmland_017.jpg,8
244 | 242,circular_farmland/circular_farmland_154.jpg,8
245 | 243,circular_farmland/circular_farmland_700.jpg,8
246 | 244,circular_farmland/circular_farmland_465.jpg,8
247 | 245,circular_farmland/circular_farmland_101.jpg,8
248 | 246,circular_farmland/circular_farmland_661.jpg,8
249 | 247,circular_farmland/circular_farmland_365.jpg,8
250 | 248,circular_farmland/circular_farmland_089.jpg,8
251 | 249,circular_farmland/circular_farmland_647.jpg,8
252 | 250,circular_farmland/circular_farmland_204.jpg,8
253 | 251,circular_farmland/circular_farmland_588.jpg,8
254 | 252,circular_farmland/circular_farmland_063.jpg,8
255 | 253,circular_farmland/circular_farmland_328.jpg,8
256 | 254,circular_farmland/circular_farmland_629.jpg,8
257 | 255,circular_farmland/circular_farmland_191.jpg,8
258 | 256,circular_farmland/circular_farmland_691.jpg,8
259 | 257,circular_farmland/circular_farmland_016.jpg,8
260 | 258,circular_farmland/circular_farmland_545.jpg,8
261 | 259,circular_farmland/circular_farmland_371.jpg,8
262 | 260,circular_farmland/circular_farmland_498.jpg,8
263 | 261,circular_farmland/circular_farmland_407.jpg,8
264 | 262,circular_farmland/circular_farmland_301.jpg,8
265 | 263,circular_farmland/circular_farmland_442.jpg,8
266 | 264,circular_farmland/circular_farmland_418.jpg,8
267 | 265,circular_farmland/circular_farmland_387.jpg,8
268 | 266,circular_farmland/circular_farmland_348.jpg,8
269 | 267,circular_farmland/circular_farmland_456.jpg,8
270 | 268,circular_farmland/circular_farmland_636.jpg,8
271 | 269,circular_farmland/circular_farmland_018.jpg,8
272 | 270,circular_farmland/circular_farmland_470.jpg,8
273 | 271,circular_farmland/circular_farmland_195.jpg,8
274 | 272,circular_farmland/circular_farmland_605.jpg,8
275 | 273,circular_farmland/circular_farmland_667.jpg,8
276 | 274,circular_farmland/circular_farmland_243.jpg,8
277 | 275,circular_farmland/circular_farmland_009.jpg,8
278 | 276,circular_farmland/circular_farmland_383.jpg,8
279 | 277,circular_farmland/circular_farmland_603.jpg,8
280 | 278,circular_farmland/circular_farmland_211.jpg,8
281 | 279,circular_farmland/circular_farmland_138.jpg,8
282 | 280,circular_farmland/circular_farmland_303.jpg,8
283 | 281,circular_farmland/circular_farmland_528.jpg,8
284 | 282,circular_farmland/circular_farmland_106.jpg,8
285 | 283,circular_farmland/circular_farmland_217.jpg,8
286 | 284,circular_farmland/circular_farmland_358.jpg,8
287 | 285,circular_farmland/circular_farmland_690.jpg,8
288 | 286,circular_farmland/circular_farmland_539.jpg,8
289 | 287,circular_farmland/circular_farmland_542.jpg,8
290 | 288,circular_farmland/circular_farmland_013.jpg,8
291 | 289,circular_farmland/circular_farmland_380.jpg,8
292 | 290,circular_farmland/circular_farmland_270.jpg,8
293 | 291,circular_farmland/circular_farmland_299.jpg,8
294 | 292,circular_farmland/circular_farmland_379.jpg,8
295 | 293,circular_farmland/circular_farmland_628.jpg,8
296 | 294,circular_farmland/circular_farmland_532.jpg,8
297 | 295,circular_farmland/circular_farmland_085.jpg,8
298 | 296,circular_farmland/circular_farmland_678.jpg,8
299 | 297,circular_farmland/circular_farmland_631.jpg,8
300 | 298,circular_farmland/circular_farmland_157.jpg,8
301 | 299,circular_farmland/circular_farmland_519.jpg,8
302 | 300,circular_farmland/circular_farmland_279.jpg,8
303 | 301,circular_farmland/circular_farmland_585.jpg,8
304 | 302,circular_farmland/circular_farmland_521.jpg,8
305 | 303,circular_farmland/circular_farmland_054.jpg,8
306 | 304,circular_farmland/circular_farmland_034.jpg,8
307 | 305,circular_farmland/circular_farmland_053.jpg,8
308 | 306,circular_farmland/circular_farmland_599.jpg,8
309 | 307,circular_farmland/circular_farmland_209.jpg,8
310 | 308,circular_farmland/circular_farmland_238.jpg,8
311 | 309,circular_farmland/circular_farmland_576.jpg,8
312 | 310,circular_farmland/circular_farmland_096.jpg,8
313 | 311,circular_farmland/circular_farmland_263.jpg,8
314 | 312,circular_farmland/circular_farmland_114.jpg,8
315 | 313,circular_farmland/circular_farmland_323.jpg,8
316 | 314,circular_farmland/circular_farmland_350.jpg,8
317 | 315,circular_farmland/circular_farmland_236.jpg,8
318 | 316,circular_farmland/circular_farmland_300.jpg,8
319 | 317,circular_farmland/circular_farmland_258.jpg,8
320 | 318,circular_farmland/circular_farmland_612.jpg,8
321 | 319,circular_farmland/circular_farmland_374.jpg,8
322 | 320,roundabout/roundabout_094.jpg,33
323 | 321,roundabout/roundabout_581.jpg,33
324 | 322,roundabout/roundabout_631.jpg,33
325 | 323,roundabout/roundabout_668.jpg,33
326 | 324,roundabout/roundabout_697.jpg,33
327 | 325,roundabout/roundabout_346.jpg,33
328 | 326,roundabout/roundabout_212.jpg,33
329 | 327,roundabout/roundabout_530.jpg,33
330 | 328,roundabout/roundabout_661.jpg,33
331 | 329,roundabout/roundabout_662.jpg,33
332 | 330,roundabout/roundabout_143.jpg,33
333 | 331,roundabout/roundabout_150.jpg,33
334 | 332,roundabout/roundabout_124.jpg,33
335 | 333,roundabout/roundabout_677.jpg,33
336 | 334,roundabout/roundabout_066.jpg,33
337 | 335,roundabout/roundabout_038.jpg,33
338 | 336,roundabout/roundabout_086.jpg,33
339 | 337,roundabout/roundabout_590.jpg,33
340 | 338,roundabout/roundabout_337.jpg,33
341 | 339,roundabout/roundabout_173.jpg,33
342 | 340,roundabout/roundabout_471.jpg,33
343 | 341,roundabout/roundabout_554.jpg,33
344 | 342,roundabout/roundabout_539.jpg,33
345 | 343,roundabout/roundabout_367.jpg,33
346 | 344,roundabout/roundabout_475.jpg,33
347 | 345,roundabout/roundabout_245.jpg,33
348 | 346,roundabout/roundabout_241.jpg,33
349 | 347,roundabout/roundabout_213.jpg,33
350 | 348,roundabout/roundabout_603.jpg,33
351 | 349,roundabout/roundabout_158.jpg,33
352 | 350,roundabout/roundabout_525.jpg,33
353 | 351,roundabout/roundabout_095.jpg,33
354 | 352,roundabout/roundabout_553.jpg,33
355 | 353,roundabout/roundabout_295.jpg,33
356 | 354,roundabout/roundabout_429.jpg,33
357 | 355,roundabout/roundabout_001.jpg,33
358 | 356,roundabout/roundabout_113.jpg,33
359 | 357,roundabout/roundabout_052.jpg,33
360 | 358,roundabout/roundabout_678.jpg,33
361 | 359,roundabout/roundabout_408.jpg,33
362 | 360,roundabout/roundabout_282.jpg,33
363 | 361,roundabout/roundabout_577.jpg,33
364 | 362,roundabout/roundabout_104.jpg,33
365 | 363,roundabout/roundabout_695.jpg,33
366 | 364,roundabout/roundabout_064.jpg,33
367 | 365,roundabout/roundabout_485.jpg,33
368 | 366,roundabout/roundabout_211.jpg,33
369 | 367,roundabout/roundabout_527.jpg,33
370 | 368,roundabout/roundabout_564.jpg,33
371 | 369,roundabout/roundabout_411.jpg,33
372 | 370,roundabout/roundabout_204.jpg,33
373 | 371,roundabout/roundabout_560.jpg,33
374 | 372,roundabout/roundabout_215.jpg,33
375 | 373,roundabout/roundabout_198.jpg,33
376 | 374,roundabout/roundabout_338.jpg,33
377 | 375,roundabout/roundabout_293.jpg,33
378 | 376,roundabout/roundabout_138.jpg,33
379 | 377,roundabout/roundabout_568.jpg,33
380 | 378,roundabout/roundabout_674.jpg,33
381 | 379,roundabout/roundabout_334.jpg,33
382 | 380,roundabout/roundabout_477.jpg,33
383 | 381,roundabout/roundabout_264.jpg,33
384 | 382,roundabout/roundabout_645.jpg,33
385 | 383,roundabout/roundabout_252.jpg,33
386 | 384,roundabout/roundabout_438.jpg,33
387 | 385,roundabout/roundabout_491.jpg,33
388 | 386,roundabout/roundabout_700.jpg,33
389 | 387,roundabout/roundabout_452.jpg,33
390 | 388,roundabout/roundabout_002.jpg,33
391 | 389,roundabout/roundabout_403.jpg,33
392 | 390,roundabout/roundabout_304.jpg,33
393 | 391,roundabout/roundabout_407.jpg,33
394 | 392,roundabout/roundabout_132.jpg,33
395 | 393,roundabout/roundabout_693.jpg,33
396 | 394,roundabout/roundabout_607.jpg,33
397 | 395,roundabout/roundabout_008.jpg,33
398 | 396,roundabout/roundabout_416.jpg,33
399 | 397,roundabout/roundabout_216.jpg,33
400 | 398,roundabout/roundabout_224.jpg,33
401 | 399,roundabout/roundabout_134.jpg,33
402 | 400,roundabout/roundabout_060.jpg,33
403 | 401,roundabout/roundabout_375.jpg,33
404 | 402,roundabout/roundabout_270.jpg,33
405 | 403,roundabout/roundabout_538.jpg,33
406 | 404,roundabout/roundabout_195.jpg,33
407 | 405,roundabout/roundabout_344.jpg,33
408 | 406,roundabout/roundabout_310.jpg,33
409 | 407,roundabout/roundabout_281.jpg,33
410 | 408,roundabout/roundabout_465.jpg,33
411 | 409,roundabout/roundabout_151.jpg,33
412 | 410,roundabout/roundabout_230.jpg,33
413 | 411,roundabout/roundabout_253.jpg,33
414 | 412,roundabout/roundabout_176.jpg,33
415 | 413,roundabout/roundabout_178.jpg,33
416 | 414,roundabout/roundabout_468.jpg,33
417 | 415,roundabout/roundabout_302.jpg,33
418 | 416,roundabout/roundabout_460.jpg,33
419 | 417,roundabout/roundabout_019.jpg,33
420 | 418,roundabout/roundabout_074.jpg,33
421 | 419,roundabout/roundabout_540.jpg,33
422 | 420,roundabout/roundabout_303.jpg,33
423 | 421,roundabout/roundabout_272.jpg,33
424 | 422,roundabout/roundabout_635.jpg,33
425 | 423,roundabout/roundabout_339.jpg,33
426 | 424,roundabout/roundabout_262.jpg,33
427 | 425,roundabout/roundabout_393.jpg,33
428 | 426,roundabout/roundabout_036.jpg,33
429 | 427,roundabout/roundabout_463.jpg,33
430 | 428,roundabout/roundabout_202.jpg,33
431 | 429,roundabout/roundabout_377.jpg,33
432 | 430,roundabout/roundabout_049.jpg,33
433 | 431,roundabout/roundabout_656.jpg,33
434 | 432,roundabout/roundabout_357.jpg,33
435 | 433,roundabout/roundabout_137.jpg,33
436 | 434,roundabout/roundabout_599.jpg,33
437 | 435,roundabout/roundabout_417.jpg,33
438 | 436,roundabout/roundabout_040.jpg,33
439 | 437,roundabout/roundabout_640.jpg,33
440 | 438,roundabout/roundabout_523.jpg,33
441 | 439,roundabout/roundabout_317.jpg,33
442 | 440,roundabout/roundabout_069.jpg,33
443 | 441,roundabout/roundabout_127.jpg,33
444 | 442,roundabout/roundabout_296.jpg,33
445 | 443,roundabout/roundabout_186.jpg,33
446 | 444,roundabout/roundabout_247.jpg,33
447 | 445,roundabout/roundabout_425.jpg,33
448 | 446,roundabout/roundabout_005.jpg,33
449 | 447,roundabout/roundabout_084.jpg,33
450 | 448,roundabout/roundabout_026.jpg,33
451 | 449,roundabout/roundabout_091.jpg,33
452 | 450,roundabout/roundabout_030.jpg,33
453 | 451,roundabout/roundabout_171.jpg,33
454 | 452,roundabout/roundabout_365.jpg,33
455 | 453,roundabout/roundabout_555.jpg,33
456 | 454,roundabout/roundabout_021.jpg,33
457 | 455,roundabout/roundabout_006.jpg,33
458 | 456,roundabout/roundabout_636.jpg,33
459 | 457,roundabout/roundabout_356.jpg,33
460 | 458,roundabout/roundabout_602.jpg,33
461 | 459,roundabout/roundabout_170.jpg,33
462 | 460,roundabout/roundabout_415.jpg,33
463 | 461,roundabout/roundabout_501.jpg,33
464 | 462,roundabout/roundabout_308.jpg,33
465 | 463,roundabout/roundabout_505.jpg,33
466 | 464,roundabout/roundabout_531.jpg,33
467 | 465,roundabout/roundabout_672.jpg,33
468 | 466,roundabout/roundabout_441.jpg,33
469 | 467,roundabout/roundabout_355.jpg,33
470 | 468,roundabout/roundabout_632.jpg,33
471 | 469,roundabout/roundabout_557.jpg,33
472 | 470,roundabout/roundabout_681.jpg,33
473 | 471,roundabout/roundabout_432.jpg,33
474 | 472,roundabout/roundabout_050.jpg,33
475 | 473,roundabout/roundabout_090.jpg,33
476 | 474,roundabout/roundabout_098.jpg,33
477 | 475,roundabout/roundabout_694.jpg,33
478 | 476,roundabout/roundabout_559.jpg,33
479 | 477,roundabout/roundabout_549.jpg,33
480 | 478,roundabout/roundabout_250.jpg,33
481 | 479,roundabout/roundabout_285.jpg,33
482 |
--------------------------------------------------------------------------------
/datafiles/xval-1.test.csv:
--------------------------------------------------------------------------------
1 | ,image,class_label
2 | 0,basketball_court/basketball_court_295.jpg,3
3 | 1,basketball_court/basketball_court_540.jpg,3
4 | 2,basketball_court/basketball_court_635.jpg,3
5 | 3,basketball_court/basketball_court_332.jpg,3
6 | 4,basketball_court/basketball_court_368.jpg,3
7 | 5,basketball_court/basketball_court_583.jpg,3
8 | 6,basketball_court/basketball_court_281.jpg,3
9 | 7,basketball_court/basketball_court_132.jpg,3
10 | 8,basketball_court/basketball_court_093.jpg,3
11 | 9,basketball_court/basketball_court_208.jpg,3
12 | 10,basketball_court/basketball_court_532.jpg,3
13 | 11,basketball_court/basketball_court_580.jpg,3
14 | 12,basketball_court/basketball_court_001.jpg,3
15 | 13,basketball_court/basketball_court_021.jpg,3
16 | 14,basketball_court/basketball_court_663.jpg,3
17 | 15,basketball_court/basketball_court_053.jpg,3
18 | 16,basketball_court/basketball_court_329.jpg,3
19 | 17,basketball_court/basketball_court_163.jpg,3
20 | 18,basketball_court/basketball_court_425.jpg,3
21 | 19,basketball_court/basketball_court_652.jpg,3
22 | 20,basketball_court/basketball_court_556.jpg,3
23 | 21,basketball_court/basketball_court_199.jpg,3
24 | 22,basketball_court/basketball_court_203.jpg,3
25 | 23,basketball_court/basketball_court_533.jpg,3
26 | 24,basketball_court/basketball_court_090.jpg,3
27 | 25,basketball_court/basketball_court_459.jpg,3
28 | 26,basketball_court/basketball_court_031.jpg,3
29 | 27,basketball_court/basketball_court_179.jpg,3
30 | 28,basketball_court/basketball_court_427.jpg,3
31 | 29,basketball_court/basketball_court_039.jpg,3
32 | 30,basketball_court/basketball_court_210.jpg,3
33 | 31,basketball_court/basketball_court_455.jpg,3
34 | 32,basketball_court/basketball_court_024.jpg,3
35 | 33,basketball_court/basketball_court_149.jpg,3
36 | 34,basketball_court/basketball_court_059.jpg,3
37 | 35,basketball_court/basketball_court_054.jpg,3
38 | 36,basketball_court/basketball_court_442.jpg,3
39 | 37,basketball_court/basketball_court_642.jpg,3
40 | 38,basketball_court/basketball_court_126.jpg,3
41 | 39,basketball_court/basketball_court_231.jpg,3
42 | 40,circular_farmland/circular_farmland_162.jpg,8
43 | 41,circular_farmland/circular_farmland_195.jpg,8
44 | 42,circular_farmland/circular_farmland_166.jpg,8
45 | 43,circular_farmland/circular_farmland_374.jpg,8
46 | 44,circular_farmland/circular_farmland_051.jpg,8
47 | 45,circular_farmland/circular_farmland_287.jpg,8
48 | 46,circular_farmland/circular_farmland_071.jpg,8
49 | 47,circular_farmland/circular_farmland_174.jpg,8
50 | 48,circular_farmland/circular_farmland_334.jpg,8
51 | 49,circular_farmland/circular_farmland_357.jpg,8
52 | 50,circular_farmland/circular_farmland_284.jpg,8
53 | 51,circular_farmland/circular_farmland_226.jpg,8
54 | 52,circular_farmland/circular_farmland_167.jpg,8
55 | 53,circular_farmland/circular_farmland_227.jpg,8
56 | 54,circular_farmland/circular_farmland_415.jpg,8
57 | 55,circular_farmland/circular_farmland_072.jpg,8
58 | 56,circular_farmland/circular_farmland_499.jpg,8
59 | 57,circular_farmland/circular_farmland_375.jpg,8
60 | 58,circular_farmland/circular_farmland_131.jpg,8
61 | 59,circular_farmland/circular_farmland_539.jpg,8
62 | 60,circular_farmland/circular_farmland_652.jpg,8
63 | 61,circular_farmland/circular_farmland_186.jpg,8
64 | 62,circular_farmland/circular_farmland_327.jpg,8
65 | 63,circular_farmland/circular_farmland_501.jpg,8
66 | 64,circular_farmland/circular_farmland_235.jpg,8
67 | 65,circular_farmland/circular_farmland_106.jpg,8
68 | 66,circular_farmland/circular_farmland_416.jpg,8
69 | 67,circular_farmland/circular_farmland_111.jpg,8
70 | 68,circular_farmland/circular_farmland_421.jpg,8
71 | 69,circular_farmland/circular_farmland_171.jpg,8
72 | 70,circular_farmland/circular_farmland_191.jpg,8
73 | 71,circular_farmland/circular_farmland_513.jpg,8
74 | 72,circular_farmland/circular_farmland_442.jpg,8
75 | 73,circular_farmland/circular_farmland_681.jpg,8
76 | 74,circular_farmland/circular_farmland_056.jpg,8
77 | 75,circular_farmland/circular_farmland_312.jpg,8
78 | 76,circular_farmland/circular_farmland_409.jpg,8
79 | 77,circular_farmland/circular_farmland_343.jpg,8
80 | 78,circular_farmland/circular_farmland_550.jpg,8
81 | 79,circular_farmland/circular_farmland_160.jpg,8
82 | 80,roundabout/roundabout_387.jpg,33
83 | 81,roundabout/roundabout_075.jpg,33
84 | 82,roundabout/roundabout_385.jpg,33
85 | 83,roundabout/roundabout_438.jpg,33
86 | 84,roundabout/roundabout_301.jpg,33
87 | 85,roundabout/roundabout_575.jpg,33
88 | 86,roundabout/roundabout_519.jpg,33
89 | 87,roundabout/roundabout_392.jpg,33
90 | 88,roundabout/roundabout_351.jpg,33
91 | 89,roundabout/roundabout_294.jpg,33
92 | 90,roundabout/roundabout_695.jpg,33
93 | 91,roundabout/roundabout_333.jpg,33
94 | 92,roundabout/roundabout_641.jpg,33
95 | 93,roundabout/roundabout_406.jpg,33
96 | 94,roundabout/roundabout_690.jpg,33
97 | 95,roundabout/roundabout_228.jpg,33
98 | 96,roundabout/roundabout_320.jpg,33
99 | 97,roundabout/roundabout_259.jpg,33
100 | 98,roundabout/roundabout_475.jpg,33
101 | 99,roundabout/roundabout_481.jpg,33
102 | 100,roundabout/roundabout_515.jpg,33
103 | 101,roundabout/roundabout_237.jpg,33
104 | 102,roundabout/roundabout_307.jpg,33
105 | 103,roundabout/roundabout_531.jpg,33
106 | 104,roundabout/roundabout_031.jpg,33
107 | 105,roundabout/roundabout_405.jpg,33
108 | 106,roundabout/roundabout_670.jpg,33
109 | 107,roundabout/roundabout_171.jpg,33
110 | 108,roundabout/roundabout_209.jpg,33
111 | 109,roundabout/roundabout_225.jpg,33
112 | 110,roundabout/roundabout_447.jpg,33
113 | 111,roundabout/roundabout_486.jpg,33
114 | 112,roundabout/roundabout_644.jpg,33
115 | 113,roundabout/roundabout_629.jpg,33
116 | 114,roundabout/roundabout_299.jpg,33
117 | 115,roundabout/roundabout_444.jpg,33
118 | 116,roundabout/roundabout_053.jpg,33
119 | 117,roundabout/roundabout_491.jpg,33
120 | 118,roundabout/roundabout_067.jpg,33
121 | 119,roundabout/roundabout_083.jpg,33
122 |
--------------------------------------------------------------------------------
/datafiles/xval-1.train.csv:
--------------------------------------------------------------------------------
1 | ,image,class_label
2 | 0,basketball_court/basketball_court_007.jpg,3
3 | 1,basketball_court/basketball_court_550.jpg,3
4 | 2,basketball_court/basketball_court_308.jpg,3
5 | 3,basketball_court/basketball_court_247.jpg,3
6 | 4,basketball_court/basketball_court_029.jpg,3
7 | 5,basketball_court/basketball_court_492.jpg,3
8 | 6,basketball_court/basketball_court_366.jpg,3
9 | 7,basketball_court/basketball_court_243.jpg,3
10 | 8,basketball_court/basketball_court_035.jpg,3
11 | 9,basketball_court/basketball_court_298.jpg,3
12 | 10,basketball_court/basketball_court_326.jpg,3
13 | 11,basketball_court/basketball_court_678.jpg,3
14 | 12,basketball_court/basketball_court_218.jpg,3
15 | 13,basketball_court/basketball_court_622.jpg,3
16 | 14,basketball_court/basketball_court_567.jpg,3
17 | 15,basketball_court/basketball_court_217.jpg,3
18 | 16,basketball_court/basketball_court_104.jpg,3
19 | 17,basketball_court/basketball_court_379.jpg,3
20 | 18,basketball_court/basketball_court_283.jpg,3
21 | 19,basketball_court/basketball_court_586.jpg,3
22 | 20,basketball_court/basketball_court_503.jpg,3
23 | 21,basketball_court/basketball_court_446.jpg,3
24 | 22,basketball_court/basketball_court_156.jpg,3
25 | 23,basketball_court/basketball_court_620.jpg,3
26 | 24,basketball_court/basketball_court_472.jpg,3
27 | 25,basketball_court/basketball_court_499.jpg,3
28 | 26,basketball_court/basketball_court_286.jpg,3
29 | 27,basketball_court/basketball_court_603.jpg,3
30 | 28,basketball_court/basketball_court_201.jpg,3
31 | 29,basketball_court/basketball_court_664.jpg,3
32 | 30,basketball_court/basketball_court_006.jpg,3
33 | 31,basketball_court/basketball_court_136.jpg,3
34 | 32,basketball_court/basketball_court_520.jpg,3
35 | 33,basketball_court/basketball_court_216.jpg,3
36 | 34,basketball_court/basketball_court_663.jpg,3
37 | 35,basketball_court/basketball_court_482.jpg,3
38 | 36,basketball_court/basketball_court_329.jpg,3
39 | 37,basketball_court/basketball_court_315.jpg,3
40 | 38,basketball_court/basketball_court_665.jpg,3
41 | 39,basketball_court/basketball_court_250.jpg,3
42 | 40,basketball_court/basketball_court_667.jpg,3
43 | 41,basketball_court/basketball_court_137.jpg,3
44 | 42,basketball_court/basketball_court_639.jpg,3
45 | 43,basketball_court/basketball_court_226.jpg,3
46 | 44,basketball_court/basketball_court_214.jpg,3
47 | 45,basketball_court/basketball_court_203.jpg,3
48 | 46,basketball_court/basketball_court_002.jpg,3
49 | 47,basketball_court/basketball_court_382.jpg,3
50 | 48,basketball_court/basketball_court_358.jpg,3
51 | 49,basketball_court/basketball_court_440.jpg,3
52 | 50,basketball_court/basketball_court_050.jpg,3
53 | 51,basketball_court/basketball_court_075.jpg,3
54 | 52,basketball_court/basketball_court_479.jpg,3
55 | 53,basketball_court/basketball_court_675.jpg,3
56 | 54,basketball_court/basketball_court_004.jpg,3
57 | 55,basketball_court/basketball_court_289.jpg,3
58 | 56,basketball_court/basketball_court_654.jpg,3
59 | 57,basketball_court/basketball_court_640.jpg,3
60 | 58,basketball_court/basketball_court_589.jpg,3
61 | 59,basketball_court/basketball_court_152.jpg,3
62 | 60,basketball_court/basketball_court_343.jpg,3
63 | 61,basketball_court/basketball_court_508.jpg,3
64 | 62,basketball_court/basketball_court_287.jpg,3
65 | 63,basketball_court/basketball_court_261.jpg,3
66 | 64,basketball_court/basketball_court_164.jpg,3
67 | 65,basketball_court/basketball_court_524.jpg,3
68 | 66,basketball_court/basketball_court_023.jpg,3
69 | 67,basketball_court/basketball_court_182.jpg,3
70 | 68,basketball_court/basketball_court_309.jpg,3
71 | 69,basketball_court/basketball_court_433.jpg,3
72 | 70,basketball_court/basketball_court_054.jpg,3
73 | 71,basketball_court/basketball_court_296.jpg,3
74 | 72,basketball_court/basketball_court_300.jpg,3
75 | 73,basketball_court/basketball_court_304.jpg,3
76 | 74,basketball_court/basketball_court_645.jpg,3
77 | 75,basketball_court/basketball_court_103.jpg,3
78 | 76,basketball_court/basketball_court_364.jpg,3
79 | 77,basketball_court/basketball_court_470.jpg,3
80 | 78,basketball_court/basketball_court_619.jpg,3
81 | 79,basketball_court/basketball_court_078.jpg,3
82 | 80,basketball_court/basketball_court_157.jpg,3
83 | 81,basketball_court/basketball_court_345.jpg,3
84 | 82,basketball_court/basketball_court_669.jpg,3
85 | 83,basketball_court/basketball_court_013.jpg,3
86 | 84,basketball_court/basketball_court_209.jpg,3
87 | 85,basketball_court/basketball_court_262.jpg,3
88 | 86,basketball_court/basketball_court_570.jpg,3
89 | 87,basketball_court/basketball_court_193.jpg,3
90 | 88,basketball_court/basketball_court_548.jpg,3
91 | 89,basketball_court/basketball_court_031.jpg,3
92 | 90,basketball_court/basketball_court_465.jpg,3
93 | 91,basketball_court/basketball_court_571.jpg,3
94 | 92,basketball_court/basketball_court_205.jpg,3
95 | 93,basketball_court/basketball_court_509.jpg,3
96 | 94,basketball_court/basketball_court_317.jpg,3
97 | 95,basketball_court/basketball_court_485.jpg,3
98 | 96,basketball_court/basketball_court_301.jpg,3
99 | 97,basketball_court/basketball_court_271.jpg,3
100 | 98,basketball_court/basketball_court_406.jpg,3
101 | 99,basketball_court/basketball_court_063.jpg,3
102 | 100,basketball_court/basketball_court_691.jpg,3
103 | 101,basketball_court/basketball_court_480.jpg,3
104 | 102,basketball_court/basketball_court_456.jpg,3
105 | 103,basketball_court/basketball_court_688.jpg,3
106 | 104,basketball_court/basketball_court_674.jpg,3
107 | 105,basketball_court/basketball_court_424.jpg,3
108 | 106,basketball_court/basketball_court_109.jpg,3
109 | 107,basketball_court/basketball_court_543.jpg,3
110 | 108,basketball_court/basketball_court_166.jpg,3
111 | 109,basketball_court/basketball_court_388.jpg,3
112 | 110,basketball_court/basketball_court_401.jpg,3
113 | 111,basketball_court/basketball_court_179.jpg,3
114 | 112,basketball_court/basketball_court_552.jpg,3
115 | 113,basketball_court/basketball_court_689.jpg,3
116 | 114,basketball_court/basketball_court_359.jpg,3
117 | 115,basketball_court/basketball_court_053.jpg,3
118 | 116,basketball_court/basketball_court_452.jpg,3
119 | 117,basketball_court/basketball_court_690.jpg,3
120 | 118,basketball_court/basketball_court_574.jpg,3
121 | 119,basketball_court/basketball_court_425.jpg,3
122 | 120,basketball_court/basketball_court_535.jpg,3
123 | 121,basketball_court/basketball_court_528.jpg,3
124 | 122,basketball_court/basketball_court_180.jpg,3
125 | 123,basketball_court/basketball_court_680.jpg,3
126 | 124,basketball_court/basketball_court_541.jpg,3
127 | 125,basketball_court/basketball_court_263.jpg,3
128 | 126,basketball_court/basketball_court_421.jpg,3
129 | 127,basketball_court/basketball_court_144.jpg,3
130 | 128,basketball_court/basketball_court_310.jpg,3
131 | 129,basketball_court/basketball_court_507.jpg,3
132 | 130,basketball_court/basketball_court_297.jpg,3
133 | 131,basketball_court/basketball_court_403.jpg,3
134 | 132,basketball_court/basketball_court_521.jpg,3
135 | 133,basketball_court/basketball_court_371.jpg,3
136 | 134,basketball_court/basketball_court_272.jpg,3
137 | 135,basketball_court/basketball_court_222.jpg,3
138 | 136,basketball_court/basketball_court_427.jpg,3
139 | 137,basketball_court/basketball_court_658.jpg,3
140 | 138,basketball_court/basketball_court_040.jpg,3
141 | 139,basketball_court/basketball_court_202.jpg,3
142 | 140,basketball_court/basketball_court_236.jpg,3
143 | 141,basketball_court/basketball_court_185.jpg,3
144 | 142,basketball_court/basketball_court_546.jpg,3
145 | 143,basketball_court/basketball_court_618.jpg,3
146 | 144,basketball_court/basketball_court_407.jpg,3
147 | 145,basketball_court/basketball_court_464.jpg,3
148 | 146,basketball_court/basketball_court_394.jpg,3
149 | 147,basketball_court/basketball_court_342.jpg,3
150 | 148,basketball_court/basketball_court_011.jpg,3
151 | 149,basketball_court/basketball_court_572.jpg,3
152 | 150,basketball_court/basketball_court_350.jpg,3
153 | 151,basketball_court/basketball_court_449.jpg,3
154 | 152,basketball_court/basketball_court_686.jpg,3
155 | 153,basketball_court/basketball_court_294.jpg,3
156 | 154,basketball_court/basketball_court_558.jpg,3
157 | 155,basketball_court/basketball_court_220.jpg,3
158 | 156,basketball_court/basketball_court_526.jpg,3
159 | 157,basketball_court/basketball_court_230.jpg,3
160 | 158,basketball_court/basketball_court_418.jpg,3
161 | 159,basketball_court/basketball_court_527.jpg,3
162 | 160,circular_farmland/circular_farmland_020.jpg,8
163 | 161,circular_farmland/circular_farmland_206.jpg,8
164 | 162,circular_farmland/circular_farmland_651.jpg,8
165 | 163,circular_farmland/circular_farmland_576.jpg,8
166 | 164,circular_farmland/circular_farmland_476.jpg,8
167 | 165,circular_farmland/circular_farmland_253.jpg,8
168 | 166,circular_farmland/circular_farmland_442.jpg,8
169 | 167,circular_farmland/circular_farmland_025.jpg,8
170 | 168,circular_farmland/circular_farmland_325.jpg,8
171 | 169,circular_farmland/circular_farmland_452.jpg,8
172 | 170,circular_farmland/circular_farmland_233.jpg,8
173 | 171,circular_farmland/circular_farmland_445.jpg,8
174 | 172,circular_farmland/circular_farmland_425.jpg,8
175 | 173,circular_farmland/circular_farmland_409.jpg,8
176 | 174,circular_farmland/circular_farmland_050.jpg,8
177 | 175,circular_farmland/circular_farmland_427.jpg,8
178 | 176,circular_farmland/circular_farmland_398.jpg,8
179 | 177,circular_farmland/circular_farmland_055.jpg,8
180 | 178,circular_farmland/circular_farmland_336.jpg,8
181 | 179,circular_farmland/circular_farmland_413.jpg,8
182 | 180,circular_farmland/circular_farmland_060.jpg,8
183 | 181,circular_farmland/circular_farmland_154.jpg,8
184 | 182,circular_farmland/circular_farmland_686.jpg,8
185 | 183,circular_farmland/circular_farmland_554.jpg,8
186 | 184,circular_farmland/circular_farmland_030.jpg,8
187 | 185,circular_farmland/circular_farmland_456.jpg,8
188 | 186,circular_farmland/circular_farmland_676.jpg,8
189 | 187,circular_farmland/circular_farmland_450.jpg,8
190 | 188,circular_farmland/circular_farmland_527.jpg,8
191 | 189,circular_farmland/circular_farmland_046.jpg,8
192 | 190,circular_farmland/circular_farmland_439.jpg,8
193 | 191,circular_farmland/circular_farmland_549.jpg,8
194 | 192,circular_farmland/circular_farmland_318.jpg,8
195 | 193,circular_farmland/circular_farmland_332.jpg,8
196 | 194,circular_farmland/circular_farmland_348.jpg,8
197 | 195,circular_farmland/circular_farmland_314.jpg,8
198 | 196,circular_farmland/circular_farmland_590.jpg,8
199 | 197,circular_farmland/circular_farmland_192.jpg,8
200 | 198,circular_farmland/circular_farmland_637.jpg,8
201 | 199,circular_farmland/circular_farmland_210.jpg,8
202 | 200,circular_farmland/circular_farmland_535.jpg,8
203 | 201,circular_farmland/circular_farmland_223.jpg,8
204 | 202,circular_farmland/circular_farmland_656.jpg,8
205 | 203,circular_farmland/circular_farmland_550.jpg,8
206 | 204,circular_farmland/circular_farmland_700.jpg,8
207 | 205,circular_farmland/circular_farmland_555.jpg,8
208 | 206,circular_farmland/circular_farmland_588.jpg,8
209 | 207,circular_farmland/circular_farmland_058.jpg,8
210 | 208,circular_farmland/circular_farmland_315.jpg,8
211 | 209,circular_farmland/circular_farmland_313.jpg,8
212 | 210,circular_farmland/circular_farmland_127.jpg,8
213 | 211,circular_farmland/circular_farmland_312.jpg,8
214 | 212,circular_farmland/circular_farmland_309.jpg,8
215 | 213,circular_farmland/circular_farmland_363.jpg,8
216 | 214,circular_farmland/circular_farmland_294.jpg,8
217 | 215,circular_farmland/circular_farmland_392.jpg,8
218 | 216,circular_farmland/circular_farmland_298.jpg,8
219 | 217,circular_farmland/circular_farmland_040.jpg,8
220 | 218,circular_farmland/circular_farmland_401.jpg,8
221 | 219,circular_farmland/circular_farmland_087.jpg,8
222 | 220,circular_farmland/circular_farmland_665.jpg,8
223 | 221,circular_farmland/circular_farmland_678.jpg,8
224 | 222,circular_farmland/circular_farmland_182.jpg,8
225 | 223,circular_farmland/circular_farmland_011.jpg,8
226 | 224,circular_farmland/circular_farmland_115.jpg,8
227 | 225,circular_farmland/circular_farmland_086.jpg,8
228 | 226,circular_farmland/circular_farmland_151.jpg,8
229 | 227,circular_farmland/circular_farmland_180.jpg,8
230 | 228,circular_farmland/circular_farmland_272.jpg,8
231 | 229,circular_farmland/circular_farmland_239.jpg,8
232 | 230,circular_farmland/circular_farmland_317.jpg,8
233 | 231,circular_farmland/circular_farmland_692.jpg,8
234 | 232,circular_farmland/circular_farmland_061.jpg,8
235 | 233,circular_farmland/circular_farmland_465.jpg,8
236 | 234,circular_farmland/circular_farmland_557.jpg,8
237 | 235,circular_farmland/circular_farmland_532.jpg,8
238 | 236,circular_farmland/circular_farmland_480.jpg,8
239 | 237,circular_farmland/circular_farmland_141.jpg,8
240 | 238,circular_farmland/circular_farmland_518.jpg,8
241 | 239,circular_farmland/circular_farmland_247.jpg,8
242 | 240,circular_farmland/circular_farmland_147.jpg,8
243 | 241,circular_farmland/circular_farmland_289.jpg,8
244 | 242,circular_farmland/circular_farmland_400.jpg,8
245 | 243,circular_farmland/circular_farmland_435.jpg,8
246 | 244,circular_farmland/circular_farmland_625.jpg,8
247 | 245,circular_farmland/circular_farmland_613.jpg,8
248 | 246,circular_farmland/circular_farmland_107.jpg,8
249 | 247,circular_farmland/circular_farmland_402.jpg,8
250 | 248,circular_farmland/circular_farmland_614.jpg,8
251 | 249,circular_farmland/circular_farmland_015.jpg,8
252 | 250,circular_farmland/circular_farmland_013.jpg,8
253 | 251,circular_farmland/circular_farmland_026.jpg,8
254 | 252,circular_farmland/circular_farmland_319.jpg,8
255 | 253,circular_farmland/circular_farmland_136.jpg,8
256 | 254,circular_farmland/circular_farmland_306.jpg,8
257 | 255,circular_farmland/circular_farmland_674.jpg,8
258 | 256,circular_farmland/circular_farmland_054.jpg,8
259 | 257,circular_farmland/circular_farmland_388.jpg,8
260 | 258,circular_farmland/circular_farmland_466.jpg,8
261 | 259,circular_farmland/circular_farmland_215.jpg,8
262 | 260,circular_farmland/circular_farmland_479.jpg,8
263 | 261,circular_farmland/circular_farmland_278.jpg,8
264 | 262,circular_farmland/circular_farmland_634.jpg,8
265 | 263,circular_farmland/circular_farmland_301.jpg,8
266 | 264,circular_farmland/circular_farmland_263.jpg,8
267 | 265,circular_farmland/circular_farmland_193.jpg,8
268 | 266,circular_farmland/circular_farmland_113.jpg,8
269 | 267,circular_farmland/circular_farmland_305.jpg,8
270 | 268,circular_farmland/circular_farmland_016.jpg,8
271 | 269,circular_farmland/circular_farmland_668.jpg,8
272 | 270,circular_farmland/circular_farmland_203.jpg,8
273 | 271,circular_farmland/circular_farmland_620.jpg,8
274 | 272,circular_farmland/circular_farmland_545.jpg,8
275 | 273,circular_farmland/circular_farmland_099.jpg,8
276 | 274,circular_farmland/circular_farmland_553.jpg,8
277 | 275,circular_farmland/circular_farmland_339.jpg,8
278 | 276,circular_farmland/circular_farmland_330.jpg,8
279 | 277,circular_farmland/circular_farmland_596.jpg,8
280 | 278,circular_farmland/circular_farmland_226.jpg,8
281 | 279,circular_farmland/circular_farmland_082.jpg,8
282 | 280,circular_farmland/circular_farmland_288.jpg,8
283 | 281,circular_farmland/circular_farmland_077.jpg,8
284 | 282,circular_farmland/circular_farmland_347.jpg,8
285 | 283,circular_farmland/circular_farmland_331.jpg,8
286 | 284,circular_farmland/circular_farmland_316.jpg,8
287 | 285,circular_farmland/circular_farmland_005.jpg,8
288 | 286,circular_farmland/circular_farmland_004.jpg,8
289 | 287,circular_farmland/circular_farmland_212.jpg,8
290 | 288,circular_farmland/circular_farmland_497.jpg,8
291 | 289,circular_farmland/circular_farmland_270.jpg,8
292 | 290,circular_farmland/circular_farmland_017.jpg,8
293 | 291,circular_farmland/circular_farmland_589.jpg,8
294 | 292,circular_farmland/circular_farmland_062.jpg,8
295 | 293,circular_farmland/circular_farmland_003.jpg,8
296 | 294,circular_farmland/circular_farmland_285.jpg,8
297 | 295,circular_farmland/circular_farmland_197.jpg,8
298 | 296,circular_farmland/circular_farmland_091.jpg,8
299 | 297,circular_farmland/circular_farmland_459.jpg,8
300 | 298,circular_farmland/circular_farmland_029.jpg,8
301 | 299,circular_farmland/circular_farmland_365.jpg,8
302 | 300,circular_farmland/circular_farmland_513.jpg,8
303 | 301,circular_farmland/circular_farmland_635.jpg,8
304 | 302,circular_farmland/circular_farmland_257.jpg,8
305 | 303,circular_farmland/circular_farmland_164.jpg,8
306 | 304,circular_farmland/circular_farmland_558.jpg,8
307 | 305,circular_farmland/circular_farmland_039.jpg,8
308 | 306,circular_farmland/circular_farmland_027.jpg,8
309 | 307,circular_farmland/circular_farmland_684.jpg,8
310 | 308,circular_farmland/circular_farmland_566.jpg,8
311 | 309,circular_farmland/circular_farmland_436.jpg,8
312 | 310,circular_farmland/circular_farmland_386.jpg,8
313 | 311,circular_farmland/circular_farmland_042.jpg,8
314 | 312,circular_farmland/circular_farmland_569.jpg,8
315 | 313,circular_farmland/circular_farmland_515.jpg,8
316 | 314,circular_farmland/circular_farmland_502.jpg,8
317 | 315,circular_farmland/circular_farmland_185.jpg,8
318 | 316,circular_farmland/circular_farmland_236.jpg,8
319 | 317,circular_farmland/circular_farmland_146.jpg,8
320 | 318,circular_farmland/circular_farmland_251.jpg,8
321 | 319,circular_farmland/circular_farmland_478.jpg,8
322 | 320,roundabout/roundabout_567.jpg,33
323 | 321,roundabout/roundabout_262.jpg,33
324 | 322,roundabout/roundabout_470.jpg,33
325 | 323,roundabout/roundabout_184.jpg,33
326 | 324,roundabout/roundabout_109.jpg,33
327 | 325,roundabout/roundabout_380.jpg,33
328 | 326,roundabout/roundabout_346.jpg,33
329 | 327,roundabout/roundabout_500.jpg,33
330 | 328,roundabout/roundabout_595.jpg,33
331 | 329,roundabout/roundabout_440.jpg,33
332 | 330,roundabout/roundabout_667.jpg,33
333 | 331,roundabout/roundabout_613.jpg,33
334 | 332,roundabout/roundabout_365.jpg,33
335 | 333,roundabout/roundabout_513.jpg,33
336 | 334,roundabout/roundabout_258.jpg,33
337 | 335,roundabout/roundabout_195.jpg,33
338 | 336,roundabout/roundabout_228.jpg,33
339 | 337,roundabout/roundabout_068.jpg,33
340 | 338,roundabout/roundabout_067.jpg,33
341 | 339,roundabout/roundabout_491.jpg,33
342 | 340,roundabout/roundabout_700.jpg,33
343 | 341,roundabout/roundabout_003.jpg,33
344 | 342,roundabout/roundabout_658.jpg,33
345 | 343,roundabout/roundabout_319.jpg,33
346 | 344,roundabout/roundabout_177.jpg,33
347 | 345,roundabout/roundabout_395.jpg,33
348 | 346,roundabout/roundabout_461.jpg,33
349 | 347,roundabout/roundabout_214.jpg,33
350 | 348,roundabout/roundabout_695.jpg,33
351 | 349,roundabout/roundabout_481.jpg,33
352 | 350,roundabout/roundabout_292.jpg,33
353 | 351,roundabout/roundabout_654.jpg,33
354 | 352,roundabout/roundabout_172.jpg,33
355 | 353,roundabout/roundabout_505.jpg,33
356 | 354,roundabout/roundabout_035.jpg,33
357 | 355,roundabout/roundabout_116.jpg,33
358 | 356,roundabout/roundabout_075.jpg,33
359 | 357,roundabout/roundabout_249.jpg,33
360 | 358,roundabout/roundabout_485.jpg,33
361 | 359,roundabout/roundabout_170.jpg,33
362 | 360,roundabout/roundabout_272.jpg,33
363 | 361,roundabout/roundabout_171.jpg,33
364 | 362,roundabout/roundabout_300.jpg,33
365 | 363,roundabout/roundabout_681.jpg,33
366 | 364,roundabout/roundabout_307.jpg,33
367 | 365,roundabout/roundabout_314.jpg,33
368 | 366,roundabout/roundabout_518.jpg,33
369 | 367,roundabout/roundabout_091.jpg,33
370 | 368,roundabout/roundabout_048.jpg,33
371 | 369,roundabout/roundabout_515.jpg,33
372 | 370,roundabout/roundabout_350.jpg,33
373 | 371,roundabout/roundabout_248.jpg,33
374 | 372,roundabout/roundabout_148.jpg,33
375 | 373,roundabout/roundabout_686.jpg,33
376 | 374,roundabout/roundabout_618.jpg,33
377 | 375,roundabout/roundabout_102.jpg,33
378 | 376,roundabout/roundabout_176.jpg,33
379 | 377,roundabout/roundabout_151.jpg,33
380 | 378,roundabout/roundabout_587.jpg,33
381 | 379,roundabout/roundabout_284.jpg,33
382 | 380,roundabout/roundabout_040.jpg,33
383 | 381,roundabout/roundabout_326.jpg,33
384 | 382,roundabout/roundabout_058.jpg,33
385 | 383,roundabout/roundabout_436.jpg,33
386 | 384,roundabout/roundabout_628.jpg,33
387 | 385,roundabout/roundabout_054.jpg,33
388 | 386,roundabout/roundabout_306.jpg,33
389 | 387,roundabout/roundabout_474.jpg,33
390 | 388,roundabout/roundabout_583.jpg,33
391 | 389,roundabout/roundabout_359.jpg,33
392 | 390,roundabout/roundabout_426.jpg,33
393 | 391,roundabout/roundabout_098.jpg,33
394 | 392,roundabout/roundabout_673.jpg,33
395 | 393,roundabout/roundabout_493.jpg,33
396 | 394,roundabout/roundabout_342.jpg,33
397 | 395,roundabout/roundabout_471.jpg,33
398 | 396,roundabout/roundabout_229.jpg,33
399 | 397,roundabout/roundabout_494.jpg,33
400 | 398,roundabout/roundabout_114.jpg,33
401 | 399,roundabout/roundabout_316.jpg,33
402 | 400,roundabout/roundabout_323.jpg,33
403 | 401,roundabout/roundabout_113.jpg,33
404 | 402,roundabout/roundabout_046.jpg,33
405 | 403,roundabout/roundabout_012.jpg,33
406 | 404,roundabout/roundabout_019.jpg,33
407 | 405,roundabout/roundabout_633.jpg,33
408 | 406,roundabout/roundabout_430.jpg,33
409 | 407,roundabout/roundabout_588.jpg,33
410 | 408,roundabout/roundabout_028.jpg,33
411 | 409,roundabout/roundabout_403.jpg,33
412 | 410,roundabout/roundabout_358.jpg,33
413 | 411,roundabout/roundabout_274.jpg,33
414 | 412,roundabout/roundabout_636.jpg,33
415 | 413,roundabout/roundabout_647.jpg,33
416 | 414,roundabout/roundabout_390.jpg,33
417 | 415,roundabout/roundabout_542.jpg,33
418 | 416,roundabout/roundabout_662.jpg,33
419 | 417,roundabout/roundabout_002.jpg,33
420 | 418,roundabout/roundabout_448.jpg,33
421 | 419,roundabout/roundabout_110.jpg,33
422 | 420,roundabout/roundabout_439.jpg,33
423 | 421,roundabout/roundabout_376.jpg,33
424 | 422,roundabout/roundabout_492.jpg,33
425 | 423,roundabout/roundabout_449.jpg,33
426 | 424,roundabout/roundabout_224.jpg,33
427 | 425,roundabout/roundabout_313.jpg,33
428 | 426,roundabout/roundabout_347.jpg,33
429 | 427,roundabout/roundabout_418.jpg,33
430 | 428,roundabout/roundabout_366.jpg,33
431 | 429,roundabout/roundabout_286.jpg,33
432 | 430,roundabout/roundabout_526.jpg,33
433 | 431,roundabout/roundabout_311.jpg,33
434 | 432,roundabout/roundabout_400.jpg,33
435 | 433,roundabout/roundabout_134.jpg,33
436 | 434,roundabout/roundabout_437.jpg,33
437 | 435,roundabout/roundabout_225.jpg,33
438 | 436,roundabout/roundabout_101.jpg,33
439 | 437,roundabout/roundabout_575.jpg,33
440 | 438,roundabout/roundabout_357.jpg,33
441 | 439,roundabout/roundabout_623.jpg,33
442 | 440,roundabout/roundabout_153.jpg,33
443 | 441,roundabout/roundabout_278.jpg,33
444 | 442,roundabout/roundabout_483.jpg,33
445 | 443,roundabout/roundabout_378.jpg,33
446 | 444,roundabout/roundabout_083.jpg,33
447 | 445,roundabout/roundabout_150.jpg,33
448 | 446,roundabout/roundabout_608.jpg,33
449 | 447,roundabout/roundabout_533.jpg,33
450 | 448,roundabout/roundabout_453.jpg,33
451 | 449,roundabout/roundabout_305.jpg,33
452 | 450,roundabout/roundabout_690.jpg,33
453 | 451,roundabout/roundabout_556.jpg,33
454 | 452,roundabout/roundabout_656.jpg,33
455 | 453,roundabout/roundabout_043.jpg,33
456 | 454,roundabout/roundabout_269.jpg,33
457 | 455,roundabout/roundabout_001.jpg,33
458 | 456,roundabout/roundabout_352.jpg,33
459 | 457,roundabout/roundabout_648.jpg,33
460 | 458,roundabout/roundabout_585.jpg,33
461 | 459,roundabout/roundabout_603.jpg,33
462 | 460,roundabout/roundabout_598.jpg,33
463 | 461,roundabout/roundabout_431.jpg,33
464 | 462,roundabout/roundabout_504.jpg,33
465 | 463,roundabout/roundabout_678.jpg,33
466 | 464,roundabout/roundabout_118.jpg,33
467 | 465,roundabout/roundabout_661.jpg,33
468 | 466,roundabout/roundabout_360.jpg,33
469 | 467,roundabout/roundabout_239.jpg,33
470 | 468,roundabout/roundabout_443.jpg,33
471 | 469,roundabout/roundabout_377.jpg,33
472 | 470,roundabout/roundabout_364.jpg,33
473 | 471,roundabout/roundabout_362.jpg,33
474 | 472,roundabout/roundabout_631.jpg,33
475 | 473,roundabout/roundabout_121.jpg,33
476 | 474,roundabout/roundabout_103.jpg,33
477 | 475,roundabout/roundabout_510.jpg,33
478 | 476,roundabout/roundabout_605.jpg,33
479 | 477,roundabout/roundabout_419.jpg,33
480 | 478,roundabout/roundabout_338.jpg,33
481 | 479,roundabout/roundabout_512.jpg,33
482 |
--------------------------------------------------------------------------------
/datafiles/xval-2.test.csv:
--------------------------------------------------------------------------------
1 | ,image,class_label
2 | 0,basketball_court/basketball_court_462.jpg,3
3 | 1,basketball_court/basketball_court_154.jpg,3
4 | 2,basketball_court/basketball_court_361.jpg,3
5 | 3,basketball_court/basketball_court_627.jpg,3
6 | 4,basketball_court/basketball_court_524.jpg,3
7 | 5,basketball_court/basketball_court_473.jpg,3
8 | 6,basketball_court/basketball_court_003.jpg,3
9 | 7,basketball_court/basketball_court_499.jpg,3
10 | 8,basketball_court/basketball_court_150.jpg,3
11 | 9,basketball_court/basketball_court_001.jpg,3
12 | 10,basketball_court/basketball_court_075.jpg,3
13 | 11,basketball_court/basketball_court_585.jpg,3
14 | 12,basketball_court/basketball_court_054.jpg,3
15 | 13,basketball_court/basketball_court_502.jpg,3
16 | 14,basketball_court/basketball_court_279.jpg,3
17 | 15,basketball_court/basketball_court_358.jpg,3
18 | 16,basketball_court/basketball_court_072.jpg,3
19 | 17,basketball_court/basketball_court_302.jpg,3
20 | 18,basketball_court/basketball_court_313.jpg,3
21 | 19,basketball_court/basketball_court_276.jpg,3
22 | 20,basketball_court/basketball_court_068.jpg,3
23 | 21,basketball_court/basketball_court_284.jpg,3
24 | 22,basketball_court/basketball_court_512.jpg,3
25 | 23,basketball_court/basketball_court_592.jpg,3
26 | 24,basketball_court/basketball_court_352.jpg,3
27 | 25,basketball_court/basketball_court_262.jpg,3
28 | 26,basketball_court/basketball_court_079.jpg,3
29 | 27,basketball_court/basketball_court_306.jpg,3
30 | 28,basketball_court/basketball_court_310.jpg,3
31 | 29,basketball_court/basketball_court_656.jpg,3
32 | 30,basketball_court/basketball_court_196.jpg,3
33 | 31,basketball_court/basketball_court_158.jpg,3
34 | 32,basketball_court/basketball_court_185.jpg,3
35 | 33,basketball_court/basketball_court_236.jpg,3
36 | 34,basketball_court/basketball_court_261.jpg,3
37 | 35,basketball_court/basketball_court_602.jpg,3
38 | 36,basketball_court/basketball_court_441.jpg,3
39 | 37,basketball_court/basketball_court_109.jpg,3
40 | 38,basketball_court/basketball_court_643.jpg,3
41 | 39,basketball_court/basketball_court_691.jpg,3
42 | 40,circular_farmland/circular_farmland_089.jpg,8
43 | 41,circular_farmland/circular_farmland_201.jpg,8
44 | 42,circular_farmland/circular_farmland_256.jpg,8
45 | 43,circular_farmland/circular_farmland_475.jpg,8
46 | 44,circular_farmland/circular_farmland_420.jpg,8
47 | 45,circular_farmland/circular_farmland_553.jpg,8
48 | 46,circular_farmland/circular_farmland_052.jpg,8
49 | 47,circular_farmland/circular_farmland_560.jpg,8
50 | 48,circular_farmland/circular_farmland_680.jpg,8
51 | 49,circular_farmland/circular_farmland_320.jpg,8
52 | 50,circular_farmland/circular_farmland_228.jpg,8
53 | 51,circular_farmland/circular_farmland_053.jpg,8
54 | 52,circular_farmland/circular_farmland_468.jpg,8
55 | 53,circular_farmland/circular_farmland_163.jpg,8
56 | 54,circular_farmland/circular_farmland_054.jpg,8
57 | 55,circular_farmland/circular_farmland_236.jpg,8
58 | 56,circular_farmland/circular_farmland_373.jpg,8
59 | 57,circular_farmland/circular_farmland_579.jpg,8
60 | 58,circular_farmland/circular_farmland_022.jpg,8
61 | 59,circular_farmland/circular_farmland_137.jpg,8
62 | 60,circular_farmland/circular_farmland_508.jpg,8
63 | 61,circular_farmland/circular_farmland_114.jpg,8
64 | 62,circular_farmland/circular_farmland_277.jpg,8
65 | 63,circular_farmland/circular_farmland_101.jpg,8
66 | 64,circular_farmland/circular_farmland_396.jpg,8
67 | 65,circular_farmland/circular_farmland_642.jpg,8
68 | 66,circular_farmland/circular_farmland_563.jpg,8
69 | 67,circular_farmland/circular_farmland_242.jpg,8
70 | 68,circular_farmland/circular_farmland_283.jpg,8
71 | 69,circular_farmland/circular_farmland_457.jpg,8
72 | 70,circular_farmland/circular_farmland_565.jpg,8
73 | 71,circular_farmland/circular_farmland_618.jpg,8
74 | 72,circular_farmland/circular_farmland_481.jpg,8
75 | 73,circular_farmland/circular_farmland_564.jpg,8
76 | 74,circular_farmland/circular_farmland_523.jpg,8
77 | 75,circular_farmland/circular_farmland_604.jpg,8
78 | 76,circular_farmland/circular_farmland_424.jpg,8
79 | 77,circular_farmland/circular_farmland_078.jpg,8
80 | 78,circular_farmland/circular_farmland_505.jpg,8
81 | 79,circular_farmland/circular_farmland_422.jpg,8
82 | 80,roundabout/roundabout_625.jpg,33
83 | 81,roundabout/roundabout_434.jpg,33
84 | 82,roundabout/roundabout_644.jpg,33
85 | 83,roundabout/roundabout_540.jpg,33
86 | 84,roundabout/roundabout_522.jpg,33
87 | 85,roundabout/roundabout_100.jpg,33
88 | 86,roundabout/roundabout_340.jpg,33
89 | 87,roundabout/roundabout_325.jpg,33
90 | 88,roundabout/roundabout_141.jpg,33
91 | 89,roundabout/roundabout_648.jpg,33
92 | 90,roundabout/roundabout_376.jpg,33
93 | 91,roundabout/roundabout_610.jpg,33
94 | 92,roundabout/roundabout_204.jpg,33
95 | 93,roundabout/roundabout_177.jpg,33
96 | 94,roundabout/roundabout_556.jpg,33
97 | 95,roundabout/roundabout_566.jpg,33
98 | 96,roundabout/roundabout_500.jpg,33
99 | 97,roundabout/roundabout_454.jpg,33
100 | 98,roundabout/roundabout_508.jpg,33
101 | 99,roundabout/roundabout_402.jpg,33
102 | 100,roundabout/roundabout_224.jpg,33
103 | 101,roundabout/roundabout_366.jpg,33
104 | 102,roundabout/roundabout_334.jpg,33
105 | 103,roundabout/roundabout_643.jpg,33
106 | 104,roundabout/roundabout_324.jpg,33
107 | 105,roundabout/roundabout_302.jpg,33
108 | 106,roundabout/roundabout_256.jpg,33
109 | 107,roundabout/roundabout_187.jpg,33
110 | 108,roundabout/roundabout_685.jpg,33
111 | 109,roundabout/roundabout_101.jpg,33
112 | 110,roundabout/roundabout_621.jpg,33
113 | 111,roundabout/roundabout_385.jpg,33
114 | 112,roundabout/roundabout_453.jpg,33
115 | 113,roundabout/roundabout_192.jpg,33
116 | 114,roundabout/roundabout_065.jpg,33
117 | 115,roundabout/roundabout_107.jpg,33
118 | 116,roundabout/roundabout_362.jpg,33
119 | 117,roundabout/roundabout_081.jpg,33
120 | 118,roundabout/roundabout_067.jpg,33
121 | 119,roundabout/roundabout_242.jpg,33
122 |
--------------------------------------------------------------------------------
/datafiles/xval-3.test.csv:
--------------------------------------------------------------------------------
1 | ,image,class_label
2 | 0,basketball_court/basketball_court_604.jpg,3
3 | 1,basketball_court/basketball_court_264.jpg,3
4 | 2,basketball_court/basketball_court_174.jpg,3
5 | 3,basketball_court/basketball_court_634.jpg,3
6 | 4,basketball_court/basketball_court_016.jpg,3
7 | 5,basketball_court/basketball_court_181.jpg,3
8 | 6,basketball_court/basketball_court_141.jpg,3
9 | 7,basketball_court/basketball_court_539.jpg,3
10 | 8,basketball_court/basketball_court_176.jpg,3
11 | 9,basketball_court/basketball_court_400.jpg,3
12 | 10,basketball_court/basketball_court_545.jpg,3
13 | 11,basketball_court/basketball_court_140.jpg,3
14 | 12,basketball_court/basketball_court_602.jpg,3
15 | 13,basketball_court/basketball_court_510.jpg,3
16 | 14,basketball_court/basketball_court_474.jpg,3
17 | 15,basketball_court/basketball_court_321.jpg,3
18 | 16,basketball_court/basketball_court_672.jpg,3
19 | 17,basketball_court/basketball_court_577.jpg,3
20 | 18,basketball_court/basketball_court_161.jpg,3
21 | 19,basketball_court/basketball_court_569.jpg,3
22 | 20,basketball_court/basketball_court_439.jpg,3
23 | 21,basketball_court/basketball_court_613.jpg,3
24 | 22,basketball_court/basketball_court_101.jpg,3
25 | 23,basketball_court/basketball_court_531.jpg,3
26 | 24,basketball_court/basketball_court_296.jpg,3
27 | 25,basketball_court/basketball_court_502.jpg,3
28 | 26,basketball_court/basketball_court_504.jpg,3
29 | 27,basketball_court/basketball_court_445.jpg,3
30 | 28,basketball_court/basketball_court_465.jpg,3
31 | 29,basketball_court/basketball_court_528.jpg,3
32 | 30,basketball_court/basketball_court_303.jpg,3
33 | 31,basketball_court/basketball_court_291.jpg,3
34 | 32,basketball_court/basketball_court_288.jpg,3
35 | 33,basketball_court/basketball_court_021.jpg,3
36 | 34,basketball_court/basketball_court_265.jpg,3
37 | 35,basketball_court/basketball_court_171.jpg,3
38 | 36,basketball_court/basketball_court_432.jpg,3
39 | 37,basketball_court/basketball_court_475.jpg,3
40 | 38,basketball_court/basketball_court_620.jpg,3
41 | 39,basketball_court/basketball_court_066.jpg,3
42 | 40,circular_farmland/circular_farmland_534.jpg,8
43 | 41,circular_farmland/circular_farmland_378.jpg,8
44 | 42,circular_farmland/circular_farmland_154.jpg,8
45 | 43,circular_farmland/circular_farmland_474.jpg,8
46 | 44,circular_farmland/circular_farmland_122.jpg,8
47 | 45,circular_farmland/circular_farmland_502.jpg,8
48 | 46,circular_farmland/circular_farmland_220.jpg,8
49 | 47,circular_farmland/circular_farmland_085.jpg,8
50 | 48,circular_farmland/circular_farmland_613.jpg,8
51 | 49,circular_farmland/circular_farmland_443.jpg,8
52 | 50,circular_farmland/circular_farmland_471.jpg,8
53 | 51,circular_farmland/circular_farmland_644.jpg,8
54 | 52,circular_farmland/circular_farmland_280.jpg,8
55 | 53,circular_farmland/circular_farmland_573.jpg,8
56 | 54,circular_farmland/circular_farmland_423.jpg,8
57 | 55,circular_farmland/circular_farmland_276.jpg,8
58 | 56,circular_farmland/circular_farmland_344.jpg,8
59 | 57,circular_farmland/circular_farmland_279.jpg,8
60 | 58,circular_farmland/circular_farmland_171.jpg,8
61 | 59,circular_farmland/circular_farmland_246.jpg,8
62 | 60,circular_farmland/circular_farmland_296.jpg,8
63 | 61,circular_farmland/circular_farmland_682.jpg,8
64 | 62,circular_farmland/circular_farmland_248.jpg,8
65 | 63,circular_farmland/circular_farmland_537.jpg,8
66 | 64,circular_farmland/circular_farmland_072.jpg,8
67 | 65,circular_farmland/circular_farmland_267.jpg,8
68 | 66,circular_farmland/circular_farmland_659.jpg,8
69 | 67,circular_farmland/circular_farmland_666.jpg,8
70 | 68,circular_farmland/circular_farmland_478.jpg,8
71 | 69,circular_farmland/circular_farmland_449.jpg,8
72 | 70,circular_farmland/circular_farmland_587.jpg,8
73 | 71,circular_farmland/circular_farmland_025.jpg,8
74 | 72,circular_farmland/circular_farmland_088.jpg,8
75 | 73,circular_farmland/circular_farmland_572.jpg,8
76 | 74,circular_farmland/circular_farmland_469.jpg,8
77 | 75,circular_farmland/circular_farmland_651.jpg,8
78 | 76,circular_farmland/circular_farmland_589.jpg,8
79 | 77,circular_farmland/circular_farmland_137.jpg,8
80 | 78,circular_farmland/circular_farmland_245.jpg,8
81 | 79,circular_farmland/circular_farmland_603.jpg,8
82 | 80,roundabout/roundabout_568.jpg,33
83 | 81,roundabout/roundabout_474.jpg,33
84 | 82,roundabout/roundabout_197.jpg,33
85 | 83,roundabout/roundabout_642.jpg,33
86 | 84,roundabout/roundabout_674.jpg,33
87 | 85,roundabout/roundabout_130.jpg,33
88 | 86,roundabout/roundabout_232.jpg,33
89 | 87,roundabout/roundabout_265.jpg,33
90 | 88,roundabout/roundabout_410.jpg,33
91 | 89,roundabout/roundabout_026.jpg,33
92 | 90,roundabout/roundabout_518.jpg,33
93 | 91,roundabout/roundabout_647.jpg,33
94 | 92,roundabout/roundabout_404.jpg,33
95 | 93,roundabout/roundabout_031.jpg,33
96 | 94,roundabout/roundabout_532.jpg,33
97 | 95,roundabout/roundabout_390.jpg,33
98 | 96,roundabout/roundabout_035.jpg,33
99 | 97,roundabout/roundabout_476.jpg,33
100 | 98,roundabout/roundabout_235.jpg,33
101 | 99,roundabout/roundabout_138.jpg,33
102 | 100,roundabout/roundabout_524.jpg,33
103 | 101,roundabout/roundabout_037.jpg,33
104 | 102,roundabout/roundabout_496.jpg,33
105 | 103,roundabout/roundabout_515.jpg,33
106 | 104,roundabout/roundabout_220.jpg,33
107 | 105,roundabout/roundabout_619.jpg,33
108 | 106,roundabout/roundabout_034.jpg,33
109 | 107,roundabout/roundabout_690.jpg,33
110 | 108,roundabout/roundabout_402.jpg,33
111 | 109,roundabout/roundabout_629.jpg,33
112 | 110,roundabout/roundabout_337.jpg,33
113 | 111,roundabout/roundabout_561.jpg,33
114 | 112,roundabout/roundabout_167.jpg,33
115 | 113,roundabout/roundabout_505.jpg,33
116 | 114,roundabout/roundabout_119.jpg,33
117 | 115,roundabout/roundabout_438.jpg,33
118 | 116,roundabout/roundabout_139.jpg,33
119 | 117,roundabout/roundabout_595.jpg,33
120 | 118,roundabout/roundabout_292.jpg,33
121 | 119,roundabout/roundabout_368.jpg,33
122 |
--------------------------------------------------------------------------------
/datafiles/xval-4.test.csv:
--------------------------------------------------------------------------------
1 | ,image,class_label
2 | 0,basketball_court/basketball_court_407.jpg,3
3 | 1,basketball_court/basketball_court_187.jpg,3
4 | 2,basketball_court/basketball_court_482.jpg,3
5 | 3,basketball_court/basketball_court_020.jpg,3
6 | 4,basketball_court/basketball_court_246.jpg,3
7 | 5,basketball_court/basketball_court_550.jpg,3
8 | 6,basketball_court/basketball_court_128.jpg,3
9 | 7,basketball_court/basketball_court_261.jpg,3
10 | 8,basketball_court/basketball_court_538.jpg,3
11 | 9,basketball_court/basketball_court_240.jpg,3
12 | 10,basketball_court/basketball_court_116.jpg,3
13 | 11,basketball_court/basketball_court_382.jpg,3
14 | 12,basketball_court/basketball_court_512.jpg,3
15 | 13,basketball_court/basketball_court_674.jpg,3
16 | 14,basketball_court/basketball_court_604.jpg,3
17 | 15,basketball_court/basketball_court_359.jpg,3
18 | 16,basketball_court/basketball_court_209.jpg,3
19 | 17,basketball_court/basketball_court_343.jpg,3
20 | 18,basketball_court/basketball_court_662.jpg,3
21 | 19,basketball_court/basketball_court_047.jpg,3
22 | 20,basketball_court/basketball_court_182.jpg,3
23 | 21,basketball_court/basketball_court_124.jpg,3
24 | 22,basketball_court/basketball_court_348.jpg,3
25 | 23,basketball_court/basketball_court_577.jpg,3
26 | 24,basketball_court/basketball_court_152.jpg,3
27 | 25,basketball_court/basketball_court_157.jpg,3
28 | 26,basketball_court/basketball_court_522.jpg,3
29 | 27,basketball_court/basketball_court_254.jpg,3
30 | 28,basketball_court/basketball_court_509.jpg,3
31 | 29,basketball_court/basketball_court_673.jpg,3
32 | 30,basketball_court/basketball_court_629.jpg,3
33 | 31,basketball_court/basketball_court_003.jpg,3
34 | 32,basketball_court/basketball_court_038.jpg,3
35 | 33,basketball_court/basketball_court_384.jpg,3
36 | 34,basketball_court/basketball_court_468.jpg,3
37 | 35,basketball_court/basketball_court_324.jpg,3
38 | 36,basketball_court/basketball_court_007.jpg,3
39 | 37,basketball_court/basketball_court_026.jpg,3
40 | 38,basketball_court/basketball_court_005.jpg,3
41 | 39,basketball_court/basketball_court_328.jpg,3
42 | 40,circular_farmland/circular_farmland_459.jpg,8
43 | 41,circular_farmland/circular_farmland_136.jpg,8
44 | 42,circular_farmland/circular_farmland_598.jpg,8
45 | 43,circular_farmland/circular_farmland_109.jpg,8
46 | 44,circular_farmland/circular_farmland_663.jpg,8
47 | 45,circular_farmland/circular_farmland_059.jpg,8
48 | 46,circular_farmland/circular_farmland_085.jpg,8
49 | 47,circular_farmland/circular_farmland_697.jpg,8
50 | 48,circular_farmland/circular_farmland_211.jpg,8
51 | 49,circular_farmland/circular_farmland_350.jpg,8
52 | 50,circular_farmland/circular_farmland_573.jpg,8
53 | 51,circular_farmland/circular_farmland_441.jpg,8
54 | 52,circular_farmland/circular_farmland_107.jpg,8
55 | 53,circular_farmland/circular_farmland_212.jpg,8
56 | 54,circular_farmland/circular_farmland_175.jpg,8
57 | 55,circular_farmland/circular_farmland_104.jpg,8
58 | 56,circular_farmland/circular_farmland_421.jpg,8
59 | 57,circular_farmland/circular_farmland_551.jpg,8
60 | 58,circular_farmland/circular_farmland_458.jpg,8
61 | 59,circular_farmland/circular_farmland_681.jpg,8
62 | 60,circular_farmland/circular_farmland_315.jpg,8
63 | 61,circular_farmland/circular_farmland_294.jpg,8
64 | 62,circular_farmland/circular_farmland_388.jpg,8
65 | 63,circular_farmland/circular_farmland_335.jpg,8
66 | 64,circular_farmland/circular_farmland_018.jpg,8
67 | 65,circular_farmland/circular_farmland_419.jpg,8
68 | 66,circular_farmland/circular_farmland_411.jpg,8
69 | 67,circular_farmland/circular_farmland_420.jpg,8
70 | 68,circular_farmland/circular_farmland_202.jpg,8
71 | 69,circular_farmland/circular_farmland_231.jpg,8
72 | 70,circular_farmland/circular_farmland_262.jpg,8
73 | 71,circular_farmland/circular_farmland_106.jpg,8
74 | 72,circular_farmland/circular_farmland_582.jpg,8
75 | 73,circular_farmland/circular_farmland_368.jpg,8
76 | 74,circular_farmland/circular_farmland_244.jpg,8
77 | 75,circular_farmland/circular_farmland_536.jpg,8
78 | 76,circular_farmland/circular_farmland_694.jpg,8
79 | 77,circular_farmland/circular_farmland_017.jpg,8
80 | 78,circular_farmland/circular_farmland_195.jpg,8
81 | 79,circular_farmland/circular_farmland_684.jpg,8
82 | 80,roundabout/roundabout_254.jpg,33
83 | 81,roundabout/roundabout_567.jpg,33
84 | 82,roundabout/roundabout_159.jpg,33
85 | 83,roundabout/roundabout_664.jpg,33
86 | 84,roundabout/roundabout_624.jpg,33
87 | 85,roundabout/roundabout_545.jpg,33
88 | 86,roundabout/roundabout_506.jpg,33
89 | 87,roundabout/roundabout_287.jpg,33
90 | 88,roundabout/roundabout_019.jpg,33
91 | 89,roundabout/roundabout_270.jpg,33
92 | 90,roundabout/roundabout_008.jpg,33
93 | 91,roundabout/roundabout_634.jpg,33
94 | 92,roundabout/roundabout_124.jpg,33
95 | 93,roundabout/roundabout_284.jpg,33
96 | 94,roundabout/roundabout_426.jpg,33
97 | 95,roundabout/roundabout_698.jpg,33
98 | 96,roundabout/roundabout_625.jpg,33
99 | 97,roundabout/roundabout_184.jpg,33
100 | 98,roundabout/roundabout_488.jpg,33
101 | 99,roundabout/roundabout_037.jpg,33
102 | 100,roundabout/roundabout_099.jpg,33
103 | 101,roundabout/roundabout_693.jpg,33
104 | 102,roundabout/roundabout_301.jpg,33
105 | 103,roundabout/roundabout_680.jpg,33
106 | 104,roundabout/roundabout_619.jpg,33
107 | 105,roundabout/roundabout_191.jpg,33
108 | 106,roundabout/roundabout_575.jpg,33
109 | 107,roundabout/roundabout_208.jpg,33
110 | 108,roundabout/roundabout_452.jpg,33
111 | 109,roundabout/roundabout_503.jpg,33
112 | 110,roundabout/roundabout_596.jpg,33
113 | 111,roundabout/roundabout_403.jpg,33
114 | 112,roundabout/roundabout_087.jpg,33
115 | 113,roundabout/roundabout_003.jpg,33
116 | 114,roundabout/roundabout_541.jpg,33
117 | 115,roundabout/roundabout_550.jpg,33
118 | 116,roundabout/roundabout_459.jpg,33
119 | 117,roundabout/roundabout_402.jpg,33
120 | 118,roundabout/roundabout_328.jpg,33
121 | 119,roundabout/roundabout_243.jpg,33
122 |
--------------------------------------------------------------------------------
/datafiles/xval_cls.densenet.5fold.csv:
--------------------------------------------------------------------------------
1 | fold_name,class_label,accuracy
2 | A,0,0.993007
3 | A,1,0.886667
4 | A,2,0.960784
5 | A,3,0.934783
6 | A,4,0.947368
7 | A,5,0.919355
8 | A,6,0.992424
9 | A,7,0.737589
10 | A,8,0.953020
11 | A,9,0.986577
12 | A,10,0.838462
13 | A,11,0.904412
14 | A,12,0.915584
15 | A,13,0.948148
16 | A,14,0.868421
17 | A,15,0.962963
18 | A,16,0.935252
19 | A,17,0.993421
20 | A,18,0.864286
21 | A,19,0.946667
22 | A,20,0.973684
23 | A,21,0.891892
24 | A,22,0.949275
25 | A,23,0.868056
26 | A,24,0.971631
27 | A,25,0.932886
28 | A,26,0.967949
29 | A,27,0.898551
30 | A,28,0.985507
31 | A,29,0.812081
32 | A,30,0.872180
33 | A,31,0.848276
34 | A,32,0.883562
35 | A,33,0.933824
36 | A,34,0.952381
37 | A,35,0.979167
38 | A,36,0.959350
39 | A,37,0.966667
40 | A,38,0.826087
41 | A,39,0.913386
42 | A,40,0.944828
43 | A,41,0.963768
44 | A,42,0.985507
45 | A,43,0.865672
46 | A,44,0.927536
47 | B,0,1.000000
48 | B,1,0.938356
49 | B,2,0.930233
50 | B,3,0.948905
51 | B,4,0.914894
52 | B,5,0.934211
53 | B,6,0.993197
54 | B,7,0.869281
55 | B,8,0.984733
56 | B,9,0.917355
57 | B,10,0.891892
58 | B,11,0.844444
59 | B,12,0.953846
60 | B,13,0.962406
61 | B,14,0.893939
62 | B,15,0.983871
63 | B,16,0.967480
64 | B,17,0.974359
65 | B,18,0.914894
66 | B,19,0.929134
67 | B,20,0.916667
68 | B,21,0.965278
69 | B,22,0.960784
70 | B,23,0.933824
71 | B,24,0.926667
72 | B,25,0.979592
73 | B,26,0.964286
74 | B,27,0.767123
75 | B,28,0.993197
76 | B,29,0.779412
77 | B,30,0.879699
78 | B,31,0.953947
79 | B,32,0.968254
80 | B,33,0.977099
81 | B,34,0.843284
82 | B,35,0.972603
83 | B,36,0.851852
84 | B,37,0.984962
85 | B,38,0.933884
86 | B,39,0.918699
87 | B,40,0.978417
88 | B,41,0.955224
89 | B,42,0.848485
90 | B,43,0.891156
91 | B,44,0.711111
92 | C,0,0.978417
93 | C,1,0.854015
94 | C,2,1.000000
95 | C,3,0.936620
96 | C,4,0.977612
97 | C,5,0.902597
98 | C,6,0.993056
99 | C,7,0.728571
100 | C,8,0.947761
101 | C,9,0.968992
102 | C,10,0.872180
103 | C,11,0.931298
104 | C,12,0.980892
105 | C,13,0.898649
106 | C,14,0.914286
107 | C,15,1.000000
108 | C,16,0.934783
109 | C,17,0.980000
110 | C,18,0.880795
111 | C,19,0.979021
112 | C,20,0.954887
113 | C,21,0.950000
114 | C,22,0.972028
115 | C,23,0.874074
116 | C,24,0.936170
117 | C,25,0.832000
118 | C,26,0.905797
119 | C,27,0.781250
120 | C,28,0.970149
121 | C,29,0.935484
122 | C,30,0.858268
123 | C,31,0.900709
124 | C,32,0.923077
125 | C,33,0.979452
126 | C,34,0.979592
127 | C,35,0.992908
128 | C,36,0.980645
129 | C,37,0.952703
130 | C,38,0.924242
131 | C,39,0.958042
132 | C,40,0.912752
133 | C,41,0.969466
134 | C,42,0.977778
135 | C,43,0.928571
136 | C,44,0.897959
137 | D,0,0.882759
138 | D,1,0.877551
139 | D,2,0.854167
140 | D,3,0.955128
141 | D,4,0.976378
142 | D,5,0.986301
143 | D,6,1.000000
144 | D,7,0.930556
145 | D,8,0.901961
146 | D,9,0.973333
147 | D,10,0.798658
148 | D,11,0.974194
149 | D,12,0.901639
150 | D,13,0.935252
151 | D,14,0.901961
152 | D,15,1.000000
153 | D,16,0.988166
154 | D,17,0.927007
155 | D,18,0.830645
156 | D,19,0.956835
157 | D,20,0.909091
158 | D,21,0.896000
159 | D,22,0.954545
160 | D,23,0.891156
161 | D,24,0.889831
162 | D,25,0.925170
163 | D,26,0.931298
164 | D,27,0.612245
165 | D,28,0.921053
166 | D,29,0.965753
167 | D,30,0.870370
168 | D,31,0.916031
169 | D,32,0.942029
170 | D,33,0.858108
171 | D,34,0.914062
172 | D,35,0.979310
173 | D,36,0.875912
174 | D,37,1.000000
175 | D,38,0.870968
176 | D,39,0.901235
177 | D,40,0.924242
178 | D,41,0.941606
179 | D,42,0.877698
180 | D,43,0.855072
181 | D,44,0.875000
182 | E,0,0.981013
183 | E,1,0.866667
184 | E,2,0.946154
185 | E,3,0.984252
186 | E,4,0.938356
187 | E,5,0.943548
188 | E,6,0.972222
189 | E,7,0.877049
190 | E,8,0.984962
191 | E,9,0.960265
192 | E,10,0.885714
193 | E,11,0.699301
194 | E,12,0.948905
195 | E,13,0.937931
196 | E,14,0.894309
197 | E,15,0.944785
198 | E,16,0.961832
199 | E,17,0.979167
200 | E,18,0.902778
201 | E,19,0.794326
202 | E,20,0.992248
203 | E,21,0.867133
204 | E,22,0.955224
205 | E,23,0.927536
206 | E,24,0.973333
207 | E,25,0.946970
208 | E,26,0.925926
209 | E,27,0.631206
210 | E,28,0.945736
211 | E,29,0.965517
212 | E,30,0.648276
213 | E,31,0.854962
214 | E,32,0.925373
215 | E,33,0.892086
216 | E,34,0.930556
217 | E,35,1.000000
218 | E,36,0.966667
219 | E,37,0.977612
220 | E,38,0.919753
221 | E,39,0.937931
222 | E,40,0.962963
223 | E,41,0.987500
224 | E,42,0.902439
225 | E,43,0.937008
226 | E,44,0.867647
227 |
--------------------------------------------------------------------------------
/datafiles/xval_cls.inception.5fold.csv:
--------------------------------------------------------------------------------
1 | fold_name,class_label,accuracy
2 | A,0,0.902098
3 | A,1,0.613333
4 | A,2,0.810458
5 | A,3,0.695652
6 | A,4,0.973684
7 | A,5,0.903226
8 | A,6,0.886364
9 | A,7,0.070922
10 | A,8,0.718121
11 | A,9,0.852349
12 | A,10,0.015385
13 | A,11,0.235294
14 | A,12,0.811688
15 | A,13,0.703704
16 | A,14,0.644737
17 | A,15,0.451852
18 | A,16,0.762590
19 | A,17,0.828947
20 | A,18,0.692857
21 | A,19,0.640000
22 | A,20,0.921053
23 | A,21,0.858108
24 | A,22,0.246377
25 | A,23,0.347222
26 | A,24,0.326241
27 | A,25,0.221477
28 | A,26,0.192308
29 | A,27,0.326087
30 | A,28,0.942029
31 | A,29,0.865772
32 | A,30,0.345865
33 | A,31,0.806897
34 | A,32,0.623288
35 | A,33,0.632353
36 | A,34,0.619048
37 | A,35,0.291667
38 | A,36,0.715447
39 | A,37,0.966667
40 | A,38,0.503106
41 | A,39,0.779528
42 | A,40,0.620690
43 | A,41,0.681159
44 | A,42,0.413043
45 | A,43,0.925373
46 | A,44,0.260870
47 | B,0,0.652174
48 | B,1,0.349315
49 | B,2,0.465116
50 | B,3,0.182482
51 | B,4,0.822695
52 | B,5,0.526316
53 | B,6,0.891156
54 | B,7,0.424837
55 | B,8,0.557252
56 | B,9,0.487603
57 | B,10,0.000000
58 | B,11,0.051852
59 | B,12,0.653846
60 | B,13,0.909774
61 | B,14,0.893939
62 | B,15,0.395161
63 | B,16,0.780488
64 | B,17,0.504274
65 | B,18,0.234043
66 | B,19,0.330709
67 | B,20,0.696970
68 | B,21,0.333333
69 | B,22,0.065359
70 | B,23,0.088235
71 | B,24,0.293333
72 | B,25,0.088435
73 | B,26,0.464286
74 | B,27,0.191781
75 | B,28,0.972789
76 | B,29,0.772059
77 | B,30,0.345865
78 | B,31,0.848684
79 | B,32,0.111111
80 | B,33,0.564885
81 | B,34,0.753731
82 | B,35,0.321918
83 | B,36,0.814815
84 | B,37,0.977444
85 | B,38,0.247934
86 | B,39,0.682927
87 | B,40,0.575540
88 | B,41,0.820896
89 | B,42,0.448485
90 | B,43,0.904762
91 | B,44,0.037037
92 | C,0,0.935252
93 | C,1,0.686131
94 | C,2,0.729167
95 | C,3,0.683099
96 | C,4,0.947761
97 | C,5,0.694805
98 | C,6,0.944444
99 | C,7,0.571429
100 | C,8,0.791045
101 | C,9,0.906977
102 | C,10,0.458647
103 | C,11,0.610687
104 | C,12,0.840764
105 | C,13,0.993243
106 | C,14,0.750000
107 | C,15,0.466667
108 | C,16,0.702899
109 | C,17,0.966667
110 | C,18,0.397351
111 | C,19,0.636364
112 | C,20,0.624060
113 | C,21,0.735714
114 | C,22,0.055944
115 | C,23,0.037037
116 | C,24,0.262411
117 | C,25,0.320000
118 | C,26,0.543478
119 | C,27,0.789062
120 | C,28,0.925373
121 | C,29,0.572581
122 | C,30,0.551181
123 | C,31,0.801418
124 | C,32,0.378205
125 | C,33,0.678082
126 | C,34,0.557823
127 | C,35,0.765957
128 | C,36,0.838710
129 | C,37,0.979730
130 | C,38,0.568182
131 | C,39,0.958042
132 | C,40,0.630872
133 | C,41,0.885496
134 | C,42,0.548148
135 | C,43,0.805195
136 | C,44,0.129252
137 | D,0,0.937931
138 | D,1,0.510204
139 | D,2,0.708333
140 | D,3,0.916667
141 | D,4,0.944882
142 | D,5,0.630137
143 | D,6,0.879699
144 | D,7,0.194444
145 | D,8,0.764706
146 | D,9,0.693333
147 | D,10,0.033557
148 | D,11,0.232258
149 | D,12,0.737705
150 | D,13,0.230216
151 | D,14,0.810458
152 | D,15,0.405594
153 | D,16,0.426036
154 | D,17,0.729927
155 | D,18,0.661290
156 | D,19,0.719424
157 | D,20,0.844156
158 | D,21,0.672000
159 | D,22,0.090909
160 | D,23,0.122449
161 | D,24,0.347458
162 | D,25,0.360544
163 | D,26,0.824427
164 | D,27,0.476190
165 | D,28,0.960526
166 | D,29,0.582192
167 | D,30,0.475309
168 | D,31,0.717557
169 | D,32,0.420290
170 | D,33,0.810811
171 | D,34,0.781250
172 | D,35,0.172414
173 | D,36,0.394161
174 | D,37,0.933333
175 | D,38,0.193548
176 | D,39,0.759259
177 | D,40,0.916667
178 | D,41,0.489051
179 | D,42,0.762590
180 | D,43,0.956522
181 | D,44,0.180556
182 | E,0,0.607595
183 | E,1,0.333333
184 | E,2,0.838462
185 | E,3,0.811024
186 | E,4,0.945205
187 | E,5,0.846774
188 | E,6,0.979167
189 | E,7,0.500000
190 | E,8,0.842105
191 | E,9,0.443709
192 | E,10,0.185714
193 | E,11,0.601399
194 | E,12,0.824818
195 | E,13,0.882759
196 | E,14,0.723577
197 | E,15,0.815951
198 | E,16,0.908397
199 | E,17,0.791667
200 | E,18,0.520833
201 | E,19,0.482270
202 | E,20,0.976744
203 | E,21,0.783217
204 | E,22,0.141791
205 | E,23,0.434783
206 | E,24,0.346667
207 | E,25,0.204545
208 | E,26,0.622222
209 | E,27,0.758865
210 | E,28,0.906977
211 | E,29,0.903448
212 | E,30,0.310345
213 | E,31,0.862595
214 | E,32,0.649254
215 | E,33,0.640288
216 | E,34,0.694444
217 | E,35,0.572581
218 | E,36,0.780000
219 | E,37,0.910448
220 | E,38,0.537037
221 | E,39,0.862069
222 | E,40,0.800000
223 | E,41,0.868750
224 | E,42,0.626016
225 | E,43,0.795276
226 | E,44,0.286765
227 |
--------------------------------------------------------------------------------
/datafiles/xval_cls.res50.5fold.csv:
--------------------------------------------------------------------------------
1 | fold_name,class_label,accuracy
2 | A,0,0.965035
3 | A,1,0.806667
4 | A,2,0.973856
5 | A,3,0.855072
6 | A,4,0.980263
7 | A,5,0.830645
8 | A,6,1.000000
9 | A,7,0.758865
10 | A,8,0.838926
11 | A,9,0.966443
12 | A,10,0.707692
13 | A,11,0.654412
14 | A,12,0.974026
15 | A,13,0.918519
16 | A,14,0.809211
17 | A,15,0.940741
18 | A,16,0.964029
19 | A,17,0.973684
20 | A,18,0.800000
21 | A,19,0.926667
22 | A,20,0.901316
23 | A,21,0.905405
24 | A,22,0.949275
25 | A,23,0.937500
26 | A,24,0.971631
27 | A,25,0.885906
28 | A,26,0.762821
29 | A,27,0.688406
30 | A,28,0.934783
31 | A,29,0.892617
32 | A,30,0.857143
33 | A,31,0.855172
34 | A,32,0.739726
35 | A,33,0.875000
36 | A,34,0.925170
37 | A,35,0.972222
38 | A,36,0.869919
39 | A,37,0.980000
40 | A,38,0.944099
41 | A,39,0.811024
42 | A,40,0.944828
43 | A,41,0.942029
44 | A,42,0.963768
45 | A,43,0.888060
46 | A,44,0.927536
47 | B,0,0.973913
48 | B,1,0.821918
49 | B,2,0.906977
50 | B,3,0.890511
51 | B,4,0.921986
52 | B,5,0.921053
53 | B,6,0.986395
54 | B,7,0.666667
55 | B,8,0.870229
56 | B,9,0.933884
57 | B,10,0.891892
58 | B,11,0.822222
59 | B,12,0.961538
60 | B,13,0.947368
61 | B,14,0.962121
62 | B,15,0.959677
63 | B,16,0.975610
64 | B,17,0.982906
65 | B,18,0.886525
66 | B,19,0.913386
67 | B,20,0.939394
68 | B,21,0.916667
69 | B,22,0.901961
70 | B,23,0.860294
71 | B,24,0.946667
72 | B,25,0.897959
73 | B,26,0.828571
74 | B,27,0.732877
75 | B,28,0.993197
76 | B,29,0.838235
77 | B,30,0.796992
78 | B,31,0.848684
79 | B,32,0.857143
80 | B,33,0.931298
81 | B,34,0.962687
82 | B,35,0.993151
83 | B,36,0.918519
84 | B,37,0.992481
85 | B,38,0.966942
86 | B,39,0.918699
87 | B,40,0.892086
88 | B,41,0.940299
89 | B,42,0.909091
90 | B,43,0.945578
91 | B,44,0.770370
92 | C,0,0.956835
93 | C,1,0.788321
94 | C,2,0.972222
95 | C,3,0.929577
96 | C,4,0.910448
97 | C,5,0.811688
98 | C,6,1.000000
99 | C,7,0.564286
100 | C,8,0.962687
101 | C,9,0.821705
102 | C,10,0.751880
103 | C,11,0.923664
104 | C,12,0.917197
105 | C,13,0.885135
106 | C,14,0.850000
107 | C,15,0.903704
108 | C,16,0.905797
109 | C,17,0.973333
110 | C,18,0.947020
111 | C,19,0.944056
112 | C,20,0.939850
113 | C,21,0.828571
114 | C,22,0.979021
115 | C,23,0.755556
116 | C,24,0.829787
117 | C,25,0.848000
118 | C,26,0.934783
119 | C,27,0.765625
120 | C,28,0.880597
121 | C,29,0.725806
122 | C,30,0.842520
123 | C,31,0.978723
124 | C,32,0.846154
125 | C,33,0.904110
126 | C,34,0.918367
127 | C,35,0.985816
128 | C,36,0.825806
129 | C,37,0.986486
130 | C,38,0.946970
131 | C,39,0.874126
132 | C,40,0.953020
133 | C,41,0.870229
134 | C,42,0.881481
135 | C,43,0.753247
136 | C,44,0.700680
137 | D,0,0.917241
138 | D,1,0.816327
139 | D,2,0.930556
140 | D,3,0.916667
141 | D,4,0.968504
142 | D,5,0.835616
143 | D,6,0.992481
144 | D,7,0.576389
145 | D,8,0.888889
146 | D,9,0.966667
147 | D,10,0.791946
148 | D,11,0.954839
149 | D,12,0.918033
150 | D,13,0.935252
151 | D,14,0.882353
152 | D,15,0.930070
153 | D,16,0.893491
154 | D,17,0.905109
155 | D,18,0.887097
156 | D,19,0.985612
157 | D,20,0.948052
158 | D,21,0.848000
159 | D,22,0.924242
160 | D,23,0.625850
161 | D,24,0.949153
162 | D,25,0.911565
163 | D,26,0.740458
164 | D,27,0.625850
165 | D,28,0.868421
166 | D,29,0.958904
167 | D,30,0.567901
168 | D,31,0.893130
169 | D,32,0.949275
170 | D,33,0.925676
171 | D,34,0.851562
172 | D,35,0.972414
173 | D,36,0.905109
174 | D,37,0.977778
175 | D,38,0.943548
176 | D,39,0.956790
177 | D,40,0.954545
178 | D,41,0.919708
179 | D,42,0.928058
180 | D,43,0.920290
181 | D,44,0.694444
182 | E,0,0.936709
183 | E,1,0.933333
184 | E,2,0.900000
185 | E,3,0.968504
186 | E,4,0.910959
187 | E,5,0.830645
188 | E,6,1.000000
189 | E,7,0.909836
190 | E,8,0.977444
191 | E,9,0.986755
192 | E,10,0.621429
193 | E,11,0.930070
194 | E,12,0.978102
195 | E,13,0.965517
196 | E,14,0.869919
197 | E,15,0.963190
198 | E,16,0.977099
199 | E,17,0.993056
200 | E,18,0.777778
201 | E,19,0.964539
202 | E,20,0.899225
203 | E,21,0.951049
204 | E,22,0.925373
205 | E,23,0.855072
206 | E,24,0.880000
207 | E,25,0.818182
208 | E,26,0.940741
209 | E,27,0.659574
210 | E,28,0.837209
211 | E,29,0.875862
212 | E,30,0.834483
213 | E,31,0.916031
214 | E,32,0.910448
215 | E,33,0.877698
216 | E,34,0.881944
217 | E,35,1.000000
218 | E,36,0.960000
219 | E,37,0.925373
220 | E,38,0.870370
221 | E,39,0.882759
222 | E,40,0.925926
223 | E,41,0.937500
224 | E,42,0.894309
225 | E,43,0.874016
226 | E,44,0.897059
227 |
--------------------------------------------------------------------------------
/notebooks/FuzzyFusion.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Fuzzy Machine Learning Model Fusion\n",
8 | "\n",
9 | "\n",
10 | " In this notebook, we will be working with the results of inference processing on remote sensing data from three deep convolutional neural networks (DCNN).\n",
11 | "\n"
12 | ]
13 | },
14 | {
15 | "cell_type": "markdown",
16 | "metadata": {},
17 | "source": [
18 | "## Deep Convolutional Neural Networks\n",
19 | "\n",
20 | "### ResNet50 \n",
21 | "\n",
22 | "\n",
23 | "\n",
24 | "**_Image from [Kaggle.com](https://kaggle.com)_**\n",
25 | "\n",
26 | "\n",
27 | "\n",
28 | "### InceptionV3\n",
29 | "\n",
30 | "\n",
31 | "\n",
32 | "**_Image from [Kaggle.com](https://kaggle.com)_**\n",
33 | "\n",
34 | "\n",
35 | "### DenseNet\n",
36 | "\n",
37 | "\n",
38 | "\n",
39 | "**_Image from [TowardsDataScience.com](https://towardsdatascience.com/)_**\n",
40 | "\n",
41 | "### Further Reading / References\n",
42 | " * [ResNet - Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)\n",
43 | " * [InceptionV3 - Rethinking the Inception Architecture for Computer Vision](https://arxiv.org/abs/1512.00567)\n",
44 | " * [DenseNet - Densely Connected Convolutional Networks](https://arxiv.org/abs/1608.06993)"
45 | ]
46 | },
47 | {
48 | "cell_type": "markdown",
49 | "metadata": {},
50 | "source": [
51 | "---\n",
52 | "\n",
53 | "## Data : Remote Sensing and Image Scene Classification\n",
54 | "\n",
55 | "\n",
56 | " The [RESISC-45 data set is described here.](https://arxiv.org/abs/1703.00121)\n",
57 | " The data is composed of 45 classes of remote image scenes.\n",
58 | " Below is a sample of the RESISC-45 data from the _arxiv_ paper.\n",
59 | "\n",
60 | "\n",
61 | "\n",
62 | "\n"
63 | ]
64 | },
65 | {
66 | "cell_type": "markdown",
67 | "metadata": {},
68 | "source": [
69 | "---\n",
70 | "\n",
71 | "## We did some heavy lifting for you!\n",
72 | "\n",
73 | "### aka - GPU Training\n",
74 | "\n",
75 | "**_Image from [Giphy](https://media.giphy.com/media/metK0W9OSCoyQ/giphy.gif)_**"
76 | ]
77 | },
78 | {
79 | "cell_type": "markdown",
80 | "metadata": {},
81 | "source": [
82 | "\n",
83 | " We have trained the three DCNN with the following hyperparameters:\n",
84 | "\n",
85 | "\n",
86 | " * Epochs : 15\n",
87 | " * Batch Size : 64\n",
88 | " * Optimizer: Adam \n",
89 | " * Initial Learning Rate: 1e-3"
90 | ]
91 | },
92 | {
93 | "cell_type": "markdown",
94 | "metadata": {},
95 | "source": [
96 | "\n",
97 | " Hardware\n",
98 | "\n",
99 | "\n",
100 | " * Nvidia V100\n",
101 | " * approximately 10 hours for each architecture's 5-fold experiments\n"
102 | ]
103 | },
104 | {
105 | "cell_type": "markdown",
106 | "metadata": {},
107 | "source": [
108 | "\n",
109 | "Performance Characteristics - 5-Fold Cross-Validation\n",
110 | "\n",
111 | "\n",
112 | " * ResNet50 : 92.1%\n",
113 | " * InceptionV3 : 60.6%\n",
114 | " * DenseNet : 89.1%"
115 | ]
116 | },
117 | {
118 | "cell_type": "markdown",
119 | "metadata": {},
120 | "source": [
121 | "---\n",
122 | "\n",
123 | "## Fusion with the Choquet Fuzzy Integral\n",
124 | "\n",
125 | "\n",
126 | "We will be using the techniques of enhanced decision information fusion from [this publication](https://doi.org/10.1109/LGRS.2018.2839092).\n",
127 | "\n",
128 | "\n",
129 | ""
130 | ]
131 | },
132 | {
133 | "cell_type": "markdown",
134 | "metadata": {},
135 | "source": [
136 | "---\n",
137 | "\n",
138 | "# Choquet Fuzzy Integral Code\n",
139 | "\n",
140 | "\n",
141 | "\n",
142 | "\n",
143 | "Import some libraries, include a path to our local src/ChI.py file.\n",
144 | ""
145 | ]
146 | },
147 | {
148 | "cell_type": "code",
149 | "execution_count": null,
150 | "metadata": {},
151 | "outputs": [],
152 | "source": [
153 | "import os\n",
154 | "import sys\n",
155 | "module_path = os.path.abspath(os.path.join('../src'))\n",
156 | "if module_path not in sys.path:\n",
157 | " sys.path.append(module_path)\n",
158 | " \n",
159 | "import ChI\n",
160 | "import numpy as np\n",
161 | "import csv\n",
162 | "from os import listdir\n",
163 | "from os.path import isfile, join\n",
164 | "import pandas as pd\n",
165 | "import cvxopt\n",
166 | "import scipy.special as sp"
167 | ]
168 | },
169 | {
170 | "cell_type": "markdown",
171 | "metadata": {},
172 | "source": [
173 | "\n",
174 | " Normalize our outputs into a typical _softmax_ vector.\n",
175 | ""
176 | ]
177 | },
178 | {
179 | "cell_type": "code",
180 | "execution_count": null,
181 | "metadata": {},
182 | "outputs": [],
183 | "source": [
184 | "def soft_max(samples):\n",
185 | " # Normalizes each sample w.r.t. each sample (soft max)\n",
186 | " for i in range(0, samples.shape[0]): # for each sample\n",
187 | " for j in range(0, samples.shape[1]): # for each network\n",
188 | " samples[i,j,:] = sp.softmax(samples[i,j,:])\n",
189 | "\n",
190 | " return [samples]\n",
191 | "\n",
192 | "\n"
193 | ]
194 | },
195 | {
196 | "cell_type": "markdown",
197 | "metadata": {},
198 | "source": [
199 | "\n",
200 | " Process some prediction and cross-validation files.\n",
201 | ""
202 | ]
203 | },
204 | {
205 | "cell_type": "code",
206 | "execution_count": null,
207 | "metadata": {},
208 | "outputs": [],
209 | "source": [
210 | "# Specify path for network outputs & csv containing cv accuracies\n",
211 | "network_path = '../datafiles/pred'\n",
212 | "cross_val_path = '../datafiles/cross_val'\n",
213 | "\n",
214 | "# set up variables\n",
215 | "image_names = []\n"
216 | ]
217 | },
218 | {
219 | "cell_type": "code",
220 | "execution_count": null,
221 | "metadata": {},
222 | "outputs": [],
223 | "source": [
224 | "csv_files = [f for f in listdir(network_path) if isfile(join(network_path, f))] # this is for the data files\n",
225 | "cross_val = [f for f in listdir(cross_val_path) if isfile(join(cross_val_path, f))] # this is for the cv accuracies\n",
226 | "num_nets = csv_files.__len__() # how many nets? "
227 | ]
228 | },
229 | {
230 | "cell_type": "code",
231 | "execution_count": null,
232 | "metadata": {},
233 | "outputs": [],
234 | "source": [
235 | "# Create dictionary to store samples\n",
236 | "data = dict.fromkeys(csv_files)\n",
237 | "\n",
238 | "# Read in all of the csv data into a dictionary\n",
239 | "densities = []\n",
240 | "for file in cross_val:\n",
241 | " csv_data = []\n",
242 | " data_info = np.genfromtxt((cross_val_path + '/' + file), usecols=(1), skip_header=True,dtype=\"f\", delimiter=',')\n",
243 | " densities.append(np.mean(data_info))\n",
244 | "\n",
245 | "densities = np.asarray(densities)\n",
246 | "print(densities)"
247 | ]
248 | },
249 | {
250 | "cell_type": "code",
251 | "execution_count": null,
252 | "metadata": {},
253 | "outputs": [],
254 | "source": [
255 | "first_net = 1 # this is a flag.\n",
256 | "print(csv_files)"
257 | ]
258 | },
259 | {
260 | "cell_type": "code",
261 | "execution_count": null,
262 | "metadata": {},
263 | "outputs": [],
264 | "source": [
265 | "for file in csv_files:\n",
266 | " csv_data = []\n",
267 | " data_info = np.genfromtxt((network_path + '/' + file), usecols=(1, 2, 3), dtype=\"|U\", delimiter=',') \n",
268 | " confidence_vectors = np.genfromtxt((network_path + '/' + file), delimiter=';')\n",
269 | "\n",
270 | " for line in range(0, data_info.__len__()):\n",
271 | " if first_net:\n",
272 | " image_names.append(data_info[line,0])\n",
273 | " csv_data.append(np.hstack((data_info[line,:-1], data_info[line,2].partition(';')[0], confidence_vectors[line, 1:])))\n",
274 | "\n",
275 | " first_net = 0\n",
276 | " data[file] = csv_data"
277 | ]
278 | },
279 | {
280 | "cell_type": "code",
281 | "execution_count": null,
282 | "metadata": {},
283 | "outputs": [],
284 | "source": [
285 | "\n",
286 | "# How many classes are there?\n",
287 | "num_classes = confidence_vectors.shape[1]# Assuming the first 4 columns are 'image\ty_true\tconfidence\ty_pred'\n",
288 | "\n",
289 | " # Now I need to build the samples and their corresponding labels\n",
290 | " # There will be the same number of ChI's as there are classes(L0\n",
291 | " # One ChI per class, so each one sample will turn into L samples\n",
292 | "num_samples = data_info.__len__()\n",
293 | "samples = np.zeros([num_samples, csv_files.__len__(), num_classes])\n",
294 | "label = np.zeros([num_samples, num_classes])\n",
295 | "\n",
296 | "samples_list = []\n",
297 | "\n",
298 | "for i in range(0,3):\n",
299 | " samples_list.append(list(pd.read_csv(f'samples{0}.csv', header=None, index_col=0).itertuples()))\n",
300 | "samples = np.asarray(samples_list).transpose(1,0,2)\n",
301 | "label = np.asarray(list(pd.read_csv(f'label.csv', header=None, index_col=0).itertuples()))\n"
302 | ]
303 | },
304 | {
305 | "cell_type": "markdown",
306 | "metadata": {},
307 | "source": [
308 | "## Start Training and Testing"
309 | ]
310 | },
311 | {
312 | "cell_type": "code",
313 | "execution_count": null,
314 | "metadata": {},
315 | "outputs": [],
316 | "source": [
317 | "##############################################\n",
318 | "# Start Training and Testing\n",
319 | "##############################################\n",
320 | "print('--Starting Training and Testing')\n",
321 | "\n",
322 | "train_samples = samples.copy()\n",
323 | "train_labels = label\n",
324 | "\n",
325 | "test_samples = samples.copy()\n",
326 | "test_labels = label\n",
327 | "##############################################\n",
328 | "# Normalize Training Data & Testing Data\n",
329 | "##############################################\n",
330 | "\n",
331 | "[train_samples] = soft_max(train_samples)\n",
332 | "[test_samples] = soft_max(test_samples)\n"
333 | ]
334 | },
335 | {
336 | "cell_type": "markdown",
337 | "metadata": {},
338 | "source": [
339 | "### Use the cross-validation performance to seed the densities, then solve the Sugeno Lambda Fuzzy Measure"
340 | ]
341 | },
342 | {
343 | "cell_type": "code",
344 | "execution_count": null,
345 | "metadata": {},
346 | "outputs": [],
347 | "source": [
348 | "print('--Training--')\n",
349 | "##############################################\n",
350 | "# Train the ChI(s)\n",
351 | "##############################################\n",
352 | "\n",
353 | "CHIs = []\n",
354 | "for j in range(0, num_classes):\n",
355 | " CHIs.append(ChI.ChoquetIntegral())\n",
356 | "\n",
357 | "for j, chi in enumerate(CHIs):\n",
358 | " print('Class ChI {}'.format(j))\n",
359 | " tr = chi.train_chi_sugeno(densities)\n",
360 | " print(chi.fm)"
361 | ]
362 | },
363 | {
364 | "cell_type": "code",
365 | "execution_count": null,
366 | "metadata": {},
367 | "outputs": [],
368 | "source": [
369 | "print('--Testing--')\n",
370 | "##############################################\n",
371 | "# Test the ChI(s)\n",
372 | "##############################################\n",
373 | "exper_out, known_out = [], []\n",
374 | "dec = []\n",
375 | "for j in range(0, test_samples.shape[0]): # for each data point\n",
376 | " out = []\n",
377 | " for k, chi in enumerate(CHIs): # for each ChI\n",
378 | " test_sample = np.transpose(test_samples[j, :, k])\n",
379 | " test_label = np.argmax(test_labels[j, :])\n",
380 | " out.append(chi.chi_sugeno(test_sample))\n",
381 | " out = np.asarray(out)\n",
382 | " exper_out.append(np.argmax(out))\n",
383 | " known_out.append(test_label)\n",
384 | "for i in range(0, exper_out.__len__()):\n",
385 | " if exper_out[i] == known_out[i]:\n",
386 | " dec.append(1)\n",
387 | " else:\n",
388 | " dec.append(0)\n",
389 | "with open('Results{}.csv'.format(i), 'w', newline='') as csvfile:\n",
390 | " spamwriter = csv.writer(csvfile, delimiter=',')\n",
391 | " for k in range(0, exper_out.__len__()):\n",
392 | " spamwriter.writerow([exper_out[k], known_out[k]])\n",
393 | "\n",
394 | "acc = np.sum(dec) / dec.__len__()\n",
395 | "print(acc)"
396 | ]
397 | },
398 | {
399 | "cell_type": "markdown",
400 | "metadata": {},
401 | "source": [
402 | "---\n",
403 | "\n",
404 | "## Data-driven learning of the fuzzy measure\n",
405 | "\n",
406 | "### Training and Testing"
407 | ]
408 | },
409 | {
410 | "cell_type": "code",
411 | "execution_count": null,
412 | "metadata": {},
413 | "outputs": [],
414 | "source": [
415 | "# DATA DRIVEN\n",
416 | "print('--Training--')\n",
417 | "##############################################\n",
418 | "# Train the ChI(s)\n",
419 | "##############################################\n",
420 | "\n",
421 | "CHIs = []\n",
422 | "for j in range(0, num_classes):\n",
423 | " CHIs.append(ChI.ChoquetIntegral())\n",
424 | " \n",
425 | "for j, chi in enumerate(CHIs):\n",
426 | " print('Class ChI {}'.format(j))\n",
427 | " train_data = np.transpose(train_samples[:,:,j])\n",
428 | " label_data = train_labels[:, j]\n",
429 | " tr = chi.train_chi_quad(train_data, label_data)\n",
430 | " print(chi.fm)\n"
431 | ]
432 | },
433 | {
434 | "cell_type": "code",
435 | "execution_count": null,
436 | "metadata": {},
437 | "outputs": [],
438 | "source": [
439 | "print('--Testing--')\n",
440 | "##############################################\n",
441 | "# Test the ChI(s)\n",
442 | "##############################################\n",
443 | "exper_out, known_out = [], []\n",
444 | "dec = []\n",
445 | "for j in range(0, test_samples.shape[0]): # for each data point\n",
446 | " out = []\n",
447 | " for k, chi in enumerate(CHIs): # for each ChI\n",
448 | " test_sample = np.transpose(test_samples[j, :, k])\n",
449 | " test_label = np.argmax(test_labels[j, :])\n",
450 | " out.append(chi.chi_quad(test_sample))\n",
451 | " out = np.asarray(out)\n",
452 | " exper_out.append(np.argmax(out))\n",
453 | " known_out.append(test_label)\n",
454 | "for i in range(0, exper_out.__len__()):\n",
455 | " if exper_out[i] == known_out[i]:\n",
456 | " dec.append(1)\n",
457 | " else:\n",
458 | " dec.append(0)\n",
459 | "with open('Results{}.csv'.format(i), 'w', newline='') as csvfile:\n",
460 | " spamwriter = csv.writer(csvfile, delimiter=',')\n",
461 | " for k in range(0, exper_out.__len__()):\n",
462 | " spamwriter.writerow([exper_out[k], known_out[k]])\n",
463 | "\n",
464 | "acc = np.sum(dec) / dec.__len__()"
465 | ]
466 | },
467 | {
468 | "cell_type": "code",
469 | "execution_count": null,
470 | "metadata": {},
471 | "outputs": [],
472 | "source": [
473 | "print(acc)"
474 | ]
475 | },
476 | {
477 | "cell_type": "markdown",
478 | "metadata": {},
479 | "source": [
480 | "# You have completed Part 3\n",
481 | "# Thanks for attending"
482 | ]
483 | }
484 | ],
485 | "metadata": {
486 | "kernelspec": {
487 | "display_name": "Python 3",
488 | "language": "python",
489 | "name": "python3"
490 | },
491 | "language_info": {
492 | "codemirror_mode": {
493 | "name": "ipython",
494 | "version": 3
495 | },
496 | "file_extension": ".py",
497 | "mimetype": "text/x-python",
498 | "name": "python",
499 | "nbconvert_exporter": "python",
500 | "pygments_lexer": "ipython3",
501 | "version": "3.7.3"
502 | }
503 | },
504 | "nbformat": 4,
505 | "nbformat_minor": 2
506 | }
507 |
--------------------------------------------------------------------------------
/notebooks/IntroductionToPyTorch.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Introduction to PyTorch\n",
8 | "\n",
9 | "\n",
10 | " In this section, we will cover some essential concepts of PyTorch.\n",
11 | "\n",
12 | "\n",
13 | "* Tensors\n",
14 | "* Autograd\n",
15 | "* Layers"
16 | ]
17 | },
18 | {
19 | "cell_type": "code",
20 | "execution_count": null,
21 | "metadata": {},
22 | "outputs": [],
23 | "source": [
24 | "%matplotlib inline\n",
25 | "from matplotlib import pyplot as plt\n",
26 | "\n",
27 | "import torch"
28 | ]
29 | },
30 | {
31 | "cell_type": "markdown",
32 | "metadata": {},
33 | "source": [
34 | "## PyTorch Tensors\n",
35 | "\n",
36 | "\n",
37 | "Tensors are the main computational data structure suitable for many scientific computations. \n",
38 | "They are N-dimensional arrays that represent highly structured data with many pre-defined operations, e.g. matices. \n",
39 | "They can be easily transferred across computing devices (GPUs) in order to accelerate the computation.
\n",
40 | "Next, we'll give some quick examples to demonstrate these advantages of tensors.\n",
41 | ""
42 | ]
43 | },
44 | {
45 | "cell_type": "markdown",
46 | "metadata": {},
47 | "source": [
48 | "### Create some ramdom vectors\n",
49 | "\n",
50 | "\n",
51 | "Below, we create two vectors and sample values from a uniform distribution.\n",
52 | ""
53 | ]
54 | },
55 | {
56 | "cell_type": "code",
57 | "execution_count": null,
58 | "metadata": {},
59 | "outputs": [],
60 | "source": [
61 | "x = torch.empty(300).uniform_(-1,1)\n",
62 | "y = torch.empty(300).uniform_(-1,1)"
63 | ]
64 | },
65 | {
66 | "cell_type": "markdown",
67 | "metadata": {},
68 | "source": [
69 | "### Run some simple calculation like the covariance\n",
70 | "\n",
71 | "\n",
72 | "Here you can see the common multi-dimentional array arithmetics happening, such as:\n",
73 | "\n",
74 | "\n",
75 | "* Broadcasting between `x` and `x.mean()`\n",
76 | "* Dot product\n",
77 | " * Type conversion\n",
78 | "\n",
79 | "> Note that the computation will be carried out using `tensor` as the data structure.\n",
80 | "> That's why you should see tensor(.) as a result."
81 | ]
82 | },
83 | {
84 | "cell_type": "code",
85 | "execution_count": null,
86 | "metadata": {},
87 | "outputs": [],
88 | "source": [
89 | "torch.dot(x - x.mean(), y - y.mean()) / x.shape[0]"
90 | ]
91 | },
92 | {
93 | "cell_type": "markdown",
94 | "metadata": {},
95 | "source": [
96 | "### Let's create a sample dataset to play with\n",
97 | "\n",
98 | "If we view the two vectors as values from the 2 axes of a 2-D coordinate system as shown below and label the data points based on their signs, we can generate the famous XOR dataset to experiment with.\n",
99 | "\n",
100 | "\n",
101 | "> Notice how we used `.numpy()` to convert the tensors into numpy arrays, which are also multi-dimensional arrays but they can be operated on CPUs only. \n",
102 | "> Similarly, if we want to transfer a tensor to GPU, we could use the method `.cuda()`."
103 | ]
104 | },
105 | {
106 | "cell_type": "code",
107 | "execution_count": null,
108 | "metadata": {},
109 | "outputs": [],
110 | "source": [
111 | "d = x*y>=0\n",
112 | "plt.figure(figsize=(6,6))\n",
113 | "plt.scatter(x.numpy(), y.numpy(), c=d.numpy())"
114 | ]
115 | },
116 | {
117 | "cell_type": "markdown",
118 | "metadata": {},
119 | "source": [
120 | "## Artificial Neuron & Autograd\n",
121 | "\n",
122 | "\n",
123 | "It is an elementary mathematical component in artificial neural networks, \n",
124 | "that can be trained to create a model suitable for some specific problem and training data. \n",
125 | "The learned knowledge is stored as weights $w$ and bias $b$. \n",
126 | "The input data is transformed using a linear combination and some nonlinear function known as the activation $\\varphi$.\n",
127 | "\n",
128 | "\n",
129 | "$$\n",
130 | "y = \\varphi(w^T x+b)\n",
131 | "$$\n",
132 | "\n",
133 | "\n",
134 | "Neural networks are typically built by stacking up layers of neurons that defines the mathetical model,\n",
135 | "which learn from training data using an optimization algorithm.\n",
136 | "This training procedure usually involves the following key steps:\n",
137 | "\n",
138 | "\n",
139 | "1. Fordward pass: passing the training data forward through the network to get some prediction\n",
140 | "2. Compute error: a criterion (aka loss function) is then applied to compare the prediction against training labels\n",
141 | "3. Compute gradient: computing a plausible direction towards which the model may get better prediction i.e. minize the error\n",
142 | "4. Update parameters: changing model parameters, include weights and biases based on the gradient (and some other factors depending on the optimization algorithm)\n",
143 | "\n",
144 | "\n",
145 | "Next, we demonstrate how this is done with PyTorch on a microscopic level with one neuron.\n",
146 | "\n",
147 | "\n",
148 | "> `requires_grad=True` enables autograd for a tensor variable. \n",
149 | "> In `y.reshape(-1, 1)`, -1 means any."
150 | ]
151 | },
152 | {
153 | "cell_type": "code",
154 | "execution_count": null,
155 | "metadata": {},
156 | "outputs": [],
157 | "source": [
158 | "# Set up a neuron with sigmoid activation and mean squared error loss.\n",
159 | "X = torch.cat((x.reshape(-1, 1), y.reshape(-1, 1)),1)\n",
160 | "w = torch.randn(2, requires_grad=True)\n",
161 | "\n",
162 | "# Forward pass\n",
163 | "y_pred = 1/(1+torch.exp(-w.dot(X[0])))\n",
164 | "\n",
165 | "# Compute error\n",
166 | "loss = (d[0] - y_pred).pow(2)\n",
167 | "\n",
168 | "# Backward pass\n",
169 | "loss.backward()\n",
170 | "\n",
171 | "# Here's the gradient of the weights wrt loss.\n",
172 | "print(f\"\"\"Autograd => {w.grad}\n",
173 | "Chain Rule => {2 * (d[0] - y_pred) * (-1) * y_pred * (1-y_pred) * X[0]}\"\"\")"
174 | ]
175 | },
176 | {
177 | "cell_type": "markdown",
178 | "metadata": {},
179 | "source": [
180 | "## Dense Neuron Layer\n",
181 | "\n",
182 | "\n",
183 | "The most simple way to create an artificial neural network is fully connect neurons between layers, which creates a perceptron model.\n",
184 | "With the understanding of how tensors and autograd work in PyTorch, let's create a simple neural network, with the help of the NN layers that PyTorch provides (specifically `torch.nn.Linear` and `torch.nn.ReLU`), to solve the XOR dataset that we have constructed earlier.\n",
185 | "\n",
186 | "\n",
187 | "\n",
188 | "(A visualization of this model: [click here](http://playground.tensorflow.org/#activation=relu&batchSize=10&dataset=xor®Dataset=reg-plane&learningRate=0.1®ularizationRate=0&noise=0&networkShape=3,2&seed=0.39975&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false))\n",
189 | ""
190 | ]
191 | },
192 | {
193 | "cell_type": "code",
194 | "execution_count": null,
195 | "metadata": {},
196 | "outputs": [],
197 | "source": [
198 | "class XORNet(torch.nn.Module):\n",
199 | " def __init__(self):\n",
200 | " super(XORNet, self).__init__()\n",
201 | " self.hidden2 = torch.nn.Linear(2, 3, bias=False)\n",
202 | " self.hidden2.weight.data = torch.Tensor([[2,0],[0,-2],[2,-2]]) # <= How to preload weights\n",
203 | " self.hidden2.weight.requires_grad = False # <= How to lock weights\n",
204 | " self.act2 = torch.nn.ReLU()\n",
205 | " self.hidden1 = torch.nn.Linear(3, 2)\n",
206 | " self.act1 = torch.nn.ReLU()\n",
207 | " self.out = torch.nn.Linear(2, 1)\n",
208 | " self.act0 = torch.nn.ReLU()\n",
209 | "\n",
210 | " def forward(self, x):\n",
211 | " x = self.act2(self.hidden2(x))\n",
212 | " x = self.act1(self.hidden1(x))\n",
213 | " x = self.act0(self.out(x))\n",
214 | " return x"
215 | ]
216 | },
217 | {
218 | "cell_type": "markdown",
219 | "metadata": {},
220 | "source": [
221 | "### Training loop\n",
222 | "\n",
223 | "\n",
224 | "We will use the first 3/4 of the samples as training set and the last 1/4 of the samples as the testing set.\n",
225 | "\n",
226 | "\n",
227 | "> Your results may vary. You may run the following cell many times. \n",
228 | "> Due to the small amount of sample data, this training loop does not show batching, but uses the entire train/test subset."
229 | ]
230 | },
231 | {
232 | "cell_type": "code",
233 | "execution_count": null,
234 | "metadata": {},
235 | "outputs": [],
236 | "source": [
237 | "model = XORNet()\n",
238 | "optimizer = torch.optim.SGD(model.parameters(), lr=0.1)\n",
239 | "\n",
240 | "loss_history = [] # Training loss\n",
241 | "vloss_history = [] # Validation loss\n",
242 | "vacc_history = [] # Validation accuracy\n",
243 | "\n",
244 | "EPOCHS = 100\n",
245 | "NUM_TRAIN = X.shape[0] // 4 * 3\n",
246 | "NUM_TEST = X.shape[0] - NUM_TRAIN\n",
247 | "\n",
248 | "for _ in range(EPOCHS):\n",
249 | " model.train()\n",
250 | " with torch.set_grad_enabled(True):\n",
251 | " # Forward pass\n",
252 | " y_pred = model(X[:NUM_TRAIN])\n",
253 | "\n",
254 | " # Compute error\n",
255 | " loss = (y_pred - d[:NUM_TRAIN].reshape(-1, 1).float()).pow(2).sum() / NUM_TRAIN\n",
256 | "\n",
257 | " # Backward pass\n",
258 | " optimizer.zero_grad()\n",
259 | " loss.backward()\n",
260 | " \n",
261 | " # Update weights\n",
262 | " optimizer.step()\n",
263 | " \n",
264 | " loss_history.append(loss.item())\n",
265 | " \n",
266 | " model.eval()\n",
267 | " with torch.set_grad_enabled(False):\n",
268 | " y_pred = model(X[NUM_TRAIN:])\n",
269 | " vloss = y_pred - d[NUM_TRAIN:].reshape(-1, 1).float()\n",
270 | " acc = torch.sum((torch.abs(vloss)<0.5).float()) / NUM_TEST\n",
271 | " vloss = vloss.pow(2).sum() / NUM_TEST\n",
272 | " vloss_history.append(vloss.item())\n",
273 | " vacc_history.append(acc.item())\n",
274 | "\n",
275 | "plt.plot(list(range(EPOCHS)), loss_history, label='loss')\n",
276 | "plt.plot(list(range(EPOCHS)), vloss_history, label='vloss')\n",
277 | "plt.plot(list(range(EPOCHS)), vacc_history, label='vacc')\n",
278 | "plt.legend()"
279 | ]
280 | },
281 | {
282 | "cell_type": "markdown",
283 | "metadata": {},
284 | "source": [
285 | "# Congrats, Part 1 complete!"
286 | ]
287 | }
288 | ],
289 | "metadata": {
290 | "kernelspec": {
291 | "display_name": "Python 3",
292 | "language": "python",
293 | "name": "python3"
294 | },
295 | "language_info": {
296 | "codemirror_mode": {
297 | "name": "ipython",
298 | "version": 3
299 | },
300 | "file_extension": ".py",
301 | "mimetype": "text/x-python",
302 | "name": "python",
303 | "nbconvert_exporter": "python",
304 | "pygments_lexer": "ipython3",
305 | "version": "3.7.3"
306 | }
307 | },
308 | "nbformat": 4,
309 | "nbformat_minor": 2
310 | }
311 |
--------------------------------------------------------------------------------
/notebooks/PyTorchResNet50.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "metadata": {},
6 | "source": [
7 | "# Deep Convolutional Neural Networks with PyTorch"
8 | ]
9 | },
10 | {
11 | "cell_type": "markdown",
12 | "metadata": {},
13 | "source": [
14 | "# Deep Neural Models in PyTorch\n",
15 | "\n",
16 | "\n",
17 | "In this session, we will learn to use deep models provide by PyTorch from the `torchvision` package to create a classifier and perform cross validation.\n",
18 | "\n",
19 | "### ResNet50 \n",
20 | "\n",
21 | "Reference [ResNet - Deep Residual Learning for Image Recognition](https://arxiv.org/abs/1512.03385)\n",
22 | "\n",
23 | "\n",
24 | "\n",
25 | "**_Image from [Kaggle.com](https://kaggle.com)_**\n",
26 | "\n",
27 | "\n"
28 | ]
29 | },
30 | {
31 | "cell_type": "code",
32 | "execution_count": null,
33 | "metadata": {},
34 | "outputs": [],
35 | "source": [
36 | "from pathlib import Path\n",
37 | "from PIL import Image\n",
38 | "import psycopg2\n",
39 | "import torch\n",
40 | "import torchvision\n",
41 | "import torchvision.models as models"
42 | ]
43 | },
44 | {
45 | "cell_type": "markdown",
46 | "metadata": {},
47 | "source": [
48 | "# Tranfer Learning with ResNet50\n",
49 | "\n",
50 | "## Instantiating a model with new classifier\n",
51 | "\n",
52 | "First, we create a pretrained model from `torchvision.models`.\n",
53 | "Then we change the final fully connected layer with a `torch.nn.Linear` layer and specify the number of input/output features.\n"
54 | ]
55 | },
56 | {
57 | "cell_type": "code",
58 | "execution_count": null,
59 | "metadata": {},
60 | "outputs": [],
61 | "source": [
62 | "# Specify number of classes\n",
63 | "NUM_CLASS = 45\n",
64 | "\n",
65 | "def build_res50(fname_load=None):\n",
66 | " model = models.resnet50(pretrained=(fname_load is None))\n",
67 | " model.fc = torch.nn.Linear(in_features=2048, out_features=NUM_CLASS)\n",
68 | " if fname_load:\n",
69 | " print('Loading pretrained weights.')\n",
70 | " model.load_state_dict(torch.load(fname_load))\n",
71 | " return model"
72 | ]
73 | },
74 | {
75 | "cell_type": "markdown",
76 | "metadata": {},
77 | "source": [
78 | "## Prepping Data Set for Training\n",
79 | "\n",
80 | "In this stage we will prepare the dataset for cross validation.\n",
81 | "
\n",
82 | "This is a little example that uses 3 classes with 200 images per class from RESISC45 dataset.\n",
83 | "
\n",
84 | "We define a class which inherits from torch.utils.data.Dataset as an to load data for PyTorch models. \n",
85 | "
\n",
86 | "The `ImageDataset` class takes (image, class_label) as input metadata and loads the image from the disk as well.\n",
87 | "
\n",
88 | "We have staged these metadata in a PostgreSQL database to facilitate cross validation, among other possible analyses, but we will also provide this metadata in a CSV format in the repository, so you do not have to rely on PostgreSQL.\n",
89 | ""
90 | ]
91 | },
92 | {
93 | "cell_type": "code",
94 | "execution_count": null,
95 | "metadata": {},
96 | "outputs": [],
97 | "source": [
98 | "DATASET = '/dsa/data/IEEE/RESISC45'\n",
99 | "METADATA = 'host=pgsql-ieee.dsa.lan dbname=resisc45 user=readonly_user password=fuzzieee'"
100 | ]
101 | },
102 | {
103 | "cell_type": "code",
104 | "execution_count": null,
105 | "metadata": {},
106 | "outputs": [],
107 | "source": [
108 | "class ImageDataset(torch.utils.data.Dataset):\n",
109 | " def __init__(self, dataset, metadata, transform):\n",
110 | " self.dataset = Path(dataset)\n",
111 | " self.transform = transform\n",
112 | " self.metadata = metadata\n",
113 | "\n",
114 | " def __getitem__(self, idx):\n",
115 | " frame_id, image, class_label = self.metadata[idx]\n",
116 | " fname = str(self.dataset / image)\n",
117 | " return dict(frame_id=frame_id, image=self.transform(Image.open(fname).convert(\"RGB\")), label=class_label)\n",
118 | "\n",
119 | " def __len__(self):\n",
120 | " return len(self.metadata)"
121 | ]
122 | },
123 | {
124 | "cell_type": "code",
125 | "execution_count": null,
126 | "metadata": {},
127 | "outputs": [],
128 | "source": [
129 | "def xval_metadata(fold, test=False):\n",
130 | " print('Loading (traintest)', '=>', ('test' if test else 'train'))\n",
131 | " with psycopg2.connect(METADATA) as conn:\n",
132 | " with conn.cursor() as cur:\n",
133 | " if test:\n",
134 | " sql = \"\"\"\n",
135 | " select id, image, class_label from (select id, image, class_label, row_number()\n",
136 | " over (partition by class_label order by random())\n",
137 | " from frame where class_label in (3,8,33)) as foo\n",
138 | " where row_number %% 5 = %s and row_number<201;\n",
139 | " \"\"\"\n",
140 | " else:\n",
141 | " sql = \"\"\"\n",
142 | " select id, image, class_label from (select id, image, class_label, row_number()\n",
143 | " over (partition by class_label order by random())\n",
144 | " from frame where class_label in (3,8,33)) as foo\n",
145 | " where row_number %% 5 != %s and row_number<201;\n",
146 | " \"\"\"\n",
147 | " cur.execute(sql, (fold,))\n",
148 | " return cur.fetchall()\n",
149 | "\n",
150 | " "
151 | ]
152 | },
153 | {
154 | "cell_type": "markdown",
155 | "metadata": {},
156 | "source": [
157 | "After each image is loaded a preprocessing pipeline can be supplied to PyTorch as well.\n",
158 | "Here's an example, where we resize the input images and convert them into PyTorch tensors (from `PIL.Image`) and normalize.\n",
159 | "\n",
160 | "\n",
161 | "> Some of the pipeline operations may only applied to tensors, others may be only applicable to images.\n",
162 | "> This is also the reason we arrage the pipeline operations in such an order. Please refer to PyTorch documentation to find out more about the pipeline usage.\n",
163 | "> ref: https://pytorch.org/docs/stable/torchvision/transforms.html"
164 | ]
165 | },
166 | {
167 | "cell_type": "code",
168 | "execution_count": null,
169 | "metadata": {},
170 | "outputs": [],
171 | "source": [
172 | "transform_pipe = torchvision.transforms.Compose([\n",
173 | " torchvision.transforms.Resize(\n",
174 | " size=(299, 299)\n",
175 | " ),\n",
176 | " torchvision.transforms.ToTensor(),\n",
177 | " torchvision.transforms.Normalize(\n",
178 | " mean=[0.485, 0.456, 0.406],\n",
179 | " std=[0.229, 0.224, 0.225]\n",
180 | " )\n",
181 | "])"
182 | ]
183 | },
184 | {
185 | "cell_type": "markdown",
186 | "metadata": {},
187 | "source": [
188 | "### Exporting metadata to .csv\n",
189 | "\n",
190 | "In case you do not wish to rely on PostgreSQL, you may use the csv exports as metadata to initialize the `ImageDataset`. This is also available at `datafiles/*.csv`."
191 | ]
192 | },
193 | {
194 | "cell_type": "raw",
195 | "metadata": {},
196 | "source": [
197 | "import pandas as pd\n",
198 | "for fold in range(5):\n",
199 | " print(\"Fold\", fold)\n",
200 | " df = pd.DataFrame(xval_metadata(fold, test=False), columns=['id', 'image', 'class_label'])\n",
201 | " df.drop(['id'], axis=1, inplace=True)\n",
202 | " print('exported', len(df))\n",
203 | " df.to_csv(f'xval-{fold}.train.csv')\n",
204 | " df = pd.DataFrame(xval_metadata(fold, test=True), columns=['id', 'image', 'class_label'])\n",
205 | " df.drop(['id'], axis=1, inplace=True)\n",
206 | " df.to_csv(f'xval-{fold}.test.csv')\n",
207 | " print('exported', len(df))"
208 | ]
209 | },
210 | {
211 | "cell_type": "markdown",
212 | "metadata": {},
213 | "source": [
214 | "\n",
215 | "To make use of this data, replace the xval_metadata() function and have it read from the CSVs.\n",
216 | " "
217 | ]
218 | },
219 | {
220 | "cell_type": "raw",
221 | "metadata": {},
222 | "source": [
223 | "import pandas as pd\n",
224 | "\n",
225 | "#### NEW DEFINITION (switch Cell Type to code to run)\n",
226 | "def xval_metadata(fold, test=False):\n",
227 | " return list(pd.read_csv(f'../datafiles/xval-{fold}.{\"test\" if test else \"train\"}.csv', index_col=0).itertuples())"
228 | ]
229 | },
230 | {
231 | "cell_type": "markdown",
232 | "metadata": {},
233 | "source": [
234 | "## Hyperparameters\n",
235 | "\n",
236 | "\n",
237 | "Hyperparameters are the variables which determines the network and the training configuration,\n",
238 | "e.g. units of neurons in each layer or the learning rate or the batch size. \n",
239 | "They are parameters that are not getting changed by the optimization algorithm,\n",
240 | "and typically they remain constants throughout the network training.\n",
241 | ""
242 | ]
243 | },
244 | {
245 | "cell_type": "code",
246 | "execution_count": null,
247 | "metadata": {},
248 | "outputs": [],
249 | "source": [
250 | "# Specify number of epochs per x-fold\n",
251 | "#EPOCHS = 15\n",
252 | "EPOCHS = 5\n",
253 | "\n",
254 | "# Specify batch size for training and testing\n",
255 | "BATCH_SIZE = 8"
256 | ]
257 | },
258 | {
259 | "cell_type": "markdown",
260 | "metadata": {},
261 | "source": [
262 | "## Training\n",
263 | "\n",
264 | "Below we are going to define the training loop for cross validation,\n",
265 | "for each cross fold, where will be a training data set with 20% held out for validation.\n",
266 | "The training and testing procedure within each cross validation iteration are the same as a standard train/test procedure, only with different data.\n",
267 | "\n",
268 | "First we create a little utility function to help progress reporting."
269 | ]
270 | },
271 | {
272 | "cell_type": "code",
273 | "execution_count": null,
274 | "metadata": {},
275 | "outputs": [],
276 | "source": [
277 | "from ipywidgets import IntProgress\n",
278 | "from IPython.display import display\n",
279 | "def progress_wrapper(iter_in, label='Fold Progress'):\n",
280 | " m = len(iter_in)-1\n",
281 | " pbar = IntProgress(description=label, min=0, max=m)\n",
282 | " display(pbar)\n",
283 | " for idx, item in enumerate(iter_in):\n",
284 | " pbar.value = idx\n",
285 | " yield item"
286 | ]
287 | },
288 | {
289 | "cell_type": "code",
290 | "execution_count": null,
291 | "metadata": {},
292 | "outputs": [],
293 | "source": [
294 | "def train(model, epochs, fname_save=None, xval_fold=None, lr=1e-3, is_inception=False):\n",
295 | " train_data = ImageDataset(\n",
296 | " dataset=DATASET,\n",
297 | " metadata=xval_metadata(xval_fold, test=False),\n",
298 | " transform=transform_pipe\n",
299 | " )\n",
300 | " train_loader = torch.utils.data.DataLoader(\n",
301 | " train_data,\n",
302 | " batch_size=BATCH_SIZE,\n",
303 | " shuffle=True\n",
304 | " )\n",
305 | " test_data = ImageDataset(\n",
306 | " dataset=DATASET,\n",
307 | " metadata=xval_metadata(xval_fold, test=True),\n",
308 | " transform=transform_pipe\n",
309 | " )\n",
310 | " test_loader = torch.utils.data.DataLoader(\n",
311 | " test_data,\n",
312 | " batch_size=BATCH_SIZE\n",
313 | " )\n",
314 | "\n",
315 | " # optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)\n",
316 | " optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
317 | " criterion = torch.nn.CrossEntropyLoss()\n",
318 | "\n",
319 | " for i in range(epochs):\n",
320 | " model.train()\n",
321 | " ###################\n",
322 | " # TRAINING #\n",
323 | " ###################\n",
324 | " samples = 0\n",
325 | " loss_sum = 0\n",
326 | " true_sum = 0\n",
327 | " for batch in progress_wrapper(train_loader, 'train'):\n",
328 | " X = batch[\"image\"]\n",
329 | " labels = batch[\"label\"]\n",
330 | " optimizer.zero_grad()\n",
331 | " with torch.set_grad_enabled(True):\n",
332 | " if is_inception:\n",
333 | " # ref: https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html\n",
334 | " y, aux_outputs = model(X)\n",
335 | " loss1 = criterion(y, labels)\n",
336 | " loss2 = criterion(aux_outputs, labels)\n",
337 | " loss = loss1 + 0.4*loss2\n",
338 | " else:\n",
339 | " y = model(X)\n",
340 | " loss = criterion(y, labels)\n",
341 | " loss.backward()\n",
342 | " optimizer.step()\n",
343 | " loss_sum += loss.item() * X.shape[0]\n",
344 | " samples += X.shape[0]\n",
345 | " num_true = torch.sum(torch.argmax(y, 1) == labels)\n",
346 | " true_sum += num_true\n",
347 | "\n",
348 | " epoch_acc = float(true_sum) / float(samples)\n",
349 | " epoch_loss = float(loss_sum) / float(samples)\n",
350 | " print(\"epoch: {} - {} loss: {}, acc: {}\".format(i + 1, \"train\", epoch_loss, epoch_acc))\n",
351 | "\n",
352 | " ###################\n",
353 | " # TESTING #\n",
354 | " ###################\n",
355 | " model.eval()\n",
356 | " samples = 0\n",
357 | " loss_sum = 0\n",
358 | " true_sum = 0\n",
359 | " for batch in test_loader:\n",
360 | " X = batch[\"image\"]\n",
361 | " labels = batch[\"label\"]\n",
362 | " with torch.set_grad_enabled(False):\n",
363 | " y = model(X)\n",
364 | " loss = criterion(y, labels)\n",
365 | " loss_sum += loss.item() * X.shape[0]\n",
366 | " samples += X.shape[0]\n",
367 | " num_true = torch.sum(torch.argmax(y, 1) == labels)\n",
368 | " true_sum += num_true\n",
369 | "\n",
370 | " epoch_acc = float(true_sum) / float(samples)\n",
371 | " epoch_loss = float(loss_sum) / float(samples)\n",
372 | " print(\"epoch: {} - {} loss: {}, acc: {}\".format(i + 1, \"test\", epoch_loss, epoch_acc))\n",
373 | "\n",
374 | " ###################\n",
375 | " # SAVING #\n",
376 | " ###################\n",
377 | " if fname_save:\n",
378 | " torch.save(model.state_dict(), fname_save)"
379 | ]
380 | },
381 | {
382 | "cell_type": "markdown",
383 | "metadata": {},
384 | "source": [
385 | "## Train - Test Methodolgies\n",
386 | "\n",
387 | "### Cross-Validation Example\n",
388 | "\n",
389 | "You should see training progress such as this (except the result are misleading because we have extremely downsampled to accelerate training in order to take a screenshot):\n",
390 | "\n",
391 | ""
392 | ]
393 | },
394 | {
395 | "cell_type": "code",
396 | "execution_count": null,
397 | "metadata": {},
398 | "outputs": [],
399 | "source": [
400 | "for fold in range(5):\n",
401 | " print(\"Fold\", fold)\n",
402 | " model = build_res50()\n",
403 | " train(model, EPOCHS, xval_fold=fold, lr=1e-3, is_inception=False)"
404 | ]
405 | },
406 | {
407 | "cell_type": "markdown",
408 | "metadata": {},
409 | "source": [
410 | "# Congrats, Part 2 complete!\n",
411 | "\n",
412 | "**Appendix: extra source codes `src/`** \n",
413 | "\n",
414 | "* fusion_csv.sql - exporting cross validation metrics and inference data for the next data fusion stage\n",
415 | "* metadata-resisc45.py - creating metadata for the RESISC45 dataset\n",
416 | "* metadata-ucm.py - creating metadata for the UCMerced dataset\n",
417 | "* tables.sql - metadata schemas definition\n",
418 | "* train-resisc45.py - training ResNet50, Inception v3 and DenseNet on the RESISC45 dataset with cross validation\n",
419 | "* train-ucm.py - training ResNet50, Inception v3 and DenseNet on the UCMerced dataset with cross validation"
420 | ]
421 | }
422 | ],
423 | "metadata": {
424 | "kernelspec": {
425 | "display_name": "Python 3",
426 | "language": "python",
427 | "name": "python3"
428 | },
429 | "language_info": {
430 | "codemirror_mode": {
431 | "name": "ipython",
432 | "version": 3
433 | },
434 | "file_extension": ".py",
435 | "mimetype": "text/x-python",
436 | "name": "python",
437 | "nbconvert_exporter": "python",
438 | "pygments_lexer": "ipython3",
439 | "version": "3.7.3"
440 | }
441 | },
442 | "nbformat": 4,
443 | "nbformat_minor": 2
444 | }
445 |
--------------------------------------------------------------------------------
/notebooks/images/DCNN_Fusion_Framework_Vert.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/notebooks/images/DCNN_Fusion_Framework_Vert.png
--------------------------------------------------------------------------------
/notebooks/images/RESISC45_p1.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/notebooks/images/RESISC45_p1.PNG
--------------------------------------------------------------------------------
/notebooks/images/RESISC45_p2.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/notebooks/images/RESISC45_p2.PNG
--------------------------------------------------------------------------------
/notebooks/images/convfilters_giphy.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/notebooks/images/convfilters_giphy.gif
--------------------------------------------------------------------------------
/notebooks/images/densenet_towarddatascience.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/notebooks/images/densenet_towarddatascience.png
--------------------------------------------------------------------------------
/notebooks/images/inceptionv3_kaggle.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/notebooks/images/inceptionv3_kaggle.png
--------------------------------------------------------------------------------
/notebooks/images/resnet50_kaggle.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/notebooks/images/resnet50_kaggle.png
--------------------------------------------------------------------------------
/notebooks/images/training.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/scottgs/FuzzyFusion_DeepLearning_Tutorial/413e21559f3e2a356339653396903202a9a8b76d/notebooks/images/training.png
--------------------------------------------------------------------------------
/src/ChI.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import itertools
3 | from cvxopt import solvers, matrix
4 |
5 |
6 | class ChoquetIntegral:
7 |
8 | def __init__(self):
9 | """Instantiation of a ChoquetIntegral.
10 |
11 | This sets up the ChI. It doesn't take any input parameters
12 | because you may want to use pass your own values in(as opposed
13 | to learning from data). To instatiate, use
14 | chi = ChoquetIntegral.ChoquetIntegral()
15 | """
16 | self.trainSamples, self.trainLabels = [], []
17 | self.testSamples, self.testLabels = [], []
18 | self.N, self.numberConstraints, self.M = 0, 0, 0
19 | self.g = 0
20 | self.fm = []
21 | self.type = []
22 |
23 | def train_chi_quad(self, x1, l1):
24 | """
25 | This trains this instance of your ChoquetIntegral w.r.t x1 and l1.
26 |
27 | :param x1: These are the training samples of size N x M(inputs x number of samples)
28 | :param l1: These are the training labels of size 1 x M(label per sample)
29 |
30 | """
31 | self.type = 'quad'
32 | self.trainSamples = x1
33 | self.trainLabels = l1
34 | self.N = self.trainSamples.shape[0]
35 | self.M = self.trainSamples.shape[1]
36 | print("Number Inputs : ", self.N, "; Number Samples : ", self.M)
37 | self.fm = self.produce_lattice()
38 |
39 | def chi_quad(self, x2):
40 | """
41 | This will produce an output for this instance of the ChI
42 |
43 | This will use the learned(or specified) Choquet integral to
44 | produce an output w.r.t. to the new input.
45 |
46 | :param x2: testing sample
47 | :return: output of the choquet integral.
48 | """
49 | if self.type == 'quad':
50 | n = len(x2)
51 | pi_i = np.argsort(x2)[::-1][:n] + 1
52 | ch = x2[pi_i[0] - 1] * (self.fm[str(pi_i[:1])])
53 | for i in range(1, n):
54 | latt_pti = np.sort(pi_i[:i + 1])
55 | latt_ptimin1 = np.sort(pi_i[:i])
56 | ch = ch + x2[pi_i[i] - 1] * (self.fm[str(latt_pti)] - self.fm[str(latt_ptimin1)])
57 | return ch
58 | else:
59 | print("If using sugeno measure, you need to use chi_sugeno.")
60 |
61 | def produce_lattice(self):
62 | """
63 | This method builds is where the lattice(or FM variables) will be learned.
64 |
65 | The FM values can be found via a quadratic program, which is used here
66 | after setting up constraint matrices. Refer to papers for complete overview.
67 |
68 | :return: Lattice, the learned FM variables.
69 | """
70 |
71 | fm_len = 2 ** self.N - 1 # nc
72 | E = np.zeros((fm_len, fm_len)) # D
73 | L = np.zeros(fm_len) # f
74 | index_keys = self.get_keys_index()
75 | for i in range(0, self.M): # it's going through one sample at a time.
76 | l = self.trainLabels[i] # this is the labels
77 | fm_coeff = self.get_fm_class_img_coeff(index_keys, self.trainSamples[:, i], fm_len) # this is Hdiff
78 | # print(fm_coeff)
79 | L = L + (-2) * l * fm_coeff
80 | E = E + np.matmul(fm_coeff.reshape((fm_len, 1)), fm_coeff.reshape((1, fm_len)))
81 |
82 | G, h, A, b = self.build_constraint_matrices(index_keys, fm_len)
83 | sol = solvers.qp(matrix(2 * E, tc='d'), matrix(L.T, tc='d'), matrix(G, tc='d'), matrix(h, tc='d'),
84 | matrix(A, tc='d'), matrix(b, tc='d'))
85 |
86 | g = sol['x']
87 | Lattice = {}
88 | for key in index_keys.keys():
89 | Lattice[key] = g[index_keys[key]]
90 | return Lattice
91 |
92 | def build_constraint_matrices(self, index_keys, fm_len):
93 | """
94 | This method builds the necessary constraint matrices.
95 |
96 |
97 |
98 | :param index_keys: map to reference lattice components
99 | :param fm_len: length of the fuzzy measure
100 | :return: the constraint matrices
101 | """
102 |
103 | vls = np.arange(1, self.N + 1)
104 | line = np.zeros(fm_len)
105 | G = line
106 | line[index_keys[str(np.array([1]))]] = -1.
107 | h = np.array([0])
108 | for i in range(2, self.N + 1):
109 | line = np.zeros(fm_len)
110 | line[index_keys[str(np.array([i]))]] = -1.
111 | G = np.vstack((G, line))
112 | h = np.vstack((h, np.array([0])))
113 | for i in range(2, self.N + 1):
114 | parent = np.array(list(itertools.combinations(vls, i)))
115 | for latt_pt in parent:
116 | for j in range(len(latt_pt) - 1, len(latt_pt)):
117 | children = np.array(list(itertools.combinations(latt_pt, j)))
118 | for latt_ch in children:
119 | line = np.zeros(fm_len)
120 | line[index_keys[str(latt_ch)]] = 1.
121 | line[index_keys[str(latt_pt)]] = -1.
122 | G = np.vstack((G, line))
123 | h = np.vstack((h, np.array([0])))
124 |
125 | line = np.zeros(fm_len)
126 | line[index_keys[str(vls)]] = 1.
127 | G = np.vstack((G, line))
128 | h = np.vstack((h, np.array([1])))
129 |
130 | # equality constraints
131 | A = np.zeros((1, fm_len))
132 | A[0, -1] = 1
133 | b = np.array([1]);
134 |
135 | return G, h, A, b
136 |
137 | def get_fm_class_img_coeff(self, Lattice, h, fm_len): # Lattice is FM_name_and_index, h is the samples, fm_len
138 | """
139 | This creates a FM map with the name as the key and the index as the value
140 |
141 | :param Lattice: dictionary with FM
142 | :param h: sample
143 | :param fm_len: fm length
144 | :return: the fm_coeff
145 | """
146 |
147 | n = len(h) # len(h) is the number of the samples
148 | fm_coeff = np.zeros(fm_len)
149 | pi_i = np.argsort(h)[::-1][:n] + 1
150 | for i in range(1, n):
151 | fm_coeff[Lattice[str(np.sort(pi_i[:i]))]] = h[pi_i[i - 1] - 1] - h[pi_i[i] - 1]
152 | fm_coeff[Lattice[str(np.sort(pi_i[:n]))]] = h[pi_i[n - 1] - 1]
153 | np.matmul(fm_coeff, np.transpose(fm_coeff))
154 | return fm_coeff
155 |
156 | def get_keys_index(self):
157 | """
158 | Sets up a dictionary for referencing FM.
159 |
160 | :return: The keys to the dictionary
161 | """
162 |
163 | vls = np.arange(1, self.N + 1)
164 | count = 0
165 | Lattice = {}
166 | for i in range(0, self.N):
167 | Lattice[str(np.array([vls[i]]))] = count
168 | count = count + 1
169 | for i in range(2, self.N + 1):
170 | A = np.array(list(itertools.combinations(vls, i)))
171 | for latt_pt in A:
172 | Lattice[str(latt_pt)] = count
173 | count = count + 1
174 | return Lattice
175 |
176 | def train_chi_sugeno(self, densities):
177 | """
178 | Trains a fm w.r.t the sugeno measure.
179 | The learned FM is in binary encoded format.
180 | """
181 | self.type = 'sugeno'
182 | self.N = densities.shape[0] # get the number of densities
183 | densities = densities + np.finfo(float).eps # condition the densities
184 |
185 | coefficients = np.zeros(self.N) # hold onto the polynomial
186 |
187 | coefficients[self.N - 1] = np.sum(densities) # compute the first term
188 |
189 | ##################################################################
190 | # compute the rest of the terms.
191 | ##################################################################
192 | for i in range(self.N, 1, -1):
193 | combos = list(itertools.combinations(range(0, self.N), i)) # generate the combinations
194 | number_of_combinations = combos.__len__() # How many elements were generated?
195 | elements = combos[0].__len__()
196 | coefficients[(self.N - 1) - i + 1] = 0
197 |
198 | ##################################################################
199 | # For each coefficient, compute its product, summing them all up
200 | ##################################################################
201 | for j in range(0, number_of_combinations):
202 | tmp = 1 # tmp used for product calculation
203 |
204 | ##################################################################
205 | # For each combo, compute the product
206 | ##################################################################
207 | for k in range(0, elements):
208 | tmp = tmp * densities[combos[j][k]]
209 |
210 | coefficients[(self.N - 1) - i + 1] = coefficients[(self.N - 1) - i + 1] + tmp
211 |
212 | coefficients[(self.N - 1)] = coefficients[(self.N - 1)] - 1
213 |
214 | s_lambda = np.max((np.roots(coefficients))) # calculate lambda
215 |
216 | ##################################################################
217 | # Now calculate the full measure
218 | ##################################################################
219 |
220 | n_vars = 2 ** self.N - 1 # get the number of FM variables
221 |
222 | fm_binary = np.zeros(n_vars) # where to store the measure values
223 |
224 | ##################################################################
225 | # First, get the densities
226 | ##################################################################
227 |
228 | for i in range(0, self.N):
229 | fm_binary[int(2 ** i) - 1] = densities[i]
230 |
231 | ##################################################################
232 | # Next, get the rest of the values
233 | ##################################################################
234 | for i in range(2, self.N):
235 | combos = list(itertools.combinations(range(0, self.N), i))
236 | number_of_combinations = combos.__len__()
237 |
238 | for j in range(0, number_of_combinations):
239 | this_set_1 = combos[j][0] # get the first element of this combination
240 | this_set_2 = combos[j][1:] # get the rest of the elements
241 |
242 | # fetch the calculated values thus far
243 | val_1 = np.cumsum(2 ** (this_set_1))
244 | term_1 = fm_binary[val_1[-1] - 1]
245 | val_2 = np.cumsum(2 ** np.asarray(this_set_2))
246 | term_2 = fm_binary[val_2[-1] - 1]
247 | val_3 = np.cumsum(2 ** np.asarray(combos[j]))
248 |
249 | # fill in the rest of the values
250 | fm_binary[val_3[-1] - 1] = term_1 + term_2 + s_lambda * term_1 * term_2
251 |
252 | fm_binary[-1] = 1 # last term is a 1
253 |
254 | self.fm = fm_binary
255 |
256 | def chi_sugeno(self, x2):
257 | """
258 | The chi_sugeno compute the output with respect to the learned, sugeno fm.
259 | This relies on the FM being in the binary encoding format.
260 |
261 |
262 | """
263 |
264 | if self.type == 'sugeno':
265 | n = len(x2)
266 | sorted_inds = np.argsort(x2)[::-1][:n] + 1 # sorted indices
267 | sorted_vals = np.append(x2[sorted_inds - 1], 0)
268 | inds = np.cumsum(2 ** (sorted_inds - 1))
269 | res = 0
270 | val = 1
271 | for i in inds:
272 | res = res + self.fm[i - 1] * (sorted_vals[val - 1] - sorted_vals[val])
273 | val += 1
274 | else:
275 | print("If using sugeno measure, you need to use chi_sugeno.")
276 |
277 | return res
278 |
--------------------------------------------------------------------------------
/src/ChoquetIntegral.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import itertools
3 | from cvxopt import solvers, matrix
4 |
5 |
6 | class ChoquetIntegral:
7 |
8 | def __init__(self):
9 | """Instantiation of a ChoquetIntegral.
10 |
11 | This sets up the ChI. It doesn't take any input parameters
12 | because you may want to use pass your own values in(as opposed
13 | to learning from data). To instatiate, use
14 | chi = ChoquetIntegral.ChoquetIntegral()
15 | """
16 | self.trainSamples, self.trainLabels = [], []
17 | self.testSamples, self.testLabels = [], []
18 | self.N, self.numberConstraints, self.M = 0, 0, 0
19 | self.g = 0
20 | self.fm = []
21 | self.type = []
22 |
23 | def train_chi_quad(self, x1, l1):
24 | """
25 | This trains this instance of your ChoquetIntegral w.r.t x1 and l1.
26 |
27 | :param x1: These are the training samples of size N x M(inputs x number of samples)
28 | :param l1: These are the training labels of size 1 x M(label per sample)
29 |
30 | """
31 | self.type = 'quad'
32 | self.trainSamples = x1
33 | self.trainLabels = l1
34 | self.N = self.trainSamples.shape[0]
35 | self.M = self.trainSamples.shape[1]
36 | print("Number Inputs : ", self.N, "; Number Samples : ", self.M)
37 | self.fm = self.produce_lattice()
38 |
39 | def chi_quad(self, x2):
40 | """
41 | This will produce an output for this instance of the ChI
42 |
43 | This will use the learned(or specified) Choquet integral to
44 | produce an output w.r.t. to the new input.
45 |
46 | :param x2: testing sample
47 | :return: output of the choquet integral.
48 | """
49 | if self.type == 'quad':
50 | n = len(x2)
51 | pi_i = np.argsort(x2)[::-1][:n] + 1
52 | ch = x2[pi_i[0] - 1] * (self.fm[str(pi_i[:1])])
53 | for i in range(1, n):
54 | latt_pti = np.sort(pi_i[:i + 1])
55 | latt_ptimin1 = np.sort(pi_i[:i])
56 | ch = ch + x2[pi_i[i] - 1] * (self.fm[str(latt_pti)] - self.fm[str(latt_ptimin1)])
57 | return ch
58 | print("If using sugeno measure, you need to use chi_sugeno.")
59 |
60 | def produce_lattice(self):
61 | """
62 | This method builds is where the lattice(or FM variables) will be learned.
63 |
64 | The FM values can be found via a quadratic program, which is used here
65 | after setting up constraint matrices. Refer to papers for complete overview.
66 |
67 | :return: Lattice, the learned FM variables.
68 | """
69 |
70 |
71 | fm_len = 2 ** self.N - 1 # nc
72 | E = np.zeros((fm_len, fm_len)) # D
73 | L = np.zeros(fm_len) # f
74 | index_keys = self.get_keys_index()
75 | for i in range(0, self.M): # it's going through one sample at a time.
76 | l = self.trainLabels[i] # this is the labels
77 | fm_coeff = self.get_fm_class_img_coeff(index_keys, self.trainSamples[:, i], fm_len) # this is Hdiff
78 | # print(fm_coeff)
79 | L = L + (-2) * l * fm_coeff
80 | E = E + np.matmul(fm_coeff.reshape((fm_len, 1)), fm_coeff.reshape((1, fm_len)))
81 |
82 | G, h, A, b = self.build_constraint_matrices(index_keys, fm_len)
83 | sol = solvers.qp(matrix(2 * E, tc='d'), matrix(L.T, tc='d'), matrix(G, tc='d'), matrix(h, tc='d'), matrix(A, tc='d'), matrix(b, tc='d'))
84 |
85 | g = sol['x']
86 | Lattice = {}
87 | for key in index_keys.keys():
88 | Lattice[key] = g[index_keys[key]]
89 | return Lattice
90 |
91 | def build_constraint_matrices(self, index_keys, fm_len):
92 | """
93 | This method builds the necessary constraint matrices.
94 |
95 |
96 |
97 | :param index_keys: map to reference lattice components
98 | :param fm_len: length of the fuzzy measure
99 | :return: the constraint matrices
100 | """
101 |
102 |
103 | vls = np.arange(1, self.N + 1)
104 | line = np.zeros(fm_len)
105 | G = line
106 | line[index_keys[str(np.array([1]))]] = -1.
107 | h = np.array([0])
108 | for i in range(2, self.N + 1):
109 | line = np.zeros(fm_len)
110 | line[index_keys[str(np.array([i]))]] = -1.
111 | G = np.vstack((G, line))
112 | h = np.vstack((h, np.array([0])))
113 | for i in range(2, self.N + 1):
114 | parent = np.array(list(itertools.combinations(vls, i)))
115 | for latt_pt in parent:
116 | for j in range(len(latt_pt) - 1, len(latt_pt)):
117 | children = np.array(list(itertools.combinations(latt_pt, j)))
118 | for latt_ch in children:
119 | line = np.zeros(fm_len)
120 | line[index_keys[str(latt_ch)]] = 1.
121 | line[index_keys[str(latt_pt)]] = -1.
122 | G = np.vstack((G, line))
123 | h = np.vstack((h, np.array([0])))
124 |
125 | line = np.zeros(fm_len)
126 | line[index_keys[str(vls)]] = 1.
127 | G = np.vstack((G, line))
128 | h = np.vstack((h, np.array([1])))
129 |
130 | # equality constraints
131 | A = np.zeros((1,fm_len))
132 | A[0,-1] = 1
133 | b = np.array([1]);
134 |
135 | return G, h, A, b
136 |
137 | def get_fm_class_img_coeff(self, Lattice, h, fm_len): # Lattice is FM_name_and_index, h is the samples, fm_len
138 | """
139 | This creates a FM map with the name as the key and the index as the value
140 |
141 | :param Lattice: dictionary with FM
142 | :param h: sample
143 | :param fm_len: fm length
144 | :return: the fm_coeff
145 | """
146 |
147 | n = len(h) # len(h) is the number of the samples
148 | fm_coeff = np.zeros(fm_len)
149 | pi_i = np.argsort(h)[::-1][:n] + 1
150 | for i in range(1, n):
151 | fm_coeff[Lattice[str(np.sort(pi_i[:i]))]] = h[pi_i[i - 1] - 1] - h[pi_i[i] - 1]
152 | fm_coeff[Lattice[str(np.sort(pi_i[:n]))]] = h[pi_i[n - 1] - 1]
153 | np.matmul(fm_coeff, np.transpose(fm_coeff))
154 | return fm_coeff
155 |
156 | def get_keys_index(self):
157 | """
158 | Sets up a dictionary for referencing FM.
159 |
160 | :return: The keys to the dictionary
161 | """
162 |
163 |
164 | vls = np.arange(1, self.N + 1)
165 | count = 0
166 | Lattice = {}
167 | for i in range(0, self.N):
168 | Lattice[str(np.array([vls[i]]))] = count
169 | count = count + 1
170 | for i in range(2, self.N + 1):
171 | A = np.array(list(itertools.combinations(vls, i)))
172 | for latt_pt in A:
173 | Lattice[str(latt_pt)] = count
174 | count = count + 1
175 | return Lattice
176 |
177 | def train_chi_sugeno(self, densities):
178 | """
179 | Trains a fm w.r.t the sugeno measure.
180 | The learned FM is in binary encoded format.
181 | """
182 | self.type = 'sugeno'
183 | self.N = densities.shape[0] # get the number of densities
184 | densities = densities + np.finfo(float).eps # condition the densities
185 |
186 | coefficients = np.zeros(self.N) # hold onto the polynomial
187 |
188 | coefficients[self.N - 1] = np.sum(densities) # compute the first term
189 |
190 | ##################################################################
191 | # compute the rest of the terms.
192 | ##################################################################
193 | for i in range(self.N, 1, -1):
194 | combos = list(itertools.combinations(range(0, self.N), i)) # generate the combinations
195 | number_of_combinations = combos.__len__() # How many elements were generated?
196 | elements = combos[0].__len__()
197 | coefficients[(self.N - 1) - i + 1] = 0
198 |
199 | ##################################################################
200 | # For each coefficient, compute its product, summing them all up
201 | ##################################################################
202 | for j in range(0, number_of_combinations):
203 | tmp = 1 # tmp used for product calculation
204 |
205 | ##################################################################
206 | # For each combo, compute the product
207 | ##################################################################
208 | for k in range(0, elements):
209 | tmp = tmp * densities[combos[j][k]]
210 |
211 | coefficients[(self.N - 1) - i + 1] = coefficients[(self.N - 1) - i + 1] + tmp
212 |
213 | coefficients[(self.N - 1)] = coefficients[(self.N - 1)] - 1
214 |
215 | s_lambda = np.max((np.roots(coefficients))) # calculate lambda
216 |
217 | ##################################################################
218 | # Now calculate the full measure
219 | ##################################################################
220 |
221 | n_vars = 2 ** self.N - 1 # get the number of FM variables
222 |
223 | fm_binary = np.zeros(n_vars) # where to store the measure values
224 |
225 | ##################################################################
226 | # First, get the densities
227 | ##################################################################
228 |
229 | for i in range(0, self.N):
230 | fm_binary[int(2 ** i) - 1] = densities[i]
231 |
232 | ##################################################################
233 | # Next, get the rest of the values
234 | ##################################################################
235 | for i in range(2, self.N):
236 | combos = list(itertools.combinations(range(0, self.N), i))
237 | number_of_combinations = combos.__len__()
238 |
239 | for j in range(0, number_of_combinations):
240 | this_set_1 = combos[j][0] # get the first element of this combination
241 | this_set_2 = combos[j][1:] # get the rest of the elements
242 |
243 | # fetch the calculated values thus far
244 | val_1 = np.cumsum(2 ** (this_set_1))
245 | term_1 = fm_binary[val_1[-1] - 1]
246 | val_2 = np.cumsum(2 ** np.asarray(this_set_2))
247 | term_2 = fm_binary[val_2[-1] - 1]
248 | val_3 = np.cumsum(2 ** np.asarray(combos[j]))
249 |
250 | # fill in the rest of the values
251 | fm_binary[val_3[-1] - 1] = term_1 + term_2 + s_lambda * term_1 * term_2
252 |
253 |
254 | fm_binary[-1] = 1 # last term is a 1
255 |
256 |
257 | self.fm = fm_binary
258 |
259 | def chi_sugeno(self, x2):
260 | """
261 | The chi_sugeno compute the output with respect to the learned, sugeno fm.
262 | This relies on the FM being in the binary encoding format.
263 |
264 |
265 | """
266 |
267 | if self.type == 'sugeno':
268 | n = len(x2)
269 | sorted_inds = np.argsort(x2)[::-1][:n] + 1 # sorted indices
270 | sorted_vals = np.append(x2[sorted_inds-1], 0)
271 | inds = np.cumsum(2**(sorted_inds-1))
272 | res = 0
273 | val = 1
274 | for i in inds:
275 | res = res+self.fm[i-1] * (sorted_vals[val-1] - sorted_vals[val])
276 | val += 1
277 |
278 |
279 | print("If using sugeno measure, you need to use chi_sugeno.")
280 |
281 | return res
282 |
283 |
284 | if __name__ == "__main__":
285 | chi_quad = ChoquetIntegral()
286 |
287 | train_data = np.random.randn(3,3)
288 | label_data = np.random.randn(3,1)
289 | chi_quad.train_chi_quad(train_data, label_data)
290 | print('Quadratic Program Fuzzy Measure : ', chi_quad.fm)
291 |
292 | chi_sugeno = ChoquetIntegral()
293 | densities = np.array([.1, .2, .3])
294 | chi_sugeno.train_chi_sugeno(densities)
295 | chi_sugeno.fm = ([.3, .3, .6, .3, .6, .6, 1])
296 |
297 | print('Sugeno-Lambda Fuzzy Measure : ', chi_sugeno.fm)
298 | print(chi_sugeno.chi_sugeno(np.array([.4, .5, .3])))
299 |
300 |
301 |
302 |
303 |
304 |
--------------------------------------------------------------------------------
/src/compute_sugeno.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | from itertools import combinations
3 |
4 |
5 | def compute_sugeno(densities):
6 |
7 | N = densities.shape[0] # get the number of densities
8 |
9 | densities = densities + np.finfo(float).eps # condition the densities
10 |
11 | coefficients = np.zeros(N) # hold onto the polynomial
12 |
13 | coefficients[N-1] = np.sum(densities) # compute the first term
14 |
15 | ##################################################################
16 | # compute the rest of the terms.
17 | ##################################################################
18 | for i in range(N, 1, -1):
19 | combos = list(combinations(range(0, N), i)) # generate the combinations
20 | number_of_combinations = combos.__len__() # How many elements were generated?
21 | elements = combos[0].__len__()
22 | coefficients[(N - 1) - i + 1] = 0
23 |
24 | ##################################################################
25 | # For each coefficient, compute its product, summing them all up
26 | ##################################################################
27 | for j in range(0, number_of_combinations):
28 | tmp = 1 # tmp used for product calculation
29 |
30 | ##################################################################
31 | # For each combo, compute the product
32 | ##################################################################
33 | for k in range(0, elements):
34 | tmp = tmp * densities[combos[j][k]]
35 |
36 | coefficients[(N-1)-i+1] = coefficients[(N-1)-i+1] + tmp
37 |
38 | coefficients[(N-1)] = coefficients[(N-1)] - 1
39 |
40 | s_lambda = np.max((np.roots(coefficients))) # calculate lambda
41 |
42 | ##################################################################
43 | # Now calculate the full measure
44 | ##################################################################
45 |
46 | n_vars = 2**N - 1 # get the number of FM variables
47 |
48 |
49 | fm_binary = np.zeros(n_vars) # where to store the measure values
50 |
51 | ##################################################################
52 | # First, get the densities
53 | ##################################################################
54 |
55 | for i in range(0, N):
56 | fm_binary[int(2**i)-1] = densities[i]
57 |
58 | ##################################################################
59 | # Next, get the rest of the values
60 | ##################################################################
61 | for i in range(2, N):
62 | combos = list(combinations(range(0, N), i))
63 | number_of_combinations = combos.__len__()
64 |
65 | for j in range(0, number_of_combinations):
66 | this_set_1 = combos[j][0] # get the first element of this combination
67 |
68 | this_set_2 = combos[j][1:] # get the rest of the elements
69 |
70 | # fetch the calculated values thus far
71 |
72 | val_1, term_1 = np.cumsum(2**(this_set_1)), fm_binary[val_1[-1]-1]
73 | val_2, term_2 = np.cumsum(2**np.asarray(this_set_2)), fm_binary[val_2[-1]-1]
74 | val_3 = np.cumsum(2**np.asarray(combos[j]))
75 |
76 | # fill in the rest of the values
77 | fm_binary[val_3[-1]-1] = term_1 + term_2 + s_lambda * term_1 * term_2
78 |
79 | fm_binary[-1] = 1 # last term is a 1
80 |
81 | return fm_binary
82 |
83 |
84 |
85 | if __name__ == '__main__':
86 | FM = compute_sugeno(np.array([.1, .25, .5, .25, .8]))
--------------------------------------------------------------------------------
/src/fuse_nets_data_driven.py:
--------------------------------------------------------------------------------
1 | import ChI
2 | import numpy as np
3 | import csv
4 | from os import listdir
5 | from os.path import isfile, join
6 | def soft_max(samples):
7 | # Normalizes each sample w.r.t. each sample (soft max)
8 | for i in range(0, samples.shape[0]): # for each sample
9 | for j in range(0, samples.shape[1]): # for each network
10 | samples[i,j,:] = np.exp(samples[i, j, :]) / sum(np.exp(samples[i, j, :]))
11 |
12 | return [samples]
13 |
14 | # Specify Network Path
15 | network_path = 'C:\\Users\\MINDFUL\\PycharmProjects\\fusion\\data\\'
16 |
17 | # set up variables
18 | image_names = []
19 |
20 | # This is how we'll get all of the different inputs. Each csv file needs to follow the correct format
21 | csv_files = [f for f in listdir(network_path) if isfile(join(network_path, f))]
22 | num_nets = csv_files.__len__()
23 |
24 | # Create dictionary to store samples
25 | data = dict.fromkeys(csv_files)
26 |
27 | # Read in all of the csv data into a dictionary
28 | first_net = 1 # this is a flag.
29 | for file in csv_files:
30 | csv_data = []
31 | data_info = np.genfromtxt((network_path + '\\' + file), usecols=(1, 2, 3, 4), dtype="|U", delimiter=',')
32 | confidence_vectors = np.genfromtxt((network_path + '\\' + file), delimiter=';')
33 |
34 | for line in range(0, data_info.__len__()):
35 | if first_net:
36 | image_names.append(data_info[line,0])
37 | csv_data.append(np.hstack((data_info[line,:-1], data_info[line,3].partition(';')[0], confidence_vectors[line, 1:])))
38 |
39 | first_net = 0
40 | data[file] = csv_data
41 |
42 | num_samples = data_info.__len__()
43 |
44 | # Now I need to build the samples and their corresponding labels
45 | # There will be the same number of ChI's as there are classes(L0
46 | # One ChI per class, so each one sample will turn into L samples
47 |
48 |
49 | # How many classes are there?
50 | num_classes = confidence_vectors.shape[1]# Assuming the first 4 columns are 'image y_true confidence y_pred'
51 |
52 | samples = np.zeros([num_samples, csv_files.__len__(), num_classes])
53 | label = np.zeros([num_samples, num_classes])
54 |
55 | print('--Formatting Samples--')
56 |
57 | ############################################################################################
58 | # Get samples for each ChI - samples is a [num_classes x num_inputs x num_samples]
59 | ############################################################################################
60 | for i, img in enumerate(image_names): # For each image
61 | for n, net in enumerate(csv_files): # For each net
62 | temp = data[net]
63 | for l in temp: #
64 | if img == l[0]:
65 | for c in range(0, num_classes):
66 | samples[i, n, c] = l[c+3]
67 | if str(c) == l[1]:
68 | label[i, c] = 1
69 | else:
70 | label[i, c] = 0
71 |
72 | ##############################################
73 | # Start Training and Testing
74 | ##############################################
75 | print('--Starting Training and Testing')
76 |
77 | train_samples = samples.copy()
78 | train_labels = label
79 |
80 | test_samples = samples.copy()
81 | test_labels = label
82 | ##############################################
83 | # Normalize Training Data & Testing Data
84 | ##############################################
85 | [train_samples] = soft_max(train_samples)
86 | [test_samples] = soft_max(test_samples)
87 |
88 | print('--Training--')
89 | ##############################################
90 | # Train the ChI(s)
91 | ##############################################
92 |
93 | CHIs = []
94 | for j in range(0, num_classes):
95 | CHIs.append(ChI.ChoquetIntegral())
96 |
97 | for j, chi in enumerate(CHIs):
98 | print('Class ChI {}'.format(j))
99 | train_data = np.transpose(train_samples[:,:,j])
100 | label_data = train_labels[:, j]
101 | tr = chi.train_chi_quad(train_data, label_data)
102 | print(chi.fm)
103 |
104 |
105 | print('--Testing--')
106 | ##############################################
107 | # Test the ChI(s)
108 | ##############################################
109 | exper_out, known_out = [], []
110 | dec = []
111 | for j in range(0, test_samples.shape[0]): # for each data point
112 | out = []
113 | for k, chi in enumerate(CHIs): # for each ChI
114 | test_sample = np.transpose(test_samples[j, :, k])
115 | test_label = np.argmax(test_labels[j, :])
116 | out.append(chi.chi_quad(test_sample))
117 | out = np.asarray(out)
118 | exper_out.append(np.argmax(out))
119 | known_out.append(test_label)
120 | for i in range(0, exper_out.__len__()):
121 | if exper_out[i] == known_out[i]:
122 | dec.append(1)
123 | else:
124 | dec.append(0)
125 | with open('Results{}.csv'.format(i), 'w', newline='') as csvfile:
126 | spamwriter = csv.writer(csvfile, delimiter=',')
127 | for k in range(0, exper_out.__len__()):
128 | spamwriter.writerow([exper_out[k], known_out[k]])
129 |
130 | acc = np.sum(dec) / dec.__len__()
131 |
132 |
--------------------------------------------------------------------------------
/src/fuse_nets_sugeno.py:
--------------------------------------------------------------------------------
1 | import ChI
2 | import numpy as np
3 | import csv
4 | from os import listdir
5 | from os.path import isfile, join
6 |
7 |
8 | def soft_max(samples):
9 | # Normalizes each sample w.r.t. each sample (soft max)
10 | for i in range(0, samples.shape[0]): # for each sample
11 | for j in range(0, samples.shape[1]): # for each network
12 | samples[i,j,:] = np.exp(samples[i, j, :]) / sum(np.exp(samples[i, j, :]))
13 |
14 | return [samples]
15 |
16 |
17 |
18 | # Specify Network Path
19 | network_path = 'C:\\Users\\MINDFUL\\PycharmProjects\\fusion\\data\\'
20 | cross_val_path = 'C:\\Users\\MINDFUL\\PycharmProjects\\fusion\\data\\cross_val\\'
21 |
22 | # set up variables
23 | image_names = []
24 |
25 | # This is how we'll get all of the different inputs. Each csv file needs to follow the correct format
26 | csv_files = [f for f in listdir(network_path) if isfile(join(network_path, f))]
27 | cross_val = [f for f in listdir(cross_val_path) if isfile(join(cross_val_path, f))]
28 | num_nets = csv_files.__len__()
29 |
30 | # Create dictionary to store samples
31 | data = dict.fromkeys(csv_files)
32 |
33 | # Read in all of the csv data into a dictionary
34 | densities = []
35 | for file in cross_val:
36 | csv_data = []
37 | data_info = np.genfromtxt((cross_val_path + '\\' + file), usecols=(1), skip_header=True,dtype="f", delimiter=',')
38 | densities.append(np.mean(data_info))
39 |
40 | densities = np.asarray(densities)
41 | first_net = 1 # this is a flag.
42 | for file in csv_files:
43 | csv_data = []
44 | data_info = np.genfromtxt((network_path + '\\' + file), usecols=(1, 2, 3, 4), dtype="|U", delimiter=',')
45 | confidence_vectors = np.genfromtxt((network_path + '\\' + file), delimiter=';')
46 |
47 | for line in range(0, data_info.__len__()):
48 | if first_net:
49 | image_names.append(data_info[line,0])
50 | csv_data.append(np.hstack((data_info[line,:-1], data_info[line,3].partition(';')[0], confidence_vectors[line, 1:])))
51 |
52 | first_net = 0
53 | data[file] = csv_data
54 |
55 |
56 | # How many classes are there?
57 | num_classes = confidence_vectors.shape[1]# Assuming the first 4 columns are 'image y_true confidence y_pred'
58 |
59 | # Now I need to build the samples and their corresponding labels
60 | # There will be the same number of ChI's as there are classes(L0
61 | # One ChI per class, so each one sample will turn into L samples
62 | num_samples = data_info.__len__()
63 | samples = np.zeros([num_samples, csv_files.__len__(), num_classes])
64 | label = np.zeros([num_samples, num_classes])
65 |
66 | print('--Formatting Samples--')
67 |
68 | ############################################################################################
69 | # Get samples for each ChI - samples is a [num_classes x num_inputs x num_samples]
70 | ############################################################################################
71 | for i, img in enumerate(image_names): # For each image
72 | for n, net in enumerate(csv_files): # For each net
73 | temp = data[net]
74 | for l in temp: #
75 | if img == l[0]:
76 | for c in range(0, num_classes):
77 | samples[i, n, c] = l[c+3]
78 | if str(c) == l[1]:
79 | label[i, c] = 1
80 | else:
81 | label[i, c] = 0
82 |
83 |
84 | ##############################################
85 | # Start Training and Testing
86 | ##############################################
87 | print('--Starting Training and Testing')
88 |
89 |
90 | train_samples = samples.copy()
91 | train_labels = label
92 |
93 | test_samples = samples.copy()
94 | test_labels = label
95 | ##############################################
96 | # Normalize Training Data & Testing Data
97 | ##############################################
98 | [train_samples] = soft_max(train_samples)
99 | [test_samples] = soft_max(test_samples)
100 |
101 | print('--Training--')
102 | ##############################################
103 | # Train the ChI(s)
104 | ##############################################
105 |
106 | CHIs = []
107 | for j in range(0, num_classes):
108 | CHIs.append(ChI.ChoquetIntegral())
109 |
110 | for j, chi in enumerate(CHIs):
111 | print('Class ChI {}'.format(j))
112 | tr = chi.train_chi_sugeno(densities)
113 | print(chi.fm)
114 |
115 |
116 | print('--Testing--')
117 | ##############################################
118 | # Test the ChI(s)
119 | ##############################################
120 | exper_out, known_out = [], []
121 | dec = []
122 | for j in range(0, test_samples.shape[0]): # for each data point
123 | out = []
124 | for k, chi in enumerate(CHIs): # for each ChI
125 | test_sample = np.transpose(test_samples[j, :, k])
126 | test_label = np.argmax(test_labels[j, :])
127 | out.append(chi.chi_sugeno(test_sample))
128 | out = np.asarray(out)
129 | exper_out.append(np.argmax(out))
130 | known_out.append(test_label)
131 | for i in range(0, exper_out.__len__()):
132 | if exper_out[i] == known_out[i]:
133 | dec.append(1)
134 | else:
135 | dec.append(0)
136 | with open('Results{}.csv'.format(i), 'w', newline='') as csvfile:
137 | spamwriter = csv.writer(csvfile, delimiter=',')
138 | for k in range(0, exper_out.__len__()):
139 | spamwriter.writerow([exper_out[k], known_out[k]])
140 |
141 | acc = np.sum(dec) / dec.__len__()
142 | print(acc)
143 |
144 |
--------------------------------------------------------------------------------
/src/fusion_csv.sql:
--------------------------------------------------------------------------------
1 | -- Usage: sudo -u postgres psql ucmerced -f fusion_csv.sql
2 |
3 | -- export cross validation accuracies (per fold per class)
4 | \copy (SELECT fold_name, subset_id AS class_label, metrics -> 'acc' AS accuracy FROM xval_metrics WHERE model_id LIKE 'res50.%' AND partition_name = '5fold' ORDER BY fold_name, subset_id) TO '/tmp/xval_cls.res50.5fold.csv' WITH CSV HEADER DELIMITER ',';
5 |
6 | -- export average cross validation accuracies (per fold)
7 | \copy (SELECT fold_name, AVG((metrics -> 'acc')::float) AS accuracy FROM xval_metrics WHERE model_id LIKE 'res50.%' AND partition_name = '5fold' GROUP BY fold_name ORDER BY fold_name) TO '/tmp/xval.res50.5fold.csv' WITH CSV HEADER DELIMITER ',';
8 |
9 | -- export inference data by model
10 | \copy (SELECT partitions -> '5fold' AS fold_name, image, class_label AS y_true, (prediction -> 'y_pred')::int AS y_pred, prediction -> 'y_vec' as y_vec FROM inference y LEFT JOIN frame x ON y.frame_id = x.id WHERE model_id LIKE 'res50.5fold::%' ORDER BY partitions -> '5fold') TO '/tmp/pred.res50.5fold.csv' WITH CSV;
11 |
12 | -- export inference data by model & fold
13 | \copy (SELECT image, class_label AS y_true, (prediction -> 'y_pred')::int AS y_pred, prediction -> 'y_vec' as y_vec FROM inference y LEFT JOIN frame x ON y.frame_id = x.id WHERE model_id = 'res50.5fold::A' ORDER BY partitions -> '5fold') TO '/tmp/pred.res50.5fold::A.csv' WITH CSV;
14 | \copy (SELECT image, class_label AS y_true, (prediction -> 'y_pred')::int AS y_pred, prediction -> 'y_vec' as y_vec FROM inference y LEFT JOIN frame x ON y.frame_id = x.id WHERE model_id = 'res50.5fold::B' ORDER BY partitions -> '5fold') TO '/tmp/pred.res50.5fold::B.csv' WITH CSV;
15 | \copy (SELECT image, class_label AS y_true, (prediction -> 'y_pred')::int AS y_pred, prediction -> 'y_vec' as y_vec FROM inference y LEFT JOIN frame x ON y.frame_id = x.id WHERE model_id = 'res50.5fold::C' ORDER BY partitions -> '5fold') TO '/tmp/pred.res50.5fold::C.csv' WITH CSV;
16 | \copy (SELECT image, class_label AS y_true, (prediction -> 'y_pred')::int AS y_pred, prediction -> 'y_vec' as y_vec FROM inference y LEFT JOIN frame x ON y.frame_id = x.id WHERE model_id = 'res50.5fold::D' ORDER BY partitions -> '5fold') TO '/tmp/pred.res50.5fold::D.csv' WITH CSV;
17 | \copy (SELECT image, class_label AS y_true, (prediction -> 'y_pred')::int AS y_pred, prediction -> 'y_vec' as y_vec FROM inference y LEFT JOIN frame x ON y.frame_id = x.id WHERE model_id = 'res50.5fold::E' ORDER BY partitions -> '5fold') TO '/tmp/pred.res50.5fold::E.csv' WITH CSV;
18 |
--------------------------------------------------------------------------------
/src/metadata-resisc45.py:
--------------------------------------------------------------------------------
1 | import itertools
2 | import pandas as pd
3 | import psycopg2, psycopg2.extras
4 | import h5py
5 | from pathlib import Path
6 | from imeta import *
7 | DATASET = '/mnt/datasets/IEEE/RESISC45'
8 | DATABASE = 'dbname=resisc45 user=postgres'
9 |
10 | def build_index(conn):
11 | print("Initializing metadata...")
12 | cur = conn.cursor()
13 | dataset = Path(DATASET)
14 | class_dirs = [x for x in dataset.iterdir() if x.is_dir()]
15 | labels = sorted(map(lambda pth: str(pth.relative_to(dataset)), class_dirs))
16 | sql = 'INSERT INTO frame (image, class_label, partitions) values %s'
17 | values = []
18 | for class_dir in class_dirs:
19 | for fname_image in class_dir.iterdir():
20 | label = str(class_dir.relative_to(dataset))
21 | label_int = labels.index(label)
22 | partitions = ','.join(['"active"=>1'] + ['"{}"=>1'.format(label), '"part"=>"{}"'.format(label)]) # TODO + ['"traintest"=>"train"'])
23 | values.append(( str(fname_image.relative_to(dataset)), label_int, partitions))
24 | psycopg2.extras.execute_values(cur, sql, values)
25 | sql = """INSERT INTO partitions (name, labels, created_on, summary, impl_version) values
26 | ('active', array['active'], now(), true, 1),
27 | ('part', array{}, now(), false, 1);""".format(labels)
28 | cur.execute(sql)
29 | conn.commit()
30 | add_xval(cur, 5, '5fold')
31 | conn.commit()
32 | add_xval(cur, 10, '10fold')
33 | conn.commit()
34 | psycopg2.extras.execute_values(cur, 'INSERT INTO class_label (label_id, label_name) values %s', list(enumerate(labels)))
35 | conn.commit()
36 |
37 | def use_downsample_v1(frac_downsample):
38 | with psycopg2.connect(DATABASE) as conn:
39 | cur = conn.cursor()
40 | cur.execute("""UPDATE frame SET partitions = delete(partitions, 'active');""")
41 | cur.execute("""UPDATE frame SET partitions = partitions || '"active"=>1' WHERE random()<%f;""", (frac_downsample, ))
42 | conn.commit()
43 |
44 | def use_downsample_v2(name, frac_downsample):
45 | with psycopg2.connect(DATABASE) as conn:
46 | cur = conn.cursor()
47 | cur.execute("""UPDATE frame SET partitions = delete(partitions, %s);""", (name, ))
48 | cur.execute("""UPDATE frame SET partitions = partitions || '"%s"=>1' WHERE random()<%f;""" % (name, frac_downsample))
49 | cur.execute("""INSERT INTO partitions (name, labels, created_on, summary, impl_version) values ('%s', array['%s'], now(), true, 1)
50 | ON CONFLICT DO NOTHING;""" % (name, name))
51 | conn.commit()
52 |
53 | if __name__ == "__main__":
54 | metadata(DATABASE, build_index)
55 | # use_downsample_v2('down10', 0.1)
56 | # use_downsample_v2('down20', 0.2)
57 |
--------------------------------------------------------------------------------
/src/metadata-ucm.py:
--------------------------------------------------------------------------------
1 | import itertools
2 | import pandas as pd
3 | import psycopg2, psycopg2.extras
4 | import h5py
5 | from pathlib import Path
6 | from imeta import *
7 | DATASET = '/mnt/datasets/IEEE/UCMerced'
8 | DATABASE = 'dbname=ucmerced user=postgres'
9 |
10 | def build_index(conn):
11 | print("Initializing metadata...")
12 | cur = conn.cursor()
13 | dataset = Path(DATASET)
14 | class_dirs = [x for x in dataset.iterdir() if x.is_dir()]
15 | labels = sorted(map(lambda pth: str(pth.relative_to(dataset)), class_dirs))
16 | sql = 'INSERT INTO frame (image, class_label, partitions) values %s'
17 | values = []
18 | for class_dir in class_dirs:
19 | for fname_image in class_dir.iterdir():
20 | label = str(class_dir.relative_to(dataset))
21 | label_int = labels.index(label)
22 | partitions = ','.join(['"active"=>1'] + ['"{}"=>1'.format(label), '"part"=>"{}"'.format(label)]) # TODO + ['"traintest"=>"train"'])
23 | values.append(( str(fname_image.relative_to(dataset)), label_int, partitions))
24 | psycopg2.extras.execute_values(cur, sql, values)
25 | sql = """INSERT INTO partitions (name, labels, created_on, summary, impl_version) values
26 | ('active', array['active'], now(), true, 1),
27 | ('part', array{}, now(), false, 1);""".format(labels)
28 | cur.execute(sql)
29 | conn.commit()
30 | add_xval(cur, 5, '5fold')
31 | conn.commit()
32 | add_xval(cur, 10, '10fold')
33 | conn.commit()
34 | psycopg2.extras.execute_values(cur, 'INSERT INTO class_label (label_id, label_name) values %s', list(enumerate(labels)))
35 | conn.commit()
36 |
37 | def use_downsample_v1(frac_downsample):
38 | with psycopg2.connect(DATABASE) as conn:
39 | cur = conn.cursor()
40 | cur.execute("""UPDATE frame SET partitions = delete(partitions, 'active');""")
41 | cur.execute("""UPDATE frame SET partitions = partitions || '"active"=>1' WHERE random()<%f;""", (frac_downsample, ))
42 | conn.commit()
43 |
44 | def use_downsample_v2(name, frac_downsample):
45 | with psycopg2.connect(DATABASE) as conn:
46 | cur = conn.cursor()
47 | cur.execute("""UPDATE frame SET partitions = delete(partitions, %s);""", (name, ))
48 | cur.execute("""UPDATE frame SET partitions = partitions || '"%s"=>1' WHERE random()<%f;""" % (name, frac_downsample))
49 | cur.execute("""INSERT INTO partitions (name, labels, created_on, summary, impl_version) values ('%s', array['%s'], now(), true, 1)
50 | ON CONFLICT DO NOTHING;""" % (name, name))
51 | conn.commit()
52 |
53 | if __name__ == "__main__":
54 | metadata(DATABASE, build_index)
55 | # use_downsample_v2('down10', 0.1)
56 | # use_downsample_v2('down20', 0.2)
57 |
--------------------------------------------------------------------------------
/src/tables.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE frame (
2 | id SERIAL PRIMARY KEY,
3 | image TEXT NOT NULL, -- image file name: use ";" to separate multiple associated images
4 | class_label INT, -- class label: may use binary one hot encoding in multi-label case
5 | partitions hstore -- denormalized partitioning information
6 | );
7 |
8 | CREATE INDEX idx_frame ON frame USING GIN (partitions);
9 |
10 | CREATE TABLE partitions (
11 | id SERIAL PRIMARY KEY,
12 | name VARCHAR(32) UNIQUE NOT NULL, -- name of the partition namespace
13 | labels VARCHAR[] NOT NULL, -- names of the partitions that exist in the namespace
14 | created_on TIMESTAMP NOT NULL,
15 | reference TEXT, -- reference to the associated data shard
16 | summary BOOLEAN NOT NULL, -- required to show in the summary
17 | impl_version INT -- data structure impl: v1: hstore tagging {"part" => 1}, v2: hstore enum {"namespace"=>"part"}
18 | );
19 |
20 | CREATE TABLE class_label (
21 | id SERIAL PRIMARY KEY, -- do not use a foreign key to reference this optional table because it could be NULL or N/A
22 | label_namespace TEXT, -- namespace for multiple datasets
23 | label_name TEXT NOT NULL, -- name of the label
24 | label_name_long TEXT, -- long name of the label
25 | label_id INT NOT NULL -- label id
26 | );
27 |
28 | CREATE TABLE xval_metrics (
29 | model_id TEXT NOT NULL, -- a unique name/id of the model e.g. res50.5fold::A
30 | subset_id INT NOT NULL, -- can be class_label if the concept applies
31 | partition_name TEXT NOT NULL, -- aka namespace of the fold name
32 | fold_name TEXT NOT NULL,
33 | metrics hstore, -- denormalized metrics
34 | PRIMARY KEY (model_id, subset_id, partition_name, fold_name)
35 | );
36 |
37 | CREATE TABLE inference (
38 | frame_id INT REFERENCES frame(id) NOT NULL, -- a reference to the frame
39 | model_id TEXT NOT NULL, -- a unique name/id of the model e.g. res50.5fold::A
40 | prediction hstore NOT NULL, -- all outputs from model inference
41 | PRIMARY KEY (frame_id, model_id)
42 | );
43 |
--------------------------------------------------------------------------------
/src/train-resisc45.py:
--------------------------------------------------------------------------------
1 | # pylint: disable=no-member
2 | import os, sys
3 | from pathlib import Path
4 | import torch
5 | import torchvision
6 | import torchvision.models as models
7 | import psycopg2
8 | from PIL import Image
9 | from tqdm import tqdm
10 |
11 | # Imagery location
12 | DATASET = '/mnt/datasets/IEEE/RESISC45'
13 |
14 | # Output space
15 | OUTPUT = '/mnt/datasets/IEEE/RESISC45'
16 |
17 | # Metadata location
18 | METADATA = 'dbname=resisc45 user=postgres'
19 |
20 | # Choose a base partition below
21 | BASE = {0:'active', 1:'down20', 2:'down10', 3:'down1'}[0]
22 |
23 | # Specify number of classes
24 | NUM_CLASS = 45
25 |
26 | # Specify number of epochs per x-fold
27 | EPOCHS = 15
28 |
29 | # Specify batch size for training and testing
30 | BATCH_SIZE = 32
31 |
32 | class ImageDataset(torch.utils.data.Dataset):
33 | def __init__(self, dataset, metadata, transform):
34 | self.dataset = Path(dataset)
35 | self.transform = transform
36 | self.metadata = metadata
37 |
38 | def __getitem__(self, idx):
39 | frame_id, image, class_label = self.metadata[idx]
40 | fname = str(self.dataset / image)
41 | return dict(frame_id=frame_id, image=self.transform(Image.open(fname).convert("RGB")), label=class_label)
42 |
43 | def __len__(self):
44 | return len(self.metadata)
45 |
46 | transform_pipe = torchvision.transforms.Compose([
47 | torchvision.transforms.Resize(
48 | size=(299, 299)
49 | ),
50 | torchvision.transforms.ToTensor(),
51 | torchvision.transforms.Normalize(
52 | mean=[0.485, 0.456, 0.406],
53 | std=[0.229, 0.224, 0.225]
54 | )
55 | ])
56 |
57 | def xval_metadata(partition, test=False):
58 | print('Loading (xval)', partition, '=>', ('test' if test else 'train'))
59 | with psycopg2.connect(METADATA) as conn:
60 | with conn.cursor() as cur:
61 | partition, fold = partition.split('::')
62 | if test:
63 | cur.execute("select id, image, class_label from frame where partitions -> %s = %s AND partitions ? %s", (partition, fold, BASE))
64 | else:
65 | cur.execute("select id, image, class_label from frame where partitions ? %s AND partitions -> %s != %s AND partitions ? %s", (partition, partition, fold, BASE))
66 | return cur.fetchall()
67 |
68 | def train(model, epochs, fname_save, xval_fold=None, lr=1e-3, is_inception=False):
69 | train_data = ImageDataset(
70 | dataset=DATASET,
71 | metadata=xval_metadata(xval_fold or 'traintest', test=False),
72 | transform=transform_pipe
73 | )
74 | train_loader = torch.utils.data.DataLoader(
75 | train_data,
76 | batch_size=BATCH_SIZE,
77 | shuffle=True
78 | )
79 | test_data = ImageDataset(
80 | dataset=DATASET,
81 | metadata=xval_metadata(xval_fold or 'traintest', test=True),
82 | transform=transform_pipe
83 | )
84 | test_loader = torch.utils.data.DataLoader(
85 | test_data,
86 | batch_size=BATCH_SIZE
87 | )
88 |
89 | # optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
90 | optimizer = torch.optim.Adam(model.parameters(), lr=lr)
91 | criterion = torch.nn.CrossEntropyLoss().cuda()
92 |
93 | for i in range(epochs):
94 | model.train()
95 | ###################
96 | # TRAINING #
97 | ###################
98 | samples = 0
99 | loss_sum = 0
100 | true_sum = 0
101 | for batch in tqdm(train_loader):
102 | X = batch["image"].cuda()
103 | labels = batch["label"].cuda()
104 | optimizer.zero_grad()
105 | with torch.set_grad_enabled(True):
106 | if is_inception:
107 | # ref: https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html
108 | y, aux_outputs = model(X)
109 | loss1 = criterion(y, labels)
110 | loss2 = criterion(aux_outputs, labels)
111 | loss = loss1 + 0.4*loss2
112 | else:
113 | y = model(X)
114 | loss = criterion(y, labels)
115 | loss.backward()
116 | optimizer.step()
117 | loss_sum += loss.item() * X.shape[0]
118 | samples += X.shape[0]
119 | num_true = torch.sum(torch.argmax(y, 1) == labels)
120 | true_sum += num_true
121 |
122 | epoch_acc = float(true_sum) / float(samples)
123 | epoch_loss = float(loss_sum) / float(samples)
124 | print("epoch: {} - {} loss: {}, acc: {}".format(i + 1, "train", epoch_loss, epoch_acc))
125 |
126 | ###################
127 | # TESTING #
128 | ###################
129 | model.eval()
130 | samples = 0
131 | loss_sum = 0
132 | true_sum = 0
133 | for batch in test_loader:
134 | X = batch["image"].cuda()
135 | labels = batch["label"].cuda()
136 | with torch.set_grad_enabled(False):
137 | y = model(X)
138 | loss = criterion(y, labels)
139 | loss_sum += loss.item() * X.shape[0]
140 | samples += X.shape[0]
141 | num_true = torch.sum(torch.argmax(y, 1) == labels)
142 | true_sum += num_true
143 |
144 | epoch_acc = float(true_sum) / float(samples)
145 | epoch_loss = float(loss_sum) / float(samples)
146 | print("epoch: {} - {} loss: {}, acc: {}".format(i + 1, "test", epoch_loss, epoch_acc))
147 |
148 | ###################
149 | # SAVING #
150 | ###################
151 | torch.save(model.state_dict(), fname_save)
152 |
153 | def get_labels():
154 | with psycopg2.connect(METADATA) as conn:
155 | with conn.cursor() as cur:
156 | cur.execute("select label_name, label_id from class_label")
157 | return dict(cur.fetchall())
158 |
159 | def xval_per_class_metadata(partition, class_label, test=False):
160 | print(f'Loading (xval.{class_label})', partition, '=>', ('test' if test else 'train'))
161 | with psycopg2.connect(METADATA) as conn:
162 | with conn.cursor() as cur:
163 | partition, fold = partition.split('::')
164 | if test:
165 | cur.execute("select id, image, class_label from frame where class_label = %s AND partitions -> %s = %s AND partitions ? %s", (class_label, partition, fold, BASE))
166 | else:
167 | cur.execute("select id, image, class_label from frame where class_label = %s AND partitions ? %s AND partitions -> %s != %s AND partitions ? %s", (class_label, partition, partition, fold, BASE))
168 | return cur.fetchall()
169 |
170 | def save_inference(cur, model_name, xval_fold, frame_ids, y_pred, y_vec):
171 | import psycopg2.extras
172 | model_id = '{}.{}'.format(model_name, xval_fold)
173 | sql = 'INSERT INTO inference (frame_id, model_id, prediction) values %s ON CONFLICT DO NOTHING;'
174 | values = []
175 | for frame_id, i_pred, i_vec in zip(frame_ids, y_pred, y_vec):
176 | values.append((frame_id, model_id, ','.join(['"y_pred"=>{}'.format(i_pred), '"y_vec"=>"{}"'.format(';'.join(map(str, i_vec)))])))
177 | psycopg2.extras.execute_values(cur, sql, values)
178 |
179 | def save_metric(cur, model_name, xval_fold, class_label, acc):
180 | model_id = '{}.{}'.format(model_name, xval_fold)
181 | sql = 'INSERT INTO xval_metrics (model_id, subset_id, partition_name, fold_name, metrics) values (%s, %s, %s, %s, %s) ON CONFLICT DO NOTHING;'
182 | partition, fold = xval_fold.split('::')
183 | cur.execute(sql, (model_id, class_label, partition, fold, ('"acc"=>%f'%acc)))
184 |
185 | def xval_evaluate(model, model_name, xval_fold):
186 | """ Evaluate a trained model and generate per-class metrics """
187 | conn = psycopg2.connect(METADATA)
188 | cur = conn.cursor()
189 | for class_label in get_labels().values():
190 | test_data = ImageDataset(dataset=DATASET, metadata=xval_per_class_metadata(xval_fold, class_label, test=True), transform=transform_pipe)
191 | test_loader = torch.utils.data.DataLoader(test_data, batch_size=BATCH_SIZE)
192 | model.eval()
193 | samples = 0
194 | true_sum = 0
195 | for batch in test_loader:
196 | frame_ids = batch["frame_id"]
197 | X = batch["image"].cuda()
198 | labels = batch["label"].cuda()
199 | with torch.set_grad_enabled(False):
200 | y = model(X)
201 | samples += X.shape[0]
202 | y_pred = torch.argmax(y, 1)
203 | num_true = torch.sum(y_pred == labels)
204 | true_sum += num_true
205 | save_inference(cur, model_name, xval_fold, frame_ids.tolist(), y_pred.cpu().tolist(), y.cpu().tolist())
206 | epoch_acc = float(true_sum) / float(samples)
207 | print("{}.{} acc: {}".format(xval_fold, class_label, epoch_acc))
208 | save_metric(cur, model_name, xval_fold, class_label, epoch_acc)
209 | conn.commit()
210 | cur.close()
211 | conn.close()
212 |
213 | def xval_iter(partition, n):
214 | for i in range(n):
215 | yield '{partition}::{fold}'.format(partition=partition, fold=chr(i+65))
216 |
217 | def build_res50(fname_load=None):
218 | model = models.resnet50(pretrained=(fname_load is None))
219 | model.fc = torch.nn.Linear(in_features=2048, out_features=NUM_CLASS)
220 | if fname_load:
221 | print('Loading pretrained weights.')
222 | model.load_state_dict(torch.load(fname_load))
223 | return model.cuda()
224 |
225 | def build_inception(fname_load=None):
226 | model = models.inception_v3(pretrained=(fname_load is None))
227 | model.fc = torch.nn.Linear(in_features=2048, out_features=NUM_CLASS)
228 | if fname_load:
229 | print('Loading pretrained weights.')
230 | model.load_state_dict(torch.load(fname_load))
231 | return model.cuda()
232 |
233 | def build_densenet(fname_load=None):
234 | model = models.densenet121(pretrained=(fname_load is None))
235 | model.classifier = torch.nn.Linear(in_features=1024, out_features=NUM_CLASS)
236 | if fname_load:
237 | print('Loading pretrained weights.')
238 | model.load_state_dict(torch.load(fname_load))
239 | return model.cuda()
240 |
241 | if __name__ == "__main__":
242 | # # Cross validation: Res50
243 | # for fold in xval_iter('5fold', 5):
244 | # model = build_res50()
245 | # train(model, EPOCHS, f'{OUTPUT}/res50.{fold}.pth', xval_fold=fold)
246 |
247 | # # Generate evaluation metrics
248 | # for fold in xval_iter('5fold', 5):
249 | # model = build_res50(f'{OUTPUT}/res50.{fold}.pth')
250 | # xval_evaluate(model, model_name='res50', xval_fold=fold)
251 |
252 | # # Cross validation: Inception
253 | # for fold in xval_iter('5fold', 5):
254 | # model = build_inception()
255 | # train(model, EPOCHS, f'{OUTPUT}/inception.{fold}.pth', xval_fold=fold, is_inception=True)
256 |
257 | # # Generate evaluation metrics
258 | # for fold in xval_iter('5fold', 5):
259 | # model = build_inception(f'{OUTPUT}/inception.{fold}.pth')
260 | # xval_evaluate(model, model_name='inception', xval_fold=fold)
261 |
262 | # Cross validation: DenseNet
263 | for fold in xval_iter('5fold', 5):
264 | model = build_densenet()
265 | train(model, EPOCHS, f'{OUTPUT}/densenet.{fold}.pth', xval_fold=fold)
266 |
267 | # Generate evaluation metrics
268 | for fold in xval_iter('5fold', 5):
269 | model = build_densenet(f'{OUTPUT}/densenet.{fold}.pth')
270 | xval_evaluate(model, model_name='densenet', xval_fold=fold)
271 |
--------------------------------------------------------------------------------
/src/train-ucm.py:
--------------------------------------------------------------------------------
1 | # pylint: disable=no-member
2 | import os, sys
3 | from pathlib import Path
4 | import torch
5 | import torchvision
6 | import torchvision.models as models
7 | import psycopg2
8 | from PIL import Image
9 | from tqdm import tqdm
10 |
11 | # Imagery location
12 | DATASET = '/mnt/datasets/IEEE/UCMerced'
13 |
14 | # Output space
15 | OUTPUT = '/mnt/datasets/IEEE/UCMerced'
16 |
17 | # Metadata location
18 | METADATA = 'dbname=ucmerced user=postgres'
19 |
20 | # Choose a base partition below
21 | BASE = {0:'active', 1:'down20', 2:'down10', 3:'down1'}[0]
22 |
23 | # Specify number of classes
24 | NUM_CLASS = 21
25 |
26 | # Specify number of epochs per x-fold
27 | EPOCHS = 30
28 |
29 | # Specify batch size for training and testing
30 | BATCH_SIZE = 32
31 |
32 | class ImageDataset(torch.utils.data.Dataset):
33 | def __init__(self, dataset, metadata, transform):
34 | self.dataset = Path(dataset)
35 | self.transform = transform
36 | self.metadata = metadata
37 |
38 | def __getitem__(self, idx):
39 | frame_id, image, class_label = self.metadata[idx]
40 | fname = str(self.dataset / image)
41 | return dict(frame_id=frame_id, image=self.transform(Image.open(fname).convert("RGB")), label=class_label)
42 |
43 | def __len__(self):
44 | return len(self.metadata)
45 |
46 | transform_pipe = torchvision.transforms.Compose([
47 | torchvision.transforms.Resize(
48 | size=(299, 299)
49 | ),
50 | torchvision.transforms.ToTensor(),
51 | torchvision.transforms.Normalize(
52 | mean=[0.485, 0.456, 0.406],
53 | std=[0.229, 0.224, 0.225]
54 | )
55 | ])
56 |
57 | def xval_metadata(partition, test=False):
58 | print('Loading (xval)', partition, '=>', ('test' if test else 'train'))
59 | with psycopg2.connect(METADATA) as conn:
60 | with conn.cursor() as cur:
61 | partition, fold = partition.split('::')
62 | if test:
63 | cur.execute("select id, image, class_label from frame where partitions -> %s = %s AND partitions ? %s", (partition, fold, BASE))
64 | else:
65 | cur.execute("select id, image, class_label from frame where partitions ? %s AND partitions -> %s != %s AND partitions ? %s", (partition, partition, fold, BASE))
66 | return cur.fetchall()
67 |
68 | def train(model, epochs, fname_save, xval_fold=None, lr=1e-3, is_inception=False):
69 | train_data = ImageDataset(
70 | dataset=DATASET,
71 | metadata=xval_metadata(xval_fold or 'traintest', test=False),
72 | transform=transform_pipe
73 | )
74 | train_loader = torch.utils.data.DataLoader(
75 | train_data,
76 | batch_size=BATCH_SIZE,
77 | shuffle=True
78 | )
79 | test_data = ImageDataset(
80 | dataset=DATASET,
81 | metadata=xval_metadata(xval_fold or 'traintest', test=True),
82 | transform=transform_pipe
83 | )
84 | test_loader = torch.utils.data.DataLoader(
85 | test_data,
86 | batch_size=BATCH_SIZE
87 | )
88 |
89 | # optimizer = torch.optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
90 | optimizer = torch.optim.Adam(model.parameters(), lr=lr)
91 | criterion = torch.nn.CrossEntropyLoss().cuda()
92 |
93 | for i in range(epochs):
94 | model.train()
95 | ###################
96 | # TRAINING #
97 | ###################
98 | samples = 0
99 | loss_sum = 0
100 | true_sum = 0
101 | for batch in tqdm(train_loader):
102 | X = batch["image"].cuda()
103 | labels = batch["label"].cuda()
104 | optimizer.zero_grad()
105 | with torch.set_grad_enabled(True):
106 | if is_inception:
107 | # ref: https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html
108 | y, aux_outputs = model(X)
109 | loss1 = criterion(y, labels)
110 | loss2 = criterion(aux_outputs, labels)
111 | loss = loss1 + 0.4*loss2
112 | else:
113 | y = model(X)
114 | loss = criterion(y, labels)
115 | loss.backward()
116 | optimizer.step()
117 | loss_sum += loss.item() * X.shape[0]
118 | samples += X.shape[0]
119 | num_true = torch.sum(torch.argmax(y, 1) == labels)
120 | true_sum += num_true
121 |
122 | epoch_acc = float(true_sum) / float(samples)
123 | epoch_loss = float(loss_sum) / float(samples)
124 | print("epoch: {} - {} loss: {}, acc: {}".format(i + 1, "train", epoch_loss, epoch_acc))
125 |
126 | ###################
127 | # TESTING #
128 | ###################
129 | model.eval()
130 | samples = 0
131 | loss_sum = 0
132 | true_sum = 0
133 | for batch in test_loader:
134 | X = batch["image"].cuda()
135 | labels = batch["label"].cuda()
136 | with torch.set_grad_enabled(False):
137 | y = model(X)
138 | loss = criterion(y, labels)
139 | loss_sum += loss.item() * X.shape[0]
140 | samples += X.shape[0]
141 | num_true = torch.sum(torch.argmax(y, 1) == labels)
142 | true_sum += num_true
143 |
144 | epoch_acc = float(true_sum) / float(samples)
145 | epoch_loss = float(loss_sum) / float(samples)
146 | print("epoch: {} - {} loss: {}, acc: {}".format(i + 1, "test", epoch_loss, epoch_acc))
147 |
148 | ###################
149 | # SAVING #
150 | ###################
151 | torch.save(model.state_dict(), fname_save)
152 |
153 | def get_labels():
154 | with psycopg2.connect(METADATA) as conn:
155 | with conn.cursor() as cur:
156 | cur.execute("select label_name, label_id from class_label")
157 | return dict(cur.fetchall())
158 |
159 | def xval_per_class_metadata(partition, class_label, test=False):
160 | print(f'Loading (xval.{class_label})', partition, '=>', ('test' if test else 'train'))
161 | with psycopg2.connect(METADATA) as conn:
162 | with conn.cursor() as cur:
163 | partition, fold = partition.split('::')
164 | if test:
165 | cur.execute("select id, image, class_label from frame where class_label = %s AND partitions -> %s = %s AND partitions ? %s", (class_label, partition, fold, BASE))
166 | else:
167 | cur.execute("select id, image, class_label from frame where class_label = %s AND partitions ? %s AND partitions -> %s != %s AND partitions ? %s", (class_label, partition, partition, fold, BASE))
168 | return cur.fetchall()
169 |
170 | def save_inference(cur, model_name, xval_fold, frame_ids, y_pred, y_vec):
171 | import psycopg2.extras
172 | model_id = '{}.{}'.format(model_name, xval_fold)
173 | sql = 'INSERT INTO inference (frame_id, model_id, prediction) values %s ON CONFLICT DO NOTHING;'
174 | values = []
175 | for frame_id, i_pred, i_vec in zip(frame_ids, y_pred, y_vec):
176 | values.append((frame_id, model_id, ','.join(['"y_pred"=>{}'.format(i_pred), '"y_vec"=>"{}"'.format(';'.join(map(str, i_vec)))])))
177 | psycopg2.extras.execute_values(cur, sql, values)
178 |
179 | def save_metric(cur, model_name, xval_fold, class_label, acc):
180 | model_id = '{}.{}'.format(model_name, xval_fold)
181 | sql = 'INSERT INTO xval_metrics (model_id, subset_id, partition_name, fold_name, metrics) values (%s, %s, %s, %s, %s) ON CONFLICT DO NOTHING;'
182 | partition, fold = xval_fold.split('::')
183 | cur.execute(sql, (model_id, class_label, partition, fold, ('"acc"=>%f'%acc)))
184 |
185 | def xval_evaluate(model, model_name, xval_fold):
186 | """ Evaluate a trained model and generate per-class metrics """
187 | conn = psycopg2.connect(METADATA)
188 | cur = conn.cursor()
189 | for class_label in get_labels().values():
190 | test_data = ImageDataset(dataset=DATASET, metadata=xval_per_class_metadata(xval_fold, class_label, test=True), transform=transform_pipe)
191 | test_loader = torch.utils.data.DataLoader(test_data, batch_size=BATCH_SIZE)
192 | model.eval()
193 | samples = 0
194 | true_sum = 0
195 | for batch in test_loader:
196 | frame_ids = batch["frame_id"]
197 | X = batch["image"].cuda()
198 | labels = batch["label"].cuda()
199 | with torch.set_grad_enabled(False):
200 | y = model(X)
201 | samples += X.shape[0]
202 | y_pred = torch.argmax(y, 1)
203 | num_true = torch.sum(y_pred == labels)
204 | true_sum += num_true
205 | save_inference(cur, model_name, xval_fold, frame_ids.tolist(), y_pred.cpu().tolist(), y.cpu().tolist())
206 | epoch_acc = float(true_sum) / float(samples)
207 | print("{}.{} acc: {}".format(xval_fold, class_label, epoch_acc))
208 | save_metric(cur, model_name, xval_fold, class_label, epoch_acc)
209 | conn.commit()
210 | cur.close()
211 | conn.close()
212 |
213 | def xval_iter(partition, n):
214 | for i in range(n):
215 | yield '{partition}::{fold}'.format(partition=partition, fold=chr(i+65))
216 |
217 | def build_res50(fname_load=None):
218 | model = models.resnet50(pretrained=(fname_load is None))
219 | model.fc = torch.nn.Linear(in_features=2048, out_features=NUM_CLASS)
220 | if fname_load:
221 | print('Loading pretrained weights.')
222 | model.load_state_dict(torch.load(fname_load))
223 | return model.cuda()
224 |
225 | def build_inception(fname_load=None):
226 | model = models.inception_v3(pretrained=(fname_load is None))
227 | model.fc = torch.nn.Linear(in_features=2048, out_features=NUM_CLASS)
228 | if fname_load:
229 | print('Loading pretrained weights.')
230 | model.load_state_dict(torch.load(fname_load))
231 | return model.cuda()
232 |
233 | def build_densenet(fname_load=None):
234 | model = models.densenet121(pretrained=(fname_load is None))
235 | model.classifier = torch.nn.Linear(in_features=1024, out_features=NUM_CLASS)
236 | if fname_load:
237 | print('Loading pretrained weights.')
238 | model.load_state_dict(torch.load(fname_load))
239 | return model.cuda()
240 |
241 | if __name__ == "__main__":
242 | # # Cross validation: Res50
243 | # for fold in xval_iter('5fold', 5):
244 | # model = build_res50()
245 | # train(model, EPOCHS, f'{OUTPUT}/res50.{fold}.pth', xval_fold=fold)
246 |
247 | # Generate evaluation metrics
248 | for fold in xval_iter('5fold', 5):
249 | model = build_res50(f'{OUTPUT}/res50.{fold}.pth')
250 | xval_evaluate(model, model_name='res50', xval_fold=fold)
251 |
252 | # # Cross validation: Inception
253 | # for fold in xval_iter('5fold', 5):
254 | # model = build_inception()
255 | # train(model, EPOCHS, f'{OUTPUT}/inception.{fold}.pth', xval_fold=fold, is_inception=True)
256 |
257 | # Generate evaluation metrics
258 | for fold in xval_iter('5fold', 5):
259 | model = build_inception(f'{OUTPUT}/inception.{fold}.pth')
260 | xval_evaluate(model, model_name='inception', xval_fold=fold)
261 |
262 | # Cross validation: DenseNet
263 | for fold in xval_iter('5fold', 5):
264 | model = build_densenet()
265 | train(model, EPOCHS, f'{OUTPUT}/densenet.{fold}.pth', xval_fold=fold)
266 |
267 | # Generate evaluation metrics
268 | for fold in xval_iter('5fold', 5):
269 | model = build_densenet(f'{OUTPUT}/densenet.{fold}.pth')
270 | xval_evaluate(model, model_name='densenet', xval_fold=fold)
271 |
--------------------------------------------------------------------------------