├── external_memory_algorithms
    ├── List_Ranking.cpp
    ├── ext_join.cpp
    ├── ext_matrix_multiplication.cpp
    ├── ext_matrix_transposition.cpp
    ├── ext_merge_sort.cpp
    ├── f_Ab.cpp
    └── tree.cpp
├── internal_memory_algorithms
    ├── KnapsackProblem.java
    ├── SegmentTree.java
    ├── bloom_filter.cpp
    ├── cache_miss
    │   ├── cache.pptx
    │   ├── cache_emulator
    │   │   ├── Cache.java
    │   │   └── hard-info.txt
    │   ├── cache_misses.pdf
    │   └── matrix_mult
    │   │   ├── cache
    │   │   ├── compile.sh
    │   │   ├── main.cpp
    │   │   ├── matrix_mult.ods
    │   │   └── run.sh
    ├── fast_transposition.cpp
    ├── hitters.cpp
    ├── matrix_multiplication.cpp
    └── web_graph
    │   ├── GraphGenerator.java
    │   ├── gasnikov-raigorodskii.pdf
    │   └── web-graph.pptx
└── literature
    ├── Exam materials.docx
    ├── ExternalMemory.pdf
    ├── materials.pdf
    ├── matmul.pdf
    └── matrix.pptx


/external_memory_algorithms/List_Ranking.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <algorithm>
  3 | 
  4 | struct Triple_1 {
  5 |     unsigned i;
  6 |     unsigned p_i;
  7 |     unsigned w;
  8 | 
  9 |     bool operator < (Triple_1 t2) {
 10 |         return i < t2.i;
 11 |     }
 12 | };
 13 | 
 14 | struct Triple_2 {
 15 |     unsigned i;
 16 |     unsigned p_i;
 17 |     unsigned w;
 18 | 
 19 |     bool operator < (Triple_2 t2) {
 20 |         return p_i < t2.p_i;
 21 |     }
 22 | };
 23 | 
 24 | struct Triple_3 {
 25 |     unsigned i;
 26 |     unsigned p_i;
 27 |     unsigned w;
 28 | 
 29 |     bool operator < (Triple_3 t2) {
 30 |         return w > t2.w;
 31 |     }
 32 | };
 33 | 
 34 | struct Triple {
 35 |     union {
 36 |         Triple_1 triple1;
 37 |         Triple_2 triple2;
 38 |         Triple_3 triple3;
 39 |     };
 40 | };
 41 | 
 42 | static const unsigned RAM_SIZE = 100080;
 43 | //static const unsigned RAM_SIZE = 3000240;
 44 | static unsigned* memory = new unsigned[RAM_SIZE];
 45 | static unsigned series_sizes[3750000 / RAM_SIZE + 1];
 46 | static unsigned list_size;
 47 | static unsigned root;
 48 | static unsigned* a;
 49 | static unsigned* b;
 50 | static unsigned* c;
 51 | static unsigned block_size;
 52 | static unsigned n_read;
 53 | 
 54 | static FILE *tempfile, *mergefile1, *mergefile2, *sortedfile1, *sortedfile2, *joinedfile;
 55 | 
 56 | unsigned min(const unsigned& a, const unsigned& b) {
 57 |     return a < b ? a : b;
 58 | }
 59 | 
 60 | void open_files() {
 61 |     tempfile = fopen("tempfile.bin", "w+b");
 62 |     mergefile1 = fopen("mergefile1.bin", "w+b");
 63 |     mergefile2 = fopen("mergefile2.bin", "w+b");
 64 |     sortedfile1 = fopen("sortedfile1.bin", "w+b");
 65 |     sortedfile2 = fopen("sortedfile2.bin", "w+b");
 66 |     joinedfile = fopen("joinedfile.bin", "w+b");
 67 | }
 68 | 
 69 | void close_files() {
 70 |     fclose(tempfile);
 71 |     fclose(mergefile1);
 72 |     fclose(mergefile2);
 73 |     fclose(sortedfile1);
 74 |     fclose(sortedfile2);
 75 |     fclose(joinedfile);
 76 | }
 77 | 
 78 | void merge(FILE* from_file, FILE* to_file, const unsigned &SERIES_NUMBER, const unsigned &sort_type) {
 79 |     block_size = RAM_SIZE / 4;
 80 | 
 81 |     a = memory;
 82 |     b = memory + block_size;
 83 |     c = memory + 2 * block_size;
 84 | 
 85 |     Triple *p1;
 86 |     Triple *p2;
 87 |     Triple *p3;
 88 |     bool cmp;
 89 |     unsigned copy_size;
 90 | 
 91 |     unsigned total_read1, total_read2;
 92 |     unsigned series1_end, series2_end;
 93 |     unsigned a_size, b_size, c_size;
 94 |     unsigned i, j, k;
 95 | 
 96 |     c_size = 2 * block_size;
 97 | 
 98 |     fseek(to_file, 0, SEEK_SET);
 99 | 
100 |     total_read1 = 0, total_read2 = 0;
101 |     for (unsigned series_i = 0; series_i < SERIES_NUMBER; series_i += 2) {
102 |         total_read1 = total_read2;
103 |         series1_end = total_read1 + 3 * series_sizes[series_i];
104 | 
105 |         //if next series does not exist, just copy last to file
106 |         if (series_i + 1 == SERIES_NUMBER) {
107 |             fseek(from_file, 4 * total_read1, SEEK_SET);
108 |             for (; total_read1 < 3*list_size; total_read1 += n_read) {
109 |                 n_read = (unsigned) fread(memory, 4, RAM_SIZE, from_file);
110 |                 fwrite(memory, 4, n_read, to_file);
111 |             }
112 |             break;
113 |         }
114 | 
115 |         total_read2 = series1_end;
116 |         series2_end = total_read2 + 3 * series_sizes[series_i + 1];
117 | 
118 |         i = j = a_size = b_size = block_size;
119 |         k = 0;
120 |         while (total_read1 != series1_end || total_read2 != series2_end || k != 0) {
121 |             if (a_size != 0 && i == a_size) {
122 |                 if (total_read1 < series1_end) {
123 |                     a_size = min(block_size, series1_end - total_read1);
124 |                     fseek(from_file, 4 * total_read1, SEEK_SET);
125 |                     fread(a, 4, a_size, from_file);
126 |                     total_read1 += a_size;
127 |                     i = 0;
128 |                 }
129 |                 else {
130 |                     i = a_size = 0;
131 |                 }
132 |             }
133 |             if (b_size !=0 && j == b_size) {
134 |                 if (total_read2 < series2_end) {
135 |                     b_size = min(block_size, series2_end - total_read2);
136 |                     fseek(from_file, 4 * total_read2, SEEK_SET);
137 |                     fread(b, 4, b_size, from_file);
138 |                     total_read2 += b_size;
139 |                     j = 0;
140 |                 }
141 |                 else {
142 |                     j = b_size = 0;
143 |                 }
144 |             }
145 | 
146 |             if (a_size == 0 && b_size == 0) {
147 |                 fwrite(c, 4, k, to_file);
148 |                 break;
149 |             }
150 |             if (k == c_size) {
151 |                 fwrite(c, 4, k, to_file);
152 |                 k = 0;
153 |             }
154 | 
155 |             if (a_size == 0) {
156 |                 copy_size = min(b_size - j, c_size - k);
157 |                 std::copy(b + j, b + j + copy_size, c + k);
158 |                 k += copy_size;
159 |                 j += copy_size;
160 |             }
161 |             else if (b_size == 0) {
162 |                 copy_size = min(a_size - i, c_size - k);
163 |                 std::copy(a + i, a + i + copy_size, c + k);
164 |                 k += copy_size;
165 |                 i += copy_size;
166 |             }
167 |             else {
168 |                 while (i < a_size && j < b_size && k < c_size) {
169 | //            c[k++] = (a[i] <= b[j]) ? a[i++] : b[j++];
170 | 
171 |                     p1 = (Triple *) &a[i];
172 |                     p2 = (Triple *) &b[j];
173 |                     p3 = (Triple *) &c[k];
174 | 
175 |                     if (sort_type == 1) {
176 |                         cmp = (p1->triple1 < p2->triple1);
177 |                     } else if (sort_type == 2) {
178 |                         cmp = (p1->triple2 < p2->triple2);
179 |                     } else if (sort_type == 3) {
180 |                         cmp = (p1->triple3 < p2->triple3);
181 |                     }
182 |                     else {
183 |                         fprintf(stdout, "Undefined sort_type\n");
184 |                         exit(1);
185 |                     }
186 | 
187 |                     if (cmp) {
188 |                         *p3 = *p1;
189 |                         i += 3;
190 |                     } else {
191 |                         *p3 = *p2;
192 |                         j += 3;
193 |                     }
194 | 
195 |                     k += 3;
196 |                 }
197 |             }
198 |         }
199 |     }
200 | }
201 | 
202 | void merge_sort(FILE* ifile, FILE* ofile, const unsigned& sort_type) {
203 |     fseek(ifile, 0, SEEK_SET);
204 | 
205 |     FILE* fileptr = (RAM_SIZE < list_size * 3) ? mergefile1 : ofile;
206 |     fseek(fileptr, 0, SEEK_SET);
207 | 
208 |     //split
209 |     unsigned counter = 0;
210 |     for (unsigned i = 0; i < 3 * list_size; i += n_read) {
211 |         n_read = (unsigned) fread(memory, 4, RAM_SIZE, ifile);
212 | 
213 |         if (sort_type == 1) {
214 |             std::sort((Triple_1*)memory, (Triple_1*)memory + n_read / 3);
215 |         }
216 |         else if (sort_type == 2) {
217 |             std::sort((Triple_2*)memory, (Triple_2*)memory + n_read / 3);
218 |         }
219 |         else if (sort_type == 3) {
220 |             std::sort((Triple_3*)memory, (Triple_3*)memory + n_read / 3);
221 |         }
222 |         else {
223 |             fprintf(stderr, "Undefined sort_type\n");
224 |             exit(1);
225 |         }
226 | 
227 |         fwrite(memory, 4, n_read, fileptr);
228 |         series_sizes[counter] = n_read / 3;
229 |         counter++;
230 |     }
231 | 
232 |     //merge
233 |     unsigned series_size = RAM_SIZE;
234 |     unsigned series_number;
235 |     unsigned iteration = 0;
236 |     while (series_size < 3*list_size) {
237 |         series_number = (3*list_size + series_size - 1) / series_size;
238 | 
239 |         if (series_number > 2) {
240 |             if (iteration & 1) {
241 |                 merge(mergefile2, mergefile1, series_number, sort_type);
242 |             }
243 |             else {
244 |                 merge(mergefile1, mergefile2, series_number, sort_type);
245 |             }
246 |         }
247 |         else {
248 |             if (iteration & 1) {
249 |                 merge(mergefile2, ofile, series_number, sort_type);
250 |             }
251 |             else {
252 |                 merge(mergefile1, ofile, series_number, sort_type);
253 |             }
254 |         }
255 | 
256 |         for (unsigned l = 0; l < series_number; l += 2) {
257 |             if (l == series_number - 1) {
258 |                 series_sizes[l/2] = series_sizes[l];
259 |             }
260 |             else {
261 |                 series_sizes[l/2] = series_sizes[l] + series_sizes[l + 1];
262 |             }
263 |         }
264 | 
265 |         series_size *= 2;
266 |         iteration++;
267 |     }
268 | }
269 | 
270 | bool join() {
271 |     block_size = RAM_SIZE / 3;
272 | 
273 |     a = memory;
274 |     b = memory + block_size;
275 |     c = memory + block_size * 2;
276 | 
277 |     fseek(sortedfile2, 0, SEEK_SET);
278 |     fseek(sortedfile1, 0, SEEK_SET);
279 |     fseek(joinedfile, 0, SEEK_SET);
280 | 
281 |     bool last_join = true;
282 |     unsigned b_i, b_n;
283 |     b_i = b_n = (unsigned)fread(b, 4, block_size, sortedfile1);
284 |     unsigned k = 0;
285 | 
286 |     for (unsigned i = 0; i < 3 * list_size; i += n_read) {
287 |         n_read = (unsigned)fread(a, 4, block_size, sortedfile2);
288 | 
289 |         for (unsigned j = 0; j < n_read; j += 3) {
290 |             while (a[j + 1] != b[k]) {
291 |                 k += 3;
292 |                 if (k == b_n) {
293 |                     b_n = (unsigned)fread(b, 4, block_size, sortedfile1);
294 |                     b_i += b_n;
295 |                     k = 0;
296 |                     if (b_n == 0 || b_i > 3*list_size) {
297 |                         fprintf(stderr, "Infinite cycle\n i=%d j=%d a[j+1]=%d block_size=%d", i, j, a[j+1], block_size);
298 |                         exit(2);
299 |                     }
300 |                 }
301 |             }
302 |             c[j] = a[j];
303 |             c[j + 1] = b[k + 1];
304 |             c[j + 2] = a[j + 2] + b[k + 2];
305 | 
306 |             if (last_join && c[j + 1] != root) {
307 | //                printf("%d w=%d root=%d\n", c[j+1], c[j+2], root);
308 |                 last_join = false;
309 |             }
310 |         }
311 |         fwrite(c, 4, n_read, joinedfile);
312 |     }
313 | 
314 |     return last_join;
315 | }
316 | 
317 | inline void cyclic_join() {
318 |     while (!join()) {
319 |         merge_sort(joinedfile, sortedfile2, 2);
320 |         merge_sort(joinedfile, sortedfile1, 1);
321 |     }
322 | }
323 | 
324 | void create_weighted(const char* ifilename) {
325 |     block_size = RAM_SIZE / 5 * 2;
326 |     a = memory;
327 |     b = memory + block_size;
328 | 
329 |     FILE* ifile = fopen(ifilename, "rb");
330 |     fread(&list_size, 4, 1, ifile);
331 | 
332 |     fseek(tempfile, 0, SEEK_SET);
333 | 
334 |     for (unsigned i = 0; i < 2*list_size; i += block_size) {
335 |         n_read = (unsigned) fread(a, 4, block_size, ifile);
336 |         for (unsigned j = 0; j < n_read / 2; ++j) {
337 |             b[3*j] = a[2*j];
338 |             b[3*j+1] = a[2*j+1];
339 |             b[3*j+2] = 1;
340 |         }
341 |         fwrite(b, 4, n_read / 2 * 3, tempfile);
342 |     }
343 |     fclose(ifile);
344 | }
345 | 
346 | void change_root_weight(FILE* file) {
347 |     fseek(file, 0, SEEK_SET);
348 |     fread(&root, 4, 1, file);
349 | 
350 |     unsigned p_w[2];
351 |     p_w[0] = root;
352 |     p_w[1] = 0;
353 |     fseek(file, 0, SEEK_CUR);
354 |     fwrite(p_w, 4, 2, file);
355 | }
356 | 
357 | void print_results(const char* ofilename) {
358 |     block_size = RAM_SIZE / 4;
359 | 
360 |     a = memory;
361 |     b = memory + block_size * 3;
362 | 
363 |     fseek(tempfile, 0, SEEK_SET);
364 | 
365 |     FILE* ofile = fopen(ofilename, "wb");
366 | 
367 |     for (unsigned i = 0; i < 3*list_size; i += 3*block_size) {
368 |         n_read = (unsigned) fread(a, 4, 3*block_size, tempfile);
369 |         for (unsigned j = 0; j < n_read / 3; ++j) {
370 |             b[j] = a[3*j];
371 |         }
372 |         fwrite(b, 4, n_read / 3, ofile);
373 |     }
374 |     fclose(ofile);
375 | }
376 | 
377 | void ranking() {
378 |     open_files();
379 | 
380 |     create_weighted("input.bin");
381 | 
382 |     merge_sort(tempfile, sortedfile2, 2);
383 |     change_root_weight(sortedfile2);
384 |     merge_sort(sortedfile2, sortedfile1, 1);
385 |     merge_sort(sortedfile1, sortedfile2, 2);
386 | 
387 |     cyclic_join();
388 |     merge_sort(joinedfile, tempfile, 3);
389 |     print_results("output.bin");
390 | 
391 |     close_files();
392 | }
393 | 
394 | int main() {
395 |     ranking();
396 |     delete[] memory;
397 | }
398 | 


--------------------------------------------------------------------------------
/external_memory_algorithms/ext_join.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <algorithm>
  3 | #include <cmath>
  4 | #include <chrono>
  5 | #include <random>
  6 | 
  7 | using namespace std;
  8 | 
  9 | void print_array(const int* arr, int n) {
 10 |     for (int i = 0; i < n; ++i) {
 11 |         cout << arr[i] << " ";
 12 |     }
 13 |     cout << endl;
 14 | }
 15 | 
 16 | char* filename(int block_size, int i, char* str) {
 17 |     sprintf(str, "%d_%d.bin", block_size, i);
 18 |     return str;
 19 | }
 20 | 
 21 | char buf[20];
 22 | 
 23 | struct Pair {
 24 |     int first;
 25 |     int second;
 26 | };
 27 | 
 28 | struct Triple {
 29 |     int first;
 30 |     int second;
 31 |     int third;
 32 | };
 33 | 
 34 | bool cmp_pair(const Pair& l, const Pair& r, bool sortBySecond) {
 35 |     return sortBySecond ? l.second <= r.second : l.first <= r.first;
 36 | }
 37 | 
 38 | void merge_2_files(int* memory, const char* filename1, const char* filename2, const char* outfilename,
 39 |                    int tuple_size, bool sort_by_second, const int PART_SIZE) {
 40 |     int* a = memory;
 41 |     int* b = memory + PART_SIZE;
 42 |     int* c = memory + 2 * PART_SIZE;
 43 | 
 44 |     FILE* f1 = nullptr;
 45 |     int f1_size = 0;
 46 |     if (filename1 != nullptr) {
 47 |         f1 = fopen(filename1, "rb");
 48 |         fread(&f1_size, 4, 1, f1);
 49 |         f1_size *= tuple_size;
 50 |     }
 51 | 
 52 |     FILE* f2 = nullptr;
 53 |     int f2_size = 0;
 54 |     if (filename2 != nullptr) {
 55 |         f2 = fopen(filename2, "rb");
 56 |         fread(&f2_size, 4, 1, f2);
 57 |         f2_size *= tuple_size;
 58 |     }
 59 | 
 60 |     FILE* f3 = fopen(outfilename, "wb");
 61 |     int f3_size = (f1_size + f2_size) / tuple_size;
 62 |     fwrite(&f3_size, 4, 1, f3);
 63 | 
 64 |     int a_rest = 0, b_rest = 0;
 65 |     int f1_count = 0, f2_count = 0;
 66 |     int a_size = 0, b_size = 0;
 67 | 
 68 |     while (f1_count < f1_size || f2_count < f2_size || a_rest != 0 || b_rest != 0) {
 69 |         if (a_size == 0 ) {
 70 |             int f1_read = min(PART_SIZE, f1_size - f1_count);
 71 |             if (f1_read > 0) {
 72 |                 fread(a, 4, f1_read, f1);
 73 |                 f1_count += f1_read;
 74 |                 a_size = f1_read;
 75 |             }
 76 |         }
 77 | 
 78 |         if (b_size == 0) {
 79 |             int f2_read = min(PART_SIZE, f2_size - f2_count);
 80 |             if (f2_read > 0) {
 81 |                 fread(b, 4, f2_read, f2);
 82 |                 f2_count += f2_read;
 83 |                 b_size = f2_read;
 84 |             }
 85 |         }
 86 | 
 87 |         int i = 0, j = 0, k = 0;
 88 | 
 89 |         if (a_size == 0) {
 90 |             copy(b, b + b_size, c);
 91 |             j = b_size;
 92 |         }
 93 |         else if (b_size == 0) {
 94 |             copy(a, a + a_size, c);
 95 |             i = a_size;
 96 |         }
 97 | 
 98 |         while (i < a_size && j < b_size) {
 99 | //            c[k++] = (a[i] <= b[j]) ? a[i++] : b[j++];
100 |             if (tuple_size == 2) {
101 |                 auto * p1 = (Pair*)&a[i];
102 |                 auto * p2 = (Pair*)&b[j];
103 |                 auto * p3 = (Pair*)&c[k];
104 |                 if (cmp_pair(*p1, *p2, sort_by_second)) {
105 |                     *p3 = *p1;
106 |                     i += tuple_size;
107 |                 }
108 |                 else {
109 |                     *p3 = *p2;
110 |                     j += tuple_size;
111 |                 }
112 |             }
113 |             else if (tuple_size == 3) {
114 |                 auto * p1 = (Triple*)&a[i];
115 |                 auto * p2 = (Triple*)&b[j];
116 |                 auto * p3 = (Triple*)&c[k];
117 |                 if (p1->first <= p2->first) {
118 |                     *p3 = *p1;
119 |                     i += tuple_size;
120 |                 }
121 |                 else {
122 |                     *p3 = *p2;
123 |                     j += tuple_size;
124 |                 }
125 |             }
126 | 
127 |             k += tuple_size;
128 |         }
129 | 
130 |         if (i < a_size && j >= b_size) {
131 |             copy(a + i, a + a_size, a);
132 |             a_rest = a_size - i;
133 |             b_rest = 0;
134 |         }
135 |         else if (j < b_size && i >= a_size) {
136 |             copy(b + j, b + b_size, b);
137 |             b_rest = b_size - j;
138 |             a_rest = 0;
139 |         }
140 |         else a_rest = b_rest = 0;
141 | 
142 | 
143 |         fwrite(c, 4, a_size + b_size - a_rest - b_rest, f3);
144 | 
145 |         a_size = a_rest;
146 |         b_size = b_rest;
147 |     }
148 | 
149 |     if (f1 != nullptr)
150 |         fclose(f1);
151 |     if (f2 != nullptr)
152 |         fclose(f2);
153 |     fclose(f3);
154 | }
155 | 
156 | void merge_sort(int* memory, const char* inputFilename, const char* outputFilename, int tuple_size, bool sortBySecond, const int PART_SIZE) {
157 |     FILE *ifile = nullptr;
158 |     FILE *ofile = nullptr;
159 |     int N;
160 | 
161 |     ifile = fopen(inputFilename, "rb");
162 | 
163 |     fread(&N, 4, 1, ifile);
164 |     N *= tuple_size;
165 | 
166 |     int block_size = min(4 * PART_SIZE, N);
167 | 
168 |     //split
169 |     for (int i = 0; i < N; i += block_size) {
170 |         int n = min(block_size, N - i);
171 |         fread(memory, 4, n, ifile);
172 | 
173 |         if(tuple_size == 2) {
174 |             auto * pairs = (Pair*)memory;
175 |             if (sortBySecond) {
176 |                 sort(pairs, pairs + n / tuple_size, [] (const Pair& l, const Pair& r) {
177 |                     return l.second < r.second;
178 |                 });
179 |             }
180 |             else {
181 |                 sort(pairs, pairs + n / tuple_size, [](const Pair& l, const Pair& r) {
182 |                     return l.first < r.first;
183 |                 });
184 |             }
185 |         }
186 |         else if (tuple_size == 3) {
187 |             auto * triples = (Triple*)memory;
188 |             sort(triples, triples + n / tuple_size, [] (const Triple& l, const Triple& r) {
189 |                 return l.first < r.first;
190 |             });
191 |         }
192 | 
193 |         ofile = fopen(filename(block_size, i / block_size, buf), "wb");
194 | 
195 |         int output_size = n / tuple_size;
196 |         fwrite(&output_size, 4, 1, ofile);
197 |         fwrite(memory, 4, n, ofile);
198 | 
199 |         fclose(ofile);
200 |     }
201 |     fclose(ifile);
202 | 
203 |     //merge
204 |     while(block_size <= N) {
205 |         auto files_number = ceil(N * 1.0 / block_size);
206 | 
207 |         if (files_number == 1) {
208 |             merge_2_files(memory, filename(block_size, 0, buf), NULL, outputFilename,tuple_size, sortBySecond, PART_SIZE);
209 |             break;
210 |         }
211 | 
212 |         for (int i = 0; i < files_number; i += 2) {
213 |             char filename1[20];
214 |             filename(block_size, i, filename1);
215 | 
216 |             char* filename2 = NULL;
217 |             if (i+1 < files_number)
218 |                 filename2 = filename(block_size, i + 1, buf);
219 | 
220 |             char outfilename[20];
221 |             filename(2 * block_size, i / 2, outfilename);
222 | 
223 |             merge_2_files(memory, filename1, filename2, outfilename, tuple_size, sortBySecond, PART_SIZE);
224 |         }
225 | 
226 |         if (2 * block_size > N) {
227 |             merge_2_files(memory, filename(2 * block_size, 0, buf), NULL, outputFilename,tuple_size, sortBySecond, PART_SIZE);
228 |             break;
229 |         }
230 | 
231 |         block_size *= 2;
232 |     }
233 | }
234 | 
235 | void join(int *memory, const char *outputFilename, const int PART_SIZE) {
236 |     int* a = memory;
237 |     int* b = memory + PART_SIZE;
238 |     int* c = memory + 2 * PART_SIZE;
239 | 
240 |     FILE* f1 = fopen("output1.bin", "rb");
241 |     int f1_size;
242 |     fread(&f1_size, 4, 1, f1);
243 |     f1_size *= 2;
244 | 
245 |     FILE* f2 = fopen("output2.bin", "rb");
246 |     int f2_size;
247 |     fread(&f2_size, 4, 1, f2);
248 |     f2_size *= 2;
249 | 
250 |     FILE* f3 = fopen(outputFilename, "wb");
251 |     int size_output = f1_size / 2;
252 |     fwrite(&size_output, 4, 1, f3);
253 | 
254 |     for (int i = 0; i < f1_size; i += PART_SIZE) {
255 |         int n = min(PART_SIZE, f1_size - i);
256 |         fread(a, 4, n, f1);
257 |         fread(b, 4, n, f2);
258 | 
259 |         int k = 0;
260 |         for (int j = 0; j < n; j+=2) {
261 |             c[k] = a[j];
262 |             c[k+1] = a[j+1];
263 |             c[k+2] = b[j+1];
264 |             k += 3;
265 |         }
266 | 
267 |         fwrite(c, 4, k, f3);
268 |     }
269 |     fclose(f3);
270 | }
271 | 
272 | void copy_to_result_file(int *memory, const char *inputFilename, const char *outputFilename, const int PART_SIZE) {
273 |     FILE* ifile = fopen(inputFilename, "rb");
274 |     int size;
275 |     fread(&size, 4, 1, ifile);
276 |     size *= 3;
277 | 
278 |     FILE* ofile = fopen(outputFilename, "wb");
279 | 
280 |     int block_size = min(4 * PART_SIZE, size);
281 | 
282 |     for (int i = 0; i < size; i += block_size) {
283 |         int n = min(block_size, size - i);
284 |         fread(memory, 4, n, ifile);
285 |         fwrite(memory, 4, n, ofile);
286 |     }
287 |     fclose(ifile);
288 |     fclose(ofile);
289 | }
290 | 
291 | void ext_join(const int PART_SIZE) {
292 |     auto* memory = new int[4 * PART_SIZE];
293 | 
294 |     merge_sort(memory, "input.bin", "output1.bin", 2, true, PART_SIZE);
295 |     merge_sort(memory, "input.bin", "output2.bin", 2, false, PART_SIZE);
296 | 
297 |     join(memory, "output3.bin", PART_SIZE);
298 | 
299 |     merge_sort(memory, "output3.bin", "output4.bin", 3, false, PART_SIZE);
300 | 
301 |     copy_to_result_file(memory, "output4.bin", "output.bin", PART_SIZE);
302 | 
303 |     delete[] memory;
304 | }
305 | 
306 | void init_data(const int SIZE) {
307 |     FILE *f = nullptr;
308 | 
309 |     f = fopen("input.bin", "wb");
310 |     if (f == nullptr) {
311 |         printf("Error opening file");
312 |     }
313 | 
314 |     int* arr = new int[SIZE];
315 |     for (int i = 0; i < SIZE; ++i) {
316 |         arr[i] = i + 1;
317 |     }
318 |     long seed = chrono::system_clock::now().time_since_epoch().count();
319 | //    shuffle (arr, arr + SIZE, default_random_engine(seed));
320 | 
321 |     int* a = new int[2*SIZE];
322 |     for (int i = 0; i < SIZE - 1; i++) {
323 |         a[2*i] = arr[i];
324 |         a[2*i+1] = arr[i+1];
325 |     }
326 |     a[2*(SIZE-1)] = arr[SIZE-1];
327 |     a[2*(SIZE-1)+1] = arr[0];
328 | 
329 |     fwrite(&SIZE, 4, 1, f);
330 |     fwrite(a, 4, 2 * SIZE, f);
331 | 
332 |     fclose(f);
333 |     delete[] arr;
334 |     delete[] a;
335 | }
336 | 
337 | 
338 | int* read_file(const char* filename, const int size) {
339 |     FILE* f = fopen(filename, "rb");
340 | 
341 |     auto* arr = new int[size];
342 |     fread(arr, 4, size, f);
343 |     return arr;
344 | }
345 | 
346 | int main() {
347 | //    init_data(size);
348 | 
349 |     ext_join(30000);
350 | }
351 | 


--------------------------------------------------------------------------------
/external_memory_algorithms/ext_matrix_multiplication.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <memory>
  3 | using namespace std;
  4 | 
  5 | void print_matrix(const unsigned char *matrix, int n, int m) {
  6 |     for (int i = 0; i < n; ++i) {
  7 |         for (int j = 0; j < m; ++j) {
  8 |             cout << int(matrix[i*m + j]) << " ";
  9 |         }
 10 |         cout << endl;
 11 |     }
 12 |     cout << endl;
 13 | }
 14 | 
 15 | void MultSwappedLoops(const unsigned char *__restrict a, const unsigned char *__restrict b, unsigned char *__restrict c,
 16 |                       int n, int m, int l, bool initC) {
 17 |     for (int i = 0; i < n; ++i) {
 18 |         if (initC) {
 19 |             for (int j = 0; j < l; ++j) {
 20 |                 c[i * l + j] = 0;
 21 |             }
 22 |         }
 23 | 
 24 |         for (int k = 0; k < m; ++k) {
 25 |             for (int j = 0; j < l; ++j) {
 26 |                 c[i * l + j] = (c[i * l + j] + a[i * m + k] * b[k * l + j]) % 256;
 27 |             }
 28 |         }
 29 |     }
 30 | }
 31 | 
 32 | void init_data() {
 33 |     FILE *f = NULL;
 34 | 
 35 | 
 36 |     f = fopen("input.bin", "w+b");
 37 |     if (f == NULL) {
 38 |         printf("Error opening file");
 39 |     }
 40 | 
 41 |     unsigned char a[9] = {2, 0, 0, 0, 1, 0, 0, 0, 1};
 42 |     unsigned char b[9] = {1, 2, 3, 4, 5, 6, 7, 8, 9};
 43 | 
 44 |     int size = 3;
 45 |     fwrite(&size, 4, 1, f);
 46 |     fwrite(&size, 4, 1, f);
 47 |     fwrite(a, 1, 9, f);
 48 | 
 49 |     fwrite(&size, 4, 1, f);
 50 |     fwrite(&size, 4, 1, f);
 51 |     fwrite(b, 1, 9, f);
 52 | 
 53 |     fclose(f);
 54 | }
 55 | 
 56 | void read_block(FILE *f, unsigned char *pBlock, int n, int m, int N) {
 57 |     for (int i = 0; i < n; ++i) {
 58 |         fread(pBlock + i * m, 1, m, f);
 59 |         fseek(f, N - m, SEEK_CUR);
 60 |     }
 61 | }
 62 | 
 63 | void write_block(FILE *f, unsigned char *pBlock, int n, int m, int N) {
 64 |     for (int i = 0; i < n; ++i) {
 65 |         fwrite(pBlock + i * m, 1, m, f);
 66 |         fseek(f, N - m, SEEK_CUR);
 67 |     }
 68 | }
 69 | 
 70 | int main() {
 71 |     const int s = 400;
 72 |     int N, N1, N2, M1, M2;
 73 |     unsigned char a[s*s];
 74 |     unsigned char b[s*s];
 75 |     unsigned char c[s*s];
 76 | 
 77 |     FILE *ifile = NULL;
 78 |     FILE *ofile = NULL;
 79 | 
 80 |     ifile = fopen("input.bin", "rb");
 81 |     ofile = fopen("output.bin", "w+b");
 82 |     if (ifile == NULL) {
 83 |         printf("Error opening file");
 84 |         exit(1);
 85 |     }
 86 | 
 87 |     fread(&N1, 4, 1, ifile);
 88 |     fread(&M1, 4, 1, ifile);
 89 | 
 90 |     if (N1 != M1) {
 91 |         printf("Not square matrix");
 92 |         exit(1);
 93 |     }
 94 | 
 95 |     N = N1;
 96 |     fwrite(&N, 4, 1, ofile);
 97 |     fwrite(&N, 4, 1, ofile);
 98 | 
 99 |     for (int i = 0; i < N; i += s) {
100 |         int n = min(s, N - i);
101 |         for (int j = 0; j < N; j += n) {
102 |             int l = min(s, N - j);
103 |             for (int k = 0; k < N; k += s) {
104 |                 int m = min(s, N - k);
105 | 
106 |                 fseek(ifile, 8 + i * N + k, SEEK_SET);
107 |                 read_block(ifile, a, n, m, N);
108 |                 fseek(ifile, 16 + N * N + k * N + j, SEEK_SET);
109 |                 read_block(ifile, b, m, l, N);
110 |                 MultSwappedLoops(a, b, c, n, m, l, k == 0);
111 |             }
112 |             fseek(ofile, 8 + i * N + j, SEEK_SET);
113 |             write_block(ofile, c, n, l, N);
114 |         }
115 |     }
116 | 
117 |     fclose(ifile);
118 |     fclose(ofile);
119 | }
120 | 


--------------------------------------------------------------------------------
/external_memory_algorithms/ext_matrix_transposition.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <memory>
  3 | using namespace std;
  4 | 
  5 | void print_matrix(const unsigned char *matrix, int n, int m) {
  6 |     for (int i = 0; i < n; ++i) {
  7 |         for (int j = 0; j < m; ++j) {
  8 |             cout << int(matrix[i*m + j]) << " ";
  9 |         }
 10 |         cout << endl;
 11 |     }
 12 |     cout << endl;
 13 | }
 14 | 
 15 | void transpose(const unsigned char *__restrict a, unsigned char *__restrict b, int n, int m) {
 16 |     for (int i = 0; i < n; ++i) {
 17 |         for (int j = 0; j < m; ++j) {
 18 |             b[j*n + i] = a[i*m + j];
 19 |         }
 20 |     }
 21 | }
 22 | 
 23 | void block_transpose(const unsigned char *__restrict a, unsigned char *__restrict b, int n, int m) {
 24 |     const int BLOCK = 8;
 25 |     for (int i = 0; i < n; i+=BLOCK) {
 26 |         for (int j = 0; j < m; j+=BLOCK) {
 27 |             for (int ib = 0; ib < BLOCK && i + ib < n; ++ib) {
 28 |                 for (int jb = 0; jb < BLOCK && j + jb < m; ++jb) {
 29 |                     b[(j+jb) * n + (i+ib)] = a[(i+ib) * m + (j+jb)];
 30 |                 }
 31 |             }
 32 |         }
 33 |     }
 34 | }
 35 | 
 36 | void init_data() {
 37 |     FILE *f = NULL;
 38 | 
 39 |     f = fopen("input.bin", "w+b");
 40 |     if (f == NULL) {
 41 |         printf("Error opening file");
 42 |     }
 43 | 
 44 |     unsigned char a[12] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0};
 45 | 
 46 |     int size1 = 6;
 47 |     int size2 = 2;
 48 |     fwrite(&size1, 4, 1, f);
 49 |     fwrite(&size2, 4, 1, f);
 50 |     fwrite(a, 1, 12, f);
 51 | 
 52 |     fclose(f);
 53 | }
 54 | 
 55 | void read_block(FILE *f, unsigned char *pBlock, int n, int m, int M) {
 56 |     if (m == M) {
 57 |         fread(pBlock, 1, n * m, f);
 58 |     }
 59 |     else {
 60 |         for (int i = 0; i < n; ++i) {
 61 |             fread(pBlock + i * m, 1, m, f);
 62 |             fseek(f, M - m, SEEK_CUR);
 63 |         }
 64 |     }
 65 | }
 66 | 
 67 | void write_block(FILE *f, unsigned char *pBlock, int n, int m, int M) {
 68 |     if (m == M) {
 69 |         fwrite(pBlock, 1, n * m, f);
 70 |     }
 71 |     else {
 72 |         for (int i = 0; i < n; ++i) {
 73 |             fwrite(pBlock + i * m, 1, m, f);
 74 |             fseek(f, M - m, SEEK_CUR);
 75 |         }
 76 |     }
 77 | }
 78 | 
 79 | int main() {
 80 | //    init_data();
 81 | 
 82 |     const int BLOCK_SIZE = 500;
 83 |     int block_n, block_m;
 84 |     int N, M;
 85 | 
 86 |     FILE *ifile = nullptr;
 87 |     FILE *ofile = nullptr;
 88 | 
 89 |     ifile = fopen("input.bin", "rb");
 90 |     ofile = fopen("output.bin", "w+b");
 91 |     if (ifile == nullptr) {
 92 |         printf("Error opening file");
 93 |         exit(1);
 94 |     }
 95 | 
 96 |     fread(&N, 4, 1, ifile);
 97 |     fread(&M, 4, 1, ifile);
 98 | 
 99 |     fwrite(&M, 4, 1, ofile);
100 |     fwrite(&N, 4, 1, ofile);
101 | 
102 |     block_n = min(BLOCK_SIZE, N);
103 |     block_m = min(M, BLOCK_SIZE * BLOCK_SIZE / block_n);
104 |     block_n = min(N, BLOCK_SIZE * BLOCK_SIZE / block_m); //enlarge block_n if block_m < BLOCK_SIZE
105 | 
106 |     unsigned char a[block_n * block_m];
107 |     unsigned char b[block_m * block_n];
108 | 
109 |     for (int i = 0; i < N; i += block_n) {
110 |         int n = min(block_n, N - i);
111 |         for (int j = 0; j < M; j += block_m) {
112 |             int m = min(block_m, M - j);
113 | 
114 |             fseek(ifile, 8 + i * M + j, SEEK_SET);
115 |             read_block(ifile, a, n, m, M);
116 | //            print_matrix(a, n, m);
117 |             transpose(a, b, n, m);
118 | //            print_matrix(b, m, n);
119 | 
120 |             fseek(ofile, 8 + j * N + i, SEEK_SET);
121 |             write_block(ofile, b, m, n, N);
122 |         }
123 |     }
124 | 
125 |     fclose(ifile);
126 |     fclose(ofile);
127 | }
128 | 


--------------------------------------------------------------------------------
/external_memory_algorithms/ext_merge_sort.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <algorithm>
  3 | #include <cmath>
  4 | 
  5 | using namespace std;
  6 | 
  7 | /*void print_array(const unsigned long long* arr, int n) {
  8 |     for (int i = 0; i < n; ++i) {
  9 |         cout << arr[i] << " ";
 10 |     }
 11 |     cout << endl;
 12 | }*/
 13 | 
 14 | /*
 15 | void init_data() {
 16 |     FILE *f = nullptr;
 17 | 
 18 |     f = fopen("input.bin", "w+b");
 19 |     if (f == nullptr) {
 20 |         printf("Error opening file");
 21 |     }
 22 | 
 23 | 
 24 |     const unsigned long long SIZE = 1000000;
 25 |     auto * a = new unsigned long long[SIZE];
 26 |     for (int i = 0; i <SIZE; ++i) {
 27 |         a[i] = SIZE - i;
 28 |     }
 29 | 
 30 |     fwrite(&SIZE, 8, 1, f);
 31 |     fwrite(a, 8,SIZE, f);
 32 | 
 33 |     delete[] a;
 34 |     fclose(f);
 35 | }
 36 | */
 37 | 
 38 | char* filename(unsigned int block_size, int i, char* str) {
 39 |     sprintf(str, "%d_%d.bin", block_size, i);
 40 |     return str;
 41 | }
 42 | 
 43 | char buf[20];
 44 | 
 45 | void merge_2_files(unsigned long long* a, unsigned long long* b, unsigned long long* c, int iter, unsigned int block_size, const unsigned int PART_SIZE) {
 46 |     FILE* f1 = fopen(filename(block_size, iter, buf), "rb");
 47 |     unsigned long long f1_size;
 48 |     fread(&f1_size, 8, 1, f1);
 49 | 
 50 |     FILE* f2 = fopen(filename(block_size, iter + 1, buf), "rb");
 51 |     unsigned long long f2_size;
 52 |     fread(&f2_size, 8, 1, f2);
 53 | 
 54 |     FILE* f3 = fopen(filename(2 * block_size, iter / 2, buf), "w+b");
 55 |     unsigned long long f3_size = f1_size + f2_size;
 56 |     fwrite(&f3_size, 8, 1, f3);
 57 | 
 58 |     unsigned int a_rest = 0, b_rest = 0;
 59 |     unsigned int f1_count = 0, f2_count = 0;
 60 |     unsigned int a_size = 0, b_size = 0;
 61 | 
 62 |     while (f1_count < f1_size || f2_count < f2_size || a_rest != 0 || b_rest != 0) {
 63 |         if (a_size == 0 ) {
 64 |             unsigned int f1_read = min(PART_SIZE, (unsigned int)(f1_size - f1_count));
 65 |             if (f1_read != 0) {
 66 |                 fread(a, 8, f1_read, f1);
 67 |                 f1_count += f1_read;
 68 |                 a_size = f1_read;
 69 |             }
 70 |         }
 71 | 
 72 |         if (b_size == 0) {
 73 |             unsigned int f2_read = min(PART_SIZE, (unsigned int)(f2_size - f2_count));
 74 |             if (f2_read != 0) {
 75 |                 fread(b, 8, f2_read, f2);
 76 |                 f2_count += f2_read;
 77 |                 b_size = f2_read;
 78 |             }
 79 |         }
 80 | 
 81 |         {
 82 |             int i = 0, j = 0, k = 0;
 83 | 
 84 |             if (a_size == 0) {
 85 |                 copy(b, b + b_size, c);
 86 |                 j = b_size;
 87 |             }
 88 |             else if (b_size == 0) {
 89 |                 copy(a, a + a_size, c);
 90 |                 i = a_size;
 91 |             }
 92 | 
 93 |             while (i < a_size && j < b_size)
 94 |                 c[k++] = (a[i] <= b[j]) ? a[i++] : b[j++];
 95 | 
 96 |             if (i < a_size) {
 97 |                 copy(a + i, a + a_size, a);
 98 |                 a_rest = a_size - i;
 99 |                 b_rest = 0;
100 |             }
101 |             else if (j < b_size) {
102 |                 copy(b + j, b + b_size, b);
103 |                 b_rest = b_size - j;
104 |                 a_rest = 0;
105 |             }
106 |             else a_rest = b_rest = 0;
107 |         }
108 | 
109 | 
110 |         fwrite(c, 8, a_size + b_size - a_rest - b_rest, f3);
111 | 
112 |         a_size = a_rest;
113 |         b_size = b_rest;
114 |     }
115 | 
116 |     fclose(f1);
117 |     fclose(f2);
118 |     fclose(f3);
119 | }
120 | 
121 | /*void read_file() {
122 |     FILE *f = nullptr;
123 | 
124 |     f = fopen("output.bin", "r+b");
125 |     if (f == nullptr) {
126 |         printf("Error opening file");
127 |     }
128 | 
129 |     unsigned long long N;
130 |     fread(&N, 8, 1, f);
131 |     cout << N << endl;
132 | 
133 |     auto * arr = new unsigned long long[N];
134 | 
135 |     fread(arr, 8, N, f);
136 |     for (int i = 0; i < N; ++i) {
137 |         cout << arr[i] << " ";
138 |     }
139 |     cout << endl;
140 | 
141 |     fclose(f);
142 |     delete[] arr;
143 | }*/
144 | 
145 | int main() {
146 | //    init_data();
147 | //    read_file();
148 | 
149 |     unsigned int PART_SIZE = 15000;
150 |     unsigned long long N;
151 | 
152 |     FILE *ifile = nullptr;
153 |     FILE *ofile = nullptr;
154 | 
155 |     ifile = fopen("input.bin", "rb");
156 |     if (ifile == nullptr) {
157 |         printf("Error opening file");
158 |         exit(1);
159 |     }
160 | 
161 |     fread(&N, 8, 1, ifile);
162 | //    if (PART_SIZE > N) PART_SIZE = N;
163 | 
164 |     auto * arr = new unsigned long long[4 * PART_SIZE];
165 |     unsigned long long* a = arr;
166 |     unsigned long long* b = arr + PART_SIZE;
167 |     unsigned long long* c = arr + 2 * PART_SIZE;
168 | 
169 |     unsigned int block_size = min(4 * PART_SIZE, (unsigned int)N);
170 | 
171 |     //split
172 |     for (int i = 0; i < N; i += block_size) {
173 |         unsigned long long n = min(block_size, (unsigned int)(N - i));
174 |         fread(arr, 8, n, ifile);
175 | 
176 |         sort(arr, arr + n);
177 | 
178 |         ofile = fopen(filename(block_size, i / block_size, buf), "w+b");
179 | 
180 |         fwrite(&n, 8, 1, ofile);
181 |         fwrite(arr, 8, n, ofile);
182 | 
183 |         fclose(ofile);
184 |     }
185 |     fclose(ifile);
186 | 
187 |     //merge
188 | 
189 |     while(block_size <= N) {
190 |         unsigned int files_number = (unsigned int)ceil(N * 1.0 / block_size);
191 | 
192 |         if (files_number == 1) {
193 |             rename(filename(block_size, 0, buf), "output.bin");
194 |             break;
195 |         }
196 | 
197 |         for (int i = 0; i < files_number; i += 2) {
198 |             if (i+1 >= files_number) {
199 |                 char buf2[20];
200 |                 rename(filename(block_size, i, buf), filename(2 * block_size, i / 2, buf2));
201 |                 break;
202 |             }
203 | 
204 |             merge_2_files(a, b, c, i, block_size, PART_SIZE);
205 |         }
206 | 
207 |         if (2 * block_size > N) {
208 |             rename(filename(2 * block_size, 0, buf), "output.bin");
209 |             break;
210 |         }
211 | 
212 |         block_size *= 2;
213 |     }
214 | 
215 |     delete[] arr;
216 | }
217 | 


--------------------------------------------------------------------------------
/external_memory_algorithms/f_Ab.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <memory>
 3 | using namespace std;
 4 | 
 5 | void multiply(const unsigned char *__restrict a, const unsigned char *__restrict b, unsigned char *__restrict c,
 6 |               int n, int m, bool initC) {
 7 |     for (int i = 0; i < n; ++i) {
 8 |         if (initC) {
 9 |             c[i] = 0;
10 |         }
11 |         for (int j = 0; j < m; ++j) {
12 |             c[i] += a[i*m + j] * b[j] % 256;
13 |         }
14 |     }
15 | }
16 | 
17 | void init_data() {
18 |     FILE *f = fopen("input.bin", "w+b");
19 |     unsigned char a[12] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0};
20 |     unsigned char b[4] = {1, 2, 3, 4};
21 | 
22 |     int size1 = 3;
23 |     int size2 = 4;
24 |     fwrite(&size1, 4, 1, f);
25 |     fwrite(&size2, 4, 1, f);
26 |     fwrite(a, 1, 12, f);
27 |     fwrite(b, 1, 4, f);
28 | 
29 |     fclose(f);
30 | }
31 | 
32 | void read_block(FILE *f, unsigned char *pBlock, int n, int m, int M) {
33 |     if (m == M) {
34 |         fread(pBlock, 1, n * m, f);
35 |     }
36 |     else {
37 |         for (int i = 0; i < n; ++i) {
38 |             fread(pBlock + i * m, 1, m, f);
39 |             fseek(f, M - m, SEEK_CUR);
40 |         }
41 |     }
42 | }
43 | 
44 | int main() {
45 | //    init_data();
46 | 
47 |     const int BLOCK_SIZE = 500;
48 |     int block_n, block_m;
49 |     int N, M;
50 | 
51 |     FILE *ifile = nullptr;
52 |     FILE *ofile = nullptr;
53 | 
54 |     ifile = fopen("input.bin", "rb");
55 |     ofile = fopen("output.bin", "w+b");
56 | 
57 |     fread(&N, 4, 1, ifile);
58 |     fread(&M, 4, 1, ifile);
59 | 
60 |     block_n = min(BLOCK_SIZE, N);
61 |     block_m = min(M, BLOCK_SIZE * BLOCK_SIZE / block_n);
62 |     block_n = min(N, BLOCK_SIZE * BLOCK_SIZE / block_m); //enlarge block_n if block_m < BLOCK_SIZE
63 | 
64 |     unsigned char a[block_n * block_m];
65 |     unsigned char b[block_m];
66 |     unsigned char c[block_n];
67 | 
68 |     for (int i = 0; i < N; i += block_n) {
69 |         int n = min(block_n, N - i);
70 |         for (int j = 0; j < M; j += block_m) {
71 |             int m = min(block_m, M - j);
72 | 
73 |             fseek(ifile, 8 + i * M + j, SEEK_SET);
74 |             read_block(ifile, a, n, m, M);
75 | 
76 |             fseek(ifile, 8 + M * N + j, SEEK_SET);
77 |             fread(b, 1, m, ifile);
78 | 
79 |             multiply(a, b, c, n, m, j==0);
80 |         }
81 |         fseek(ofile, i, SEEK_SET);
82 |         fwrite(c, 1, n, ofile);
83 |     }
84 | 
85 |     fclose(ifile);
86 |     fclose(ofile);
87 | }
88 | 


--------------------------------------------------------------------------------
/external_memory_algorithms/tree.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <algorithm>
  3 | 
  4 | struct Triple_1 {
  5 |     unsigned i;
  6 |     unsigned p_i;
  7 |     unsigned w;
  8 | 
  9 |     bool operator < (Triple_1 t2) {
 10 |         return i < t2.i;
 11 |     }
 12 | };
 13 | 
 14 | struct Triple_2 {
 15 |     unsigned i;
 16 |     unsigned p_i;
 17 |     unsigned w;
 18 | 
 19 |     bool operator < (Triple_2 t2) {
 20 |         return p_i < t2.p_i;
 21 |     }
 22 | };
 23 | 
 24 | struct Triple {
 25 |     union {
 26 |         Triple_1 triple1;
 27 |         Triple_2 triple2;
 28 |     };
 29 | };
 30 | 
 31 | static const unsigned RAM_SIZE = 100080;
 32 | static unsigned* memory = new unsigned[RAM_SIZE];
 33 | static unsigned series_sizes[3750000 / RAM_SIZE + 1];
 34 | static unsigned list_size;
 35 | static unsigned root;
 36 | static unsigned* a;
 37 | static unsigned* b;
 38 | static unsigned* c;
 39 | static unsigned block_size;
 40 | static unsigned n_read;
 41 | 
 42 | static FILE *tempfile, *mergefile1, *mergefile2, *sortedfile1, *sortedfile2, *joinedfile;
 43 | 
 44 | unsigned min(const unsigned& a, const unsigned& b) {
 45 |     return a < b ? a : b;
 46 | }
 47 | 
 48 | void open_files() {
 49 |     tempfile = fopen("tempfile.bin", "w+b");
 50 |     mergefile1 = fopen("mergefile1.bin", "w+b");
 51 |     mergefile2 = fopen("mergefile2.bin", "w+b");
 52 |     sortedfile1 = fopen("sortedfile1.bin", "w+b");
 53 |     sortedfile2 = fopen("sortedfile2.bin", "w+b");
 54 |     joinedfile = fopen("joinedfile.bin", "w+b");
 55 | }
 56 | 
 57 | void close_files() {
 58 |     fclose(tempfile);
 59 |     fclose(mergefile1);
 60 |     fclose(mergefile2);
 61 |     fclose(sortedfile1);
 62 |     fclose(sortedfile2);
 63 |     fclose(joinedfile);
 64 | }
 65 | 
 66 | void merge(FILE* from_file, FILE* to_file, const unsigned &SERIES_NUMBER, const unsigned &sort_type) {
 67 |     block_size = RAM_SIZE / 4;
 68 | 
 69 |     a = memory;
 70 |     b = memory + block_size;
 71 |     c = memory + 2 * block_size;
 72 | 
 73 |     Triple *p1;
 74 |     Triple *p2;
 75 |     Triple *p3;
 76 |     bool cmp;
 77 |     unsigned copy_size;
 78 | 
 79 |     unsigned total_read1, total_read2;
 80 |     unsigned series1_end, series2_end;
 81 |     unsigned a_size, b_size, c_size;
 82 |     unsigned i, j, k;
 83 | 
 84 |     c_size = 2 * block_size;
 85 | 
 86 |     fseek(to_file, 0, SEEK_SET);
 87 | 
 88 |     total_read1 = 0, total_read2 = 0;
 89 |     for (unsigned series_i = 0; series_i < SERIES_NUMBER; series_i += 2) {
 90 |         total_read1 = total_read2;
 91 |         series1_end = total_read1 + 3 * series_sizes[series_i];
 92 | 
 93 |         //if next series does not exist, just copy last to file
 94 |         if (series_i + 1 == SERIES_NUMBER) {
 95 |             fseek(from_file, 4 * total_read1, SEEK_SET);
 96 |             for (; total_read1 < 3*list_size; total_read1 += n_read) {
 97 |                 n_read = (unsigned) fread(memory, 4, RAM_SIZE, from_file);
 98 |                 fwrite(memory, 4, n_read, to_file);
 99 |             }
100 |             break;
101 |         }
102 | 
103 |         total_read2 = series1_end;
104 |         series2_end = total_read2 + 3 * series_sizes[series_i + 1];
105 | 
106 |         i = j = a_size = b_size = block_size;
107 |         k = 0;
108 |         while (total_read1 != series1_end || total_read2 != series2_end || k != 0) {
109 |             if (a_size != 0 && i == a_size) {
110 |                 if (total_read1 < series1_end) {
111 |                     a_size = min(block_size, series1_end - total_read1);
112 |                     fseek(from_file, 4 * total_read1, SEEK_SET);
113 |                     fread(a, 4, a_size, from_file);
114 |                     total_read1 += a_size;
115 |                     i = 0;
116 |                 }
117 |                 else {
118 |                     i = a_size = 0;
119 |                 }
120 |             }
121 |             if (b_size !=0 && j == b_size) {
122 |                 if (total_read2 < series2_end) {
123 |                     b_size = min(block_size, series2_end - total_read2);
124 |                     fseek(from_file, 4 * total_read2, SEEK_SET);
125 |                     fread(b, 4, b_size, from_file);
126 |                     total_read2 += b_size;
127 |                     j = 0;
128 |                 }
129 |                 else {
130 |                     j = b_size = 0;
131 |                 }
132 |             }
133 | 
134 |             if (a_size == 0 && b_size == 0) {
135 |                 fwrite(c, 4, k, to_file);
136 |                 break;
137 |             }
138 |             if (k == c_size) {
139 |                 fwrite(c, 4, k, to_file);
140 |                 k = 0;
141 |             }
142 | 
143 |             if (a_size == 0) {
144 |                 copy_size = min(b_size - j, c_size - k);
145 |                 std::copy(b + j, b + j + copy_size, c + k);
146 |                 k += copy_size;
147 |                 j += copy_size;
148 |             }
149 |             else if (b_size == 0) {
150 |                 copy_size = min(a_size - i, c_size - k);
151 |                 std::copy(a + i, a + i + copy_size, c + k);
152 |                 k += copy_size;
153 |                 i += copy_size;
154 |             }
155 |             else {
156 |                 while (i < a_size && j < b_size && k < c_size) {
157 | //            c[k++] = (a[i] <= b[j]) ? a[i++] : b[j++];
158 | 
159 |                     p1 = (Triple *) &a[i];
160 |                     p2 = (Triple *) &b[j];
161 |                     p3 = (Triple *) &c[k];
162 | 
163 |                     if (sort_type == 1) {
164 |                         cmp = (p1->triple1 < p2->triple1);
165 |                     } else if (sort_type == 2) {
166 |                         cmp = (p1->triple2 < p2->triple2);
167 |                     } else {
168 |                         fprintf(stdout, "Undefined sort_type\n");
169 |                         exit(1);
170 |                     }
171 | 
172 |                     if (cmp) {
173 |                         *p3 = *p1;
174 |                         i += 3;
175 |                     } else {
176 |                         *p3 = *p2;
177 |                         j += 3;
178 |                     }
179 | 
180 |                     k += 3;
181 |                 }
182 |             }
183 |         }
184 |     }
185 | }
186 | 
187 | void merge_sort(FILE* ifile, FILE* ofile, const unsigned& sort_type) {
188 |     fseek(ifile, 0, SEEK_SET);
189 | 
190 |     FILE* fileptr = (RAM_SIZE < list_size * 3) ? mergefile1 : ofile;
191 |     fseek(fileptr, 0, SEEK_SET);
192 | 
193 |     //split
194 |     unsigned counter = 0;
195 |     for (unsigned i = 0; i < 3 * list_size; i += n_read) {
196 |         n_read = (unsigned) fread(memory, 4, RAM_SIZE, ifile);
197 | 
198 |         if (sort_type == 1) {
199 |             std::sort((Triple_1*)memory, (Triple_1*)memory + n_read / 3);
200 |         }
201 |         else if (sort_type == 2) {
202 |             std::sort((Triple_2*)memory, (Triple_2*)memory + n_read / 3);
203 |         }
204 |         else {
205 |             fprintf(stderr, "Undefined sort_type\n");
206 |             exit(1);
207 |         }
208 | 
209 |         fwrite(memory, 4, n_read, fileptr);
210 |         series_sizes[counter] = n_read / 3;
211 |         counter++;
212 |     }
213 | 
214 |     //merge
215 |     unsigned series_size = RAM_SIZE;
216 |     unsigned series_number;
217 |     unsigned iteration = 0;
218 |     while (series_size < 3*list_size) {
219 |         series_number = (3*list_size + series_size - 1) / series_size;
220 | 
221 |         if (series_number > 2) {
222 |             if (iteration & 1) {
223 |                 merge(mergefile2, mergefile1, series_number, sort_type);
224 |             }
225 |             else {
226 |                 merge(mergefile1, mergefile2, series_number, sort_type);
227 |             }
228 |         }
229 |         else {
230 |             if (iteration & 1) {
231 |                 merge(mergefile2, ofile, series_number, sort_type);
232 |             }
233 |             else {
234 |                 merge(mergefile1, ofile, series_number, sort_type);
235 |             }
236 |         }
237 | 
238 |         for (unsigned l = 0; l < series_number; l += 2) {
239 |             if (l == series_number - 1) {
240 |                 series_sizes[l/2] = series_sizes[l];
241 |             }
242 |             else {
243 |                 series_sizes[l/2] = series_sizes[l] + series_sizes[l + 1];
244 |             }
245 |         }
246 | 
247 |         series_size *= 2;
248 |         iteration++;
249 |     }
250 | }
251 | 
252 | bool join() {
253 |     block_size = RAM_SIZE / 3;
254 | 
255 |     a = memory;
256 |     b = memory + block_size;
257 |     c = memory + block_size * 2;
258 | 
259 |     fseek(sortedfile2, 0, SEEK_SET);
260 |     fseek(sortedfile1, 0, SEEK_SET);
261 |     fseek(joinedfile, 0, SEEK_SET);
262 | 
263 |     bool last_join = true;
264 |     unsigned b_i, b_n;
265 |     b_i = b_n = (unsigned)fread(b, 4, block_size, sortedfile1);
266 |     unsigned k = 0;
267 | 
268 |     for (unsigned i = 0; i < 3 * list_size; i += n_read) {
269 |         n_read = (unsigned)fread(a, 4, block_size, sortedfile2);
270 | 
271 |         for (unsigned j = 0; j < n_read; j += 3) {
272 |             while (a[j + 1] != b[k]) {
273 |                 k += 3;
274 |                 if (k == b_n) {
275 |                     b_n = (unsigned)fread(b, 4, block_size, sortedfile1);
276 |                     b_i += b_n;
277 |                     k = 0;
278 |                     if (b_n == 0 || b_i > 3*list_size) {
279 |                         fprintf(stderr, "Infinite cycle\n i=%d j=%d a[j+1]=%d block_size=%d", i, j, a[j+1], block_size);
280 |                         exit(2);
281 |                     }
282 |                 }
283 |             }
284 |             c[j] = a[j];
285 |             c[j + 1] = b[k + 1];
286 |             c[j + 2] = a[j + 2] + b[k + 2];
287 | 
288 |             if (last_join && c[j + 1] != root) {
289 | //                printf("%d w=%d root=%d\n", c[j+1], c[j+2], root);
290 |                 last_join = false;
291 |             }
292 |         }
293 |         fwrite(c, 4, n_read, joinedfile);
294 |     }
295 |     return last_join;
296 | }
297 | 
298 | unsigned find_root() {
299 |     block_size = RAM_SIZE / 3;
300 | 
301 |     a = memory;
302 |     b = memory + block_size;
303 |     c = memory + block_size * 2;
304 | 
305 |     fseek(sortedfile2, 0, SEEK_SET);
306 |     fseek(sortedfile1, 0, SEEK_SET);
307 |     fseek(tempfile, 0, SEEK_SET);
308 | 
309 |     unsigned b_i, b_n;
310 |     b_i = b_n = (unsigned)fread(b, 4, block_size, sortedfile1);
311 |     unsigned k = 0;
312 | 
313 |     for (unsigned i = 0; i < 3 * list_size; i += n_read) {
314 |         n_read = (unsigned)fread(a, 4, block_size, sortedfile2);
315 | 
316 |         for (unsigned j = 0; j < n_read; j += 3) {
317 |             while (a[j + 1] != b[k]) {
318 |                 if (a[j+1] < b[k]) {
319 |                     return a[j+1];
320 |                 }
321 |                 k += 3;
322 |                 if (k == b_n) {
323 |                     b_n = (unsigned)fread(b, 4, block_size, sortedfile1);
324 |                     b_i += b_n;
325 |                     k = 0;
326 |                     if (b_n == 0 || b_i > 3*list_size) {
327 |                         return a[j+1];
328 |                     }
329 |                 }
330 |             }
331 |             c[j] = a[j];
332 |             c[j + 1] = b[k + 1];
333 |             c[j + 2] = a[j + 2] + b[k + 2];
334 |         }
335 |         fwrite(c, 4, n_read, tempfile);
336 |     }
337 |     fprintf(stderr, "Root wasn't found\n");
338 |     exit(2);
339 | }
340 | 
341 | 
342 | inline void cyclic_join() {
343 |     while (!join()) {
344 |         merge_sort(joinedfile, sortedfile2, 2);
345 |         merge_sort(joinedfile, sortedfile1, 1);
346 |     }
347 | }
348 | 
349 | void create_weighted(const char* ifilename) {
350 |     block_size = RAM_SIZE / 5 * 2;
351 |     a = memory;
352 |     b = memory + block_size;
353 | 
354 |     FILE* ifile = fopen(ifilename, "rb");
355 |     fread(&list_size, 4, 1, ifile);
356 |     list_size--;
357 | 
358 |     fseek(tempfile, 0, SEEK_SET);
359 | 
360 |     for (unsigned i = 0; i < 2 * list_size; i += block_size) {
361 |         n_read = (unsigned) fread(a, 4, block_size, ifile);
362 |         for (unsigned j = 0; j < n_read / 2; ++j) {
363 |             b[3 * j] = a[2 * j];
364 |             b[3 * j + 1] = a[2 * j + 1];
365 |             b[3 * j + 2] = 1;
366 |         }
367 |         fwrite(b, 4, n_read / 2 * 3, tempfile);
368 |     }
369 | 
370 |     fclose(ifile);
371 | }
372 | 
373 | void add_root(FILE* file) {
374 |     fseek(file, 0, SEEK_END);
375 | 
376 |     unsigned triple[3];
377 |     triple[0] = root;
378 |     triple[1] = root;
379 |     triple[2] = 0;
380 |     fwrite(triple, 4, 3, file);
381 | 
382 |     list_size++;
383 | }
384 | 
385 | void print_results(const char* ofilename) {
386 |     block_size = RAM_SIZE / 5 * 3;
387 |     a = memory;
388 |     b = memory + block_size;
389 | 
390 |     fseek(tempfile, 0, SEEK_SET);
391 | 
392 |     FILE* ofile = fopen(ofilename, "wb");
393 | 
394 |     for (unsigned i = 0; i < 3*list_size; i += block_size) {
395 |         n_read = (unsigned) fread(a, 4, block_size, tempfile);
396 |         for (unsigned j = 0; j < n_read / 3; ++j) {
397 |             b[2*j] = a[3*j];
398 |             b[2*j+1] = a[3*j+2];
399 |         }
400 |         fwrite(b, 4, n_read / 3 * 2, ofile);
401 |     }
402 |     fclose(ofile);
403 | }
404 | 
405 | void ranking() {
406 |     open_files();
407 | 
408 |     create_weighted("input.bin");
409 | 
410 |     merge_sort(tempfile, sortedfile1, 1);
411 |     merge_sort(sortedfile1, sortedfile2, 2);
412 | 
413 |     root = find_root();
414 |     add_root(sortedfile2);
415 |     merge_sort(sortedfile2, sortedfile1, 1);
416 |     merge_sort(sortedfile1, sortedfile2, 2);
417 | 
418 |     cyclic_join();
419 |     merge_sort(joinedfile, tempfile, 1);
420 |     print_results("output.bin");
421 | 
422 |     close_files();
423 | }
424 | 
425 | int main() {
426 |     ranking();
427 |     delete[] memory;
428 | }
429 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/KnapsackProblem.java:
--------------------------------------------------------------------------------
  1 | import javafx.util.Pair;
  2 | 
  3 | import java.util.*;
  4 | 
  5 | public class KnapsackProblem {
  6 | 
  7 |     public static void main(String[] args) {
  8 |         Scanner scanner = new Scanner(System.in);
  9 |         int n = scanner.nextInt();
 10 |         int W = scanner.nextInt();
 11 | 
 12 |         Thing[] first = new Thing[n / 2];
 13 |         Thing[] second = new Thing[n - n / 2];
 14 |         int weight, cost;
 15 |         for (int i = 0; i < first.length; i++) {
 16 |             weight = scanner.nextInt();
 17 |             cost = scanner.nextInt();
 18 |             first[i] = new Thing(weight, cost, i + 1);
 19 |         }
 20 |         for (int i = 0; i < second.length; i++) {
 21 |             weight = scanner.nextInt();
 22 |             cost = scanner.nextInt();
 23 |             second[i] = new Thing(weight, cost, i + n / 2 + 1);
 24 |         }
 25 |         scanner.close();
 26 | 
 27 |         NavigableSet<Knapsack> firstSet = createKnapsackNavigableSet(first, W);
 28 |         List<Knapsack> secondList = createKnapsackList(second, W);
 29 | 
 30 |         first = null;
 31 |         second = null;
 32 | 
 33 |         Pair<Knapsack, Knapsack> bestKnapsacks = null;
 34 |         long bestCost = 0;
 35 |         long sumCost;
 36 |         for (Knapsack secondKnapsack : secondList) {
 37 |             Knapsack newKnapsack = new Knapsack();
 38 |             newKnapsack.setTotalWeight(W - secondKnapsack.getTotalWeight());
 39 |             Knapsack firstKnapsack = firstSet.floor(newKnapsack);
 40 | 
 41 |             sumCost = firstKnapsack.getTotalCost() + secondKnapsack.getTotalCost();
 42 |             if (bestCost < sumCost) {
 43 |                 bestKnapsacks = new Pair<>(firstKnapsack, secondKnapsack);
 44 |                 bestCost = sumCost;
 45 |             }
 46 |         }
 47 | 
 48 | 
 49 |         List<Integer> thingNumbers = bestKnapsacks.getKey().getThingNumbers();
 50 |         thingNumbers.addAll(bestKnapsacks.getValue().getThingNumbers());
 51 | 
 52 |         System.out.println(thingNumbers.size());
 53 |         thingNumbers.stream()
 54 |                 .sorted()
 55 |                 .forEach(el -> System.out.print(el + " "));
 56 |     }
 57 | 
 58 |     private static List<Knapsack> createKnapsackList(Thing[] things, int W) {
 59 |         int N = things.length;
 60 | 
 61 |         List<Knapsack> result = new ArrayList<>();
 62 |         result.add(new Knapsack());
 63 | 
 64 |         for (int mask = 0; mask < (1 << N); mask++) {
 65 |             Knapsack knapsack = new Knapsack();
 66 |             for (int j = 0; j < N; j++) {
 67 |                 if ((mask & (1 << j)) != 0) {
 68 |                     knapsack.add(things[j]);
 69 |                 }
 70 |             }
 71 |             if (knapsack.getTotalWeight() <= W) {
 72 |                 result.add(knapsack);
 73 |             }
 74 |         }
 75 |         return result;
 76 |     }
 77 | 
 78 |     private static NavigableSet<Knapsack> createKnapsackNavigableSet(Thing[] things, int W) {
 79 |         int N = things.length;
 80 | 
 81 |         NavigableSet<Knapsack> result = new TreeSet<>();
 82 | 
 83 |         result.add(new Knapsack());
 84 | 
 85 |         for (int mask = 0; mask < (1 << N); mask++) {
 86 |             Knapsack knapsack = new Knapsack();
 87 |             for (int j = 0; j < N; j++) {
 88 |                 if ((mask & (1 << j)) != 0) {
 89 |                     knapsack.add(things[j]);
 90 |                 }
 91 |             }
 92 |             if (knapsack.getTotalWeight() <= W) {
 93 |                 Knapsack floorKnapsack = result.floor(knapsack);
 94 |                 if (floorKnapsack == null) {
 95 |                     result.add(knapsack);
 96 |                 }
 97 |                 else if (floorKnapsack.getTotalCost() < knapsack.getTotalCost()) {
 98 |                     if (floorKnapsack.getTotalWeight() == knapsack.getTotalWeight()) {
 99 |                         result.remove(floorKnapsack);
100 |                     }
101 |                     result.add(knapsack);
102 |                 }
103 | 
104 |                 Knapsack higherKnapsack = result.higher(knapsack);
105 |                 while (higherKnapsack != null) {
106 |                     if (knapsack.getTotalCost() >= higherKnapsack.getTotalCost()) {
107 |                         result.remove(higherKnapsack);
108 |                     }
109 |                     else {
110 |                         break;
111 |                     }
112 |                     higherKnapsack = result.higher(knapsack);
113 |                 }
114 |             }
115 |         }
116 |         return result;
117 |     }
118 | }
119 | 
120 | class Knapsack implements Comparable<Knapsack>{
121 | 
122 |     long totalWeight = 0;
123 |     long totalCost = 0;
124 |     List<Integer> thingNumbers = new ArrayList<>();
125 | 
126 |     public Knapsack() {
127 |     }
128 | 
129 |     public void add(Thing thing) {
130 |         thingNumbers.add(thing.getNumber());
131 |         totalWeight += thing.getWeight();
132 |         totalCost += thing.getCost();
133 |     }
134 | 
135 |     public long getTotalWeight() {
136 |         return totalWeight;
137 |     }
138 | 
139 |     public long getTotalCost() {
140 |         return totalCost;
141 |     }
142 | 
143 |     public List<Integer> getThingNumbers() {
144 |         return thingNumbers;
145 |     }
146 | 
147 |     @Override
148 |     public int compareTo(Knapsack knapsack) {
149 |         if (totalWeight != knapsack.getTotalWeight()) {
150 |             return totalWeight < knapsack.getTotalWeight() ? -1 : 1;
151 |         }
152 |         else return 0;
153 |     }
154 | 
155 |     public void setTotalWeight(long totalWeight) {
156 |         this.totalWeight = totalWeight;
157 |     }
158 | }
159 | 
160 | class Thing {
161 | 
162 |     int weight;
163 |     int cost;
164 |     int number;
165 | 
166 |     public Thing(int weight, int cost, int number) {
167 |         this.weight = weight;
168 |         this.cost = cost;
169 |         this.number = number;
170 |     }
171 | 
172 |     public int getWeight() {
173 |         return weight;
174 |     }
175 | 
176 |     public int getCost() {
177 |         return cost;
178 |     }
179 | 
180 |     public int getNumber() {
181 |         return number;
182 |     }
183 | }
184 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/SegmentTree.java:
--------------------------------------------------------------------------------
  1 | import java.util.Scanner;
  2 | 
  3 | public class SegmentTree {
  4 | 
  5 |     private Node[] tree;
  6 | 
  7 |     private final int BEGIN = 0;
  8 |     private final int END;
  9 |     private final int ROOT = 0;
 10 | 
 11 |     public static void main(String... arg) throws InterruptedException {
 12 |         Thread t = new Thread(null, null, "", 4 * 1024 * 1024) {
 13 |             @Override
 14 |             public void run() {
 15 |                 Scanner scanner = new Scanner(System.in);
 16 |                 int n = scanner.nextInt();
 17 |                 int q = scanner.nextInt();
 18 | 
 19 |                 int[] array = new int[n];
 20 |                 for (int i = 0; i < n; i++) {
 21 |                     array[i] = scanner.nextInt();
 22 |                 }
 23 | 
 24 |                 SegmentTree tree = new SegmentTree(array.length);
 25 |                 tree.init(array);
 26 | 
 27 |                 for (int i = 0; i < q; i++) {
 28 |                     int queryType = scanner.nextInt();
 29 |                     int num1 = scanner.nextInt();
 30 |                     int num2 = scanner.nextInt();
 31 | 
 32 |                     switch (queryType) {
 33 |                         case 1:
 34 |                             tree.update(num1 - 1, num2);
 35 |                             break;
 36 |                         case 2:
 37 |                             tree.increment(num1 - 1, num2 - 1);
 38 |                             break;
 39 |                         case 3:
 40 | //                            System.out.println(String.format("%d - %d: %d", num1 - 1, num2 - 1, tree.sum(num1 - 1, num2 - 1, true)));
 41 |                             System.out.println(tree.sum(num1 - 1, num2 - 1, true));
 42 |                             break;
 43 |                         case 4:
 44 | //                            System.out.println(String.format("%d - %d: %d", num1 - 1, num2 - 1, tree.sum(num1 - 1, num2 - 1, false)));
 45 |                             System.out.println(tree.sum(num1 - 1, num2 - 1, false));
 46 |                             break;
 47 |                     }
 48 |                 }
 49 |                 scanner.close();
 50 |             }
 51 |         };
 52 |         t.start();
 53 |         t.join();
 54 |     }
 55 | 
 56 |     public SegmentTree(int size) {
 57 |         int h = (int) (Math.ceil(Math.log(size) / Math.log(2)));
 58 |         int N = 2 * (int) Math.pow(2, h) - 1;
 59 |         tree = new Node[N];
 60 |         END = size - 1;
 61 |     }
 62 | 
 63 |     public void init(int[] arr) {
 64 |         init(arr, BEGIN, END, ROOT);
 65 |     }
 66 | 
 67 |     public long sum(int l, int r, boolean forEven) {
 68 |         return sum(BEGIN, END, l, r, ROOT, forEven);
 69 |     }
 70 | 
 71 |     public void increment(int l, int r) {
 72 |         increment(BEGIN, END, l, r, ROOT);
 73 |     }
 74 | 
 75 |     public void update(int pos, int value) {
 76 |         update(BEGIN, END, pos, value, ROOT);
 77 |     }
 78 | 
 79 |     private Node init(int[] arr, int start, int end, int current) {
 80 |         if (start == end) {
 81 |             tree[current] = new Node(arr[start]);
 82 |             return tree[current];
 83 |         }
 84 | 
 85 |         int middle = middle(start, end);
 86 | 
 87 |         tree[current] = new Node(
 88 |                 init(arr, start, middle, left(current)),
 89 |                 init(arr, middle + 1, end, right(current)));
 90 | 
 91 |         return tree[current];
 92 |     }
 93 | 
 94 |     private void update(int start, int end, int updatePos, int value, int current) {
 95 |         if (updatePos < start || updatePos > end) {
 96 |             return;
 97 |         }
 98 | 
 99 |         push(current);
100 | 
101 |         if (start == end) {
102 |             tree[current] = new Node(value);
103 |         } else {
104 |             int middle = middle(start, end);
105 |             update(start, middle, updatePos, value, left(current));
106 |             update(middle + 1, end, updatePos, value, right(current));
107 | 
108 |             refresh(current);
109 |         }
110 |     }
111 | 
112 |     private long sum(int start, int end, int l, int r, int current, boolean forEven) {
113 |         if (end < l || start > r) {
114 |             return 0;
115 |         }
116 | 
117 |         push(current);
118 | 
119 |         if (l <= start && r >= end) {
120 |             refresh(current);
121 |             return forEven ? tree[current].getEvenSum() : tree[current].getOddSum();
122 |         } else {
123 |             int middle = middle(start, end);
124 |             long sum = sum(start, middle, l, r, left(current), forEven) +
125 |             sum(middle + 1, end, l, r, right(current), forEven);
126 |             refresh(current);
127 |             return sum;
128 |         }
129 |     }
130 | 
131 |     private void increment(int start, int end, int l, int r, int current) {
132 |         if (end < l || start > r) {
133 |             return;
134 |         }
135 | 
136 |         if (l <= start && r >= end) {
137 |             tree[current].increment();
138 |             push(current);
139 |             refresh(current);
140 |         } else {
141 |             push(current);
142 |             int middle = middle(start, end);
143 |             increment(start, middle, l, r, left(current));
144 |             increment(middle + 1, end, l, r, right(current));
145 | 
146 |             refresh(current);
147 |         }
148 |     }
149 | 
150 |     private void refresh(int current) {
151 |         if (left(current) < tree.length && tree[left(current)] != null) {
152 |             tree[current].refresh(tree[left(current)], tree[right(current)]);
153 |         }
154 |     }
155 | 
156 |     private void push(int current) {
157 |         int increments = tree[current].getIncrements();
158 |         if (increments != 0) {
159 |             if (left(current) < tree.length && tree[left(current)] != null) {
160 |                 tree[left(current)].addToIncrements(increments);
161 |                 tree[right(current)].addToIncrements(increments);
162 |             } else {
163 |                 if (increments % 2 == 1) {
164 |                     swapEvenAndOdd(tree[current]);
165 |                 }
166 |                 tree[current].setEvenSum(tree[current].getEvenSum() + tree[current].getEvenAmount() * increments);
167 |                 tree[current].setOddSum(tree[current].getOddSum() + tree[current].getOddAmount() * increments);
168 |             }
169 |             tree[current].setIncrements(0);
170 |         }
171 |     }
172 | 
173 |     private void swapEvenAndOdd(Node node) {
174 |         long temp = node.getEvenSum();
175 |         node.setEvenSum(node.getOddSum());
176 |         node.setOddSum(temp);
177 | 
178 |         int temp2 = node.getEvenAmount();
179 |         node.setEvenAmount(node.getOddAmount());
180 |         node.setOddAmount(temp2);
181 |     }
182 | 
183 |     private int left(int nodeIndex) {
184 |         return 2 * nodeIndex + 1;
185 |     }
186 | 
187 |     private int right(int nodeIndex) {
188 |         return 2 * nodeIndex + 2;
189 |     }
190 | 
191 | 
192 |     private int middle(int start, int end) {
193 |         return (start + (end - start) / 2);
194 |     }
195 | 
196 |     class Node {
197 | 
198 |         private int increments = 0;
199 |         private long oddSum = 0;
200 |         private long evenSum = 0;
201 |         private int oddAmount = 0;
202 |         private int evenAmount = 0;
203 | 
204 |         public Node() {
205 |         }
206 | 
207 |         public Node(int value) {
208 |             if (value % 2 == 0) {
209 |                 evenSum = value;
210 |                 evenAmount = 1;
211 |             } else {
212 |                 oddSum = value;
213 |                 oddAmount = 1;
214 |             }
215 |         }
216 | 
217 |         public Node(Node left, Node right) {
218 |             refresh(left, right);
219 |         }
220 | 
221 |         public void refresh(Node left, Node right) {
222 |             evenSum = left.calculateEvenSum() + right.calculateEvenSum();
223 |             oddSum = left.calculateOddSum() + right.calculateOddSum();
224 |             evenAmount = left.calculateEvenAmount() + right.calculateEvenAmount();
225 |             oddAmount = left.calculateOddAmount() + right.calculateOddAmount();
226 |         }
227 | 
228 |         public long calculateEvenSum() {
229 |             return increments % 2 == 0
230 |             ? evenSum + evenAmount * increments
231 |             : oddSum + oddAmount * increments;
232 |         }
233 | 
234 |         public long calculateOddSum() {
235 |             return increments % 2 == 0
236 |             ? oddSum + oddAmount * increments
237 |             : evenSum + evenAmount * increments;
238 |         }
239 | 
240 |         public int calculateEvenAmount() {
241 |             return increments % 2 == 0 ? evenAmount : oddAmount;
242 |         }
243 | 
244 |         public int calculateOddAmount() {
245 |             return increments % 2 == 0 ? oddAmount : evenAmount;
246 |         }
247 | 
248 |         public void addToIncrements(long value) {
249 |             increments += value;
250 |         }
251 | 
252 |         public void increment() {
253 |             increments++;
254 |         }
255 | 
256 |         public int getIncrements() {
257 |             return increments;
258 |         }
259 | 
260 |         public void setIncrements(int increments) {
261 |             this.increments = increments;
262 |         }
263 | 
264 |         public long getOddSum() {
265 |             return oddSum;
266 |         }
267 | 
268 |         public void setOddSum(long oddSum) {
269 |             this.oddSum = oddSum;
270 |         }
271 | 
272 |         public long getEvenSum() {
273 |             return evenSum;
274 |         }
275 | 
276 |         public void setEvenSum(long evenSum) {
277 |             this.evenSum = evenSum;
278 |         }
279 | 
280 |         public int getOddAmount() {
281 |             return oddAmount;
282 |         }
283 | 
284 |         public void setOddAmount(int oddAmount) {
285 |             this.oddAmount = oddAmount;
286 |         }
287 | 
288 |         public int getEvenAmount() {
289 |             return evenAmount;
290 |         }
291 | 
292 |         public void setEvenAmount(int evenAmount) {
293 |             this.evenAmount = evenAmount;
294 |         }
295 |     }
296 | }
297 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/bloom_filter.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <fstream>
 3 | #include <vector>
 4 | using namespace std;
 5 | 
 6 | int custom_hash(string s) {
 7 |     int hash = 0;
 8 |     for (char i : s) {
 9 |         hash += i;
10 |         hash += (hash << 10);
11 |         hash ^= (hash >> 6);
12 |     }
13 |     hash += (hash << 3);
14 |     hash ^= (hash >> 11);
15 |     hash += (hash << 15);
16 |     return hash;
17 | }
18 | 
19 | std::hash<std::string> hash_fn;
20 | 
21 | int compute_hash(const string &s, const vector<bool> &bits, int i) {
22 |     return abs((int)((hash_fn(s) + (i * custom_hash(s))) % bits.size()));
23 | }
24 | 
25 | void add(const string &s, int functions_count, vector<bool> &bits) {
26 |     for (int i = 0; i < functions_count; i++) {
27 |         bits[compute_hash(s, bits, i)] = true;
28 |     }
29 | }
30 | 
31 | bool contains(const string &s, int functions_count, const vector<bool> &bits) {
32 |     for (int i = 0; i < functions_count; i++) {
33 |         if (!bits[compute_hash(s, bits, i)])
34 |             return false;
35 |     }
36 |     return true;
37 | }
38 | 
39 | int main() {
40 |     const int INPUT_SIZE = 50 * 1024;
41 |     const int OUTPUT_SIZE = 50 * 1024;
42 |     const int SIZE = 11200 * 1024;
43 |     const int HASHES_NUM = 6;
44 |     int queries_num, query_type;
45 |     string str;
46 |     bool exists;
47 |     vector<bool> filter(SIZE, false);
48 | 
49 |     auto * input_buf = new char[INPUT_SIZE];
50 |     auto * output_buf = new char[OUTPUT_SIZE];
51 | 
52 |     ifstream ifile("input.txt");
53 |     ifile.rdbuf()->pubsetbuf(input_buf, SIZE);
54 | 
55 |     ofstream ofile("output.txt");
56 |     ofile.rdbuf()->pubsetbuf(output_buf, SIZE);
57 | 
58 |     ifile >> queries_num;
59 |     for (int i = 0; i < queries_num; ++i) {
60 |         ifile >> query_type >> str;
61 | 
62 |         if(query_type == 1) {
63 |             add(str, HASHES_NUM, filter);
64 |         }
65 |         else {
66 |             exists = contains(str, HASHES_NUM, filter);
67 |             ofile << exists;
68 |         }
69 |     }
70 | 
71 |     ifile.close();
72 |     ofile.close();
73 | 
74 |     delete(input_buf);
75 |     delete(output_buf);
76 | }
77 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/cache.pptx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/internal_memory_algorithms/cache_miss/cache.pptx


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/cache_emulator/Cache.java:
--------------------------------------------------------------------------------
  1 | import java.util.Scanner;
  2 | 
  3 | public class Cache {
  4 | 
  5 |     private int cacheSize = 32768;
  6 |     private int banksCount = 4;
  7 |     private int lineSize = 64;
  8 | 
  9 |     private int linesInBank;
 10 | 
 11 |     private Cell[][] cache;
 12 | 
 13 |     private long ageCounter = 0;
 14 |     private long hitCounter = 0;
 15 |     private long missCounter = 0;
 16 | 
 17 |     public static void main(String[] args) {
 18 |         Scanner scanner = new Scanner(System.in);
 19 |         int cacheSize = scanner.nextInt();
 20 |         int banksCount = scanner.nextInt();
 21 |         int lineSize = scanner.nextInt();
 22 |         int n = scanner.nextInt();
 23 | 
 24 |         Cache cache = new Cache(cacheSize, banksCount, lineSize);
 25 | 
 26 |         for (int i = 0; i < n; i++) {
 27 |             cache.get(scanner.nextInt());
 28 |         }
 29 | 
 30 |         System.out.println(cache.getHitCounter() + "\t" + cache.getMissCounter());
 31 |     }
 32 | 
 33 |     public Cache(int cacheSize, int banksCount, int lineSize) {
 34 |         this.cacheSize = cacheSize;
 35 |         this.banksCount = banksCount;
 36 |         this.lineSize = lineSize;
 37 | 
 38 |         linesInBank = (cacheSize / lineSize) / banksCount;
 39 | 
 40 |         cache = new Cell[banksCount][linesInBank];
 41 | 
 42 |         for (int i = 0; i < cache.length; i++) {
 43 |             for (int j = 0; j < cache[i].length; j++) {
 44 |                 cache[i][j] = new Cell(0, -1);
 45 |             }
 46 |         }
 47 |     }
 48 | 
 49 |     public boolean get(int address) {
 50 |         int lineNumber = address / lineSize;
 51 |         int lineIndex = lineNumber % linesInBank;
 52 | 
 53 |         int oldestCellIndex = -1;
 54 |         long minAge = Long.MAX_VALUE;
 55 | 
 56 |         for (int i = 0; i < banksCount; i++) {
 57 |             if (cache[i][lineIndex].getLine() == lineNumber) {
 58 |                 cache[i][lineIndex] = new Cell(++ageCounter, lineNumber);
 59 |                 hitCounter++;
 60 |                 return true;
 61 |             }
 62 |             else if (cache[i][lineIndex].getAge() < minAge) {
 63 |                 oldestCellIndex = i;
 64 |                 minAge = cache[i][lineIndex].getAge();
 65 |             }
 66 |         }
 67 |         cache[oldestCellIndex][lineIndex] = new Cell(++ageCounter, lineNumber);
 68 |         missCounter++;
 69 |         return false;
 70 |     }
 71 | 
 72 |     public long getHitCounter() {
 73 |         return hitCounter;
 74 |     }
 75 | 
 76 |     public long getMissCounter() {
 77 |         return missCounter;
 78 |     }
 79 | 
 80 |     private class Cell {
 81 | 
 82 |         private long age;
 83 | 
 84 |         private int line;
 85 | 
 86 |         public Cell(long age, int line) {
 87 |             this.age = age;
 88 |             this.line = line;
 89 |         }
 90 | 
 91 |         public long getAge() {
 92 |             return age;
 93 |         }
 94 | 
 95 |         public int getLine() {
 96 |             return line;
 97 |         }
 98 |     }
 99 | }
100 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/cache_emulator/hard-info.txt:
--------------------------------------------------------------------------------
1 | L1 (data) 8-way set-associative, 64 sets, 32KB size
2 | L3 (unified) 12-way set-associative, 4096 sets, 32072KB size


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/cache_misses.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/internal_memory_algorithms/cache_miss/cache_misses.pdf


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/matrix_mult/cache:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/internal_memory_algorithms/cache_miss/matrix_mult/cache


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/matrix_mult/compile.sh:
--------------------------------------------------------------------------------
1 | g++ -g -std=c++11 main.cpp -o cache
2 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/matrix_mult/main.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <random>
  3 | #include <ctime>
  4 | 
  5 | 
  6 | //__declspec(align(16))
  7 | // __restrict
  8 | namespace {
  9 |     void MultSimple(const float* __restrict a, const float* __restrict b, float* __restrict c, int n)
 10 |     {
 11 |         for (int i = 0; i < n; ++i) {
 12 |             for (int j = 0; j < n; ++j) {
 13 |                 c[i * n + j] = 0.f;
 14 |                 for (int k = 0; k < n; ++k) {
 15 |                     c[i * n + j] += a[i * n + k] * b[k * n + j];
 16 |                 }
 17 |             }
 18 |         }
 19 |     }
 20 | 
 21 |     void MultSimpleBlock(const float* __restrict a, const float* __restrict b, float* __restrict c, int n)
 22 |     {
 23 |         const int BLOCK = 10;
 24 |         for (int i = 0; i < n; i += BLOCK) {
 25 |             for (int j = 0; j < n; j += BLOCK) {
 26 |                 for (int k = 0; k < n; k += BLOCK) {
 27 |                     for (int ib = 0; ib < BLOCK && i + ib < n; ++ib) {
 28 |                         for (int jb = 0; jb < BLOCK && j + jb < n; ++jb) {
 29 |                             if (k == 0) 
 30 |                                 c[(i + ib) * n + (j + jb)] = 0.f;
 31 |                             for (int kb = 0; k + kb < n && kb < BLOCK; ++kb) {
 32 |                                 c[(i + ib) * n + j + jb] += a[(i + ib) * n + k + kb] * b[(k + kb) * n + j + jb];
 33 |                             }                        
 34 |                         }
 35 |                     }
 36 |                 }
 37 |             }
 38 |         }
 39 |     }
 40 |     
 41 |     void MultSwappedLoops(const float* __restrict a, const float* __restrict b, float* __restrict c, int n)
 42 |     {
 43 |         for (int i = 0; i < n; ++i) {
 44 |             for (int j = 0; j < n; ++j) {
 45 |                 c[i * n + j] = 0.f;
 46 |             }
 47 | 
 48 |             for (int k = 0; k < n; ++k) {
 49 |                 for (int j = 0; j < n; ++j) {
 50 |                     c[i * n + j] += a[i * n + k] * b[k * n + j];
 51 |                 }
 52 |             }
 53 |         }
 54 |     }
 55 | 
 56 |     void FillRandom(float* a, int n)
 57 |     {
 58 |         std::default_random_engine eng;
 59 |         std::uniform_real_distribution<float> dist;
 60 | 
 61 |         for (int i = 0; i < n; ++i) {
 62 |             for (int j = 0; j < n; ++j) {
 63 |                 a[i * n + j] = dist(eng);
 64 |             }
 65 |         }
 66 |     }
 67 | }
 68 | 
 69 | int main(int argc, char* argv[])
 70 | {
 71 |     const int n = atoi(argv[1]);
 72 |     std::cerr << "n = " << n << std::endl;
 73 |     
 74 |     float* a = new float[n * n];
 75 |     float* b = new float[n * n];
 76 |     float* c = new float[n * n];
 77 | 
 78 |     FillRandom(a, n);
 79 |     FillRandom(b, n);
 80 | 
 81 |     {
 82 |         const auto startTime = std::clock();
 83 |         MultSimple(a, b, c, n);
 84 |         const auto endTime = std::clock();
 85 | 
 86 |         std::cerr << "timeSimple: " << double(endTime - startTime) / CLOCKS_PER_SEC << '\n';
 87 |     }
 88 | 
 89 |     if (false) {
 90 |         const auto startTime = std::clock();
 91 |         MultSimpleBlock(a, b, c, n);
 92 |         const auto endTime = std::clock();
 93 | 
 94 |         std::cerr << "timeBlock: " << double(endTime - startTime) / CLOCKS_PER_SEC << '\n';
 95 |     }
 96 |     
 97 |     if (false) {
 98 |         const auto startTime = std::clock();
 99 |         MultSwappedLoops(a, b, c, n);
100 |         const auto endTime = std::clock();
101 | 
102 |         std::cerr << "timeSwapped: " << double(endTime - startTime) / CLOCKS_PER_SEC << '\n';
103 |     }
104 |     
105 |     delete[] a;
106 |     delete[] b;
107 |     delete[] c;
108 | }
109 | 
110 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/matrix_mult/matrix_mult.ods:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/internal_memory_algorithms/cache_miss/matrix_mult/matrix_mult.ods


--------------------------------------------------------------------------------
/internal_memory_algorithms/cache_miss/matrix_mult/run.sh:
--------------------------------------------------------------------------------
1 | ./cache 1023
2 | ./cache 1024
3 | ./cache 1025
4 | ./cache 1055
5 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/fast_transposition.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <vector>
 3 | 
 4 | using namespace std;
 5 | 
 6 | vector<int> generate_input(int n, int seed) {
 7 |     vector<int> d(n * n);
 8 |     for (size_t i = 0; i < d.size(); ++i) {
 9 |         d[i] = seed;
10 |         seed = ((long long) seed * 197 + 2017) & 987654;
11 |     }
12 |     return d;
13 | }
14 | 
15 | long long get_hash(const vector<int> &d) {
16 |     const long long MOD = 987654321054321LL;
17 |     const long long MUL = 179;
18 | 
19 |     long long result_value = 0;
20 |     for (size_t i = 0; i < d.size(); ++i)
21 |         result_value = (result_value * MUL + d[i]) & MOD;
22 |     return result_value;
23 | }
24 | 
25 | void transpose(vector<int> &m, int imin, int jmin, int side, int n) {
26 |     const int BLOCK = 8;
27 |     int tmp;
28 |     for (int i = 0; i < side; i += BLOCK) {
29 |         for (int j = i; j < side; j += BLOCK) {
30 |             for (int ib = i; ib < i + BLOCK && ib < side; ++ib) {
31 |                 for (int jb = (j <= ib) ? ib + 1 : j; jb < j + BLOCK && jb < side; ++jb) {
32 |                     /*tmp = m[(imin + ib) * n + (jmin + jb)];
33 |                     m[(imin + ib) * n + (jmin + jb)] = m[(imin + jb) * n + (jmin + ib)];
34 |                     m[(imin + jb) * n + (jmin + ib)] = tmp;*/
35 |                     m[(imin + ib) * n + (jmin + jb)] ^= m[(imin + jb) * n + (jmin + ib)];
36 |                     m[(imin + jb) * n + (jmin + ib)] ^= m[(imin + ib) * n + (jmin + jb)];
37 |                     m[(imin + ib) * n + (jmin + jb)] ^= m[(imin + jb) * n + (jmin + ib)];
38 |                 }
39 |             }
40 |         }
41 |     }
42 | }
43 | 
44 | void print(vector<int> &m, int n) {
45 |     for (int i = 0; i < n; ++i) {
46 |         for (int j = 0; j < n; ++j) {
47 |             cout << m[i * n + j] << "\t";
48 |         }
49 |         cout << endl;
50 |     }
51 |     cout << endl;
52 | }
53 | 
54 | int main() {
55 |     int n, k, imin, jmin, side;
56 |     int seed;
57 |     cin >> n;
58 |     cin >> seed;
59 |     cin >> k;
60 | 
61 |     vector<int> m = generate_input(n, seed);
62 | 
63 |     for (int i = 0; i < k; ++i) {
64 |         cin >> imin;
65 |         cin >> jmin;
66 |         cin >> side;
67 |         transpose(m, imin, jmin, side, n);
68 |     }
69 | 
70 |     cout << get_hash(m);
71 |     return 0;
72 | }


--------------------------------------------------------------------------------
/internal_memory_algorithms/hitters.cpp:
--------------------------------------------------------------------------------
 1 | #include <iostream>
 2 | #include <vector>
 3 | #include <map>
 4 | 
 5 | using namespace std;
 6 | 
 7 | int main() {
 8 |     int k;
 9 |     cin >> k;
10 |     k--;
11 | 
12 |     map<int, int> hitters;
13 |     map<int, int>::iterator it;
14 | 
15 |     int x = 1;
16 |     while (x != -1) {
17 |         scanf("%d", &x);
18 |         if (x == 0) {
19 |             for (auto el : hitters) {
20 |                 printf("%d ", el.first);
21 |             }
22 | 
23 |             int value = hitters.begin()->first;
24 |             for(int i = k - (int)hitters.size(); i > 0; --i) {
25 |                 printf("%d ", value);
26 |             }
27 |             printf("\n");
28 |         }
29 |         else {
30 |             if ((it = hitters.find(x)) != hitters.end()) {
31 |                 it->second++;
32 |             }
33 |             else if (hitters.size() < k) {
34 |                 hitters[x] = 1;
35 |             }
36 |             else {
37 |                 bool saved = false;
38 |                 int saved_key;
39 |                 int saved_value;
40 |                 for (it = hitters.begin(); it != hitters.end();) {
41 |                     it->second--;
42 | 
43 |                     if (!saved && it->second >= 0) {
44 |                         saved = true;
45 |                         saved_key = it->first;
46 |                         saved_value = it->second;
47 |                     }
48 | 
49 |                     if (it->second <= 0)
50 |                         it = hitters.erase(it);
51 |                     else
52 |                         it++;
53 |                 }
54 | 
55 |                 if (hitters.empty())
56 |                     hitters[saved_key] = saved_value;
57 |             }
58 |         }
59 |     }
60 | }
61 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/matrix_multiplication.cpp:
--------------------------------------------------------------------------------
  1 | #include <iostream>
  2 | #include <random>
  3 | #include <ctime>
  4 | 
  5 | 
  6 | //__declspec(align(16))
  7 | // __restrict
  8 | namespace {
  9 |     void MultSimple(const double* __restrict a, const double* __restrict b, double* __restrict c, int n)
 10 |     {
 11 |         for (int i = 0; i < n; ++i) {
 12 |             for (int j = 0; j < n; ++j) {
 13 |                 c[i * n + j] = 0.0;
 14 |                 for (int k = 0; k < n; ++k) {
 15 |                     c[i * n + j] += a[i * n + k] * b[k * n + j];
 16 |                 }
 17 |             }
 18 |         }
 19 |     }
 20 | 
 21 |     void MultSimpleBlock(const double* __restrict a, const double* __restrict b, double* __restrict c, int n)
 22 |     {
 23 |         const int BLOCK = 10;
 24 |         for (int i = 0; i < n; i += BLOCK) {
 25 |             for (int j = 0; j < n; j += BLOCK) {
 26 |                 for (int k = 0; k < n; k += BLOCK) {
 27 |                     for (int ib = 0; ib < BLOCK && i + ib < n; ++ib) {
 28 |                         for (int jb = 0; jb < BLOCK && j + jb < n; ++jb) {
 29 |                             if (k == 0) 
 30 |                                 c[(i + ib) * n + (j + jb)] = 0.0;
 31 |                             for (int kb = 0; k + kb < n && kb < BLOCK; ++kb) {
 32 |                                 c[(i + ib) * n + j + jb] += a[(i + ib) * n + k + kb] * b[(k + kb) * n + j + jb];
 33 |                             }                        
 34 |                         }
 35 |                     }
 36 |                 }
 37 |             }
 38 |         }
 39 |     }
 40 |     
 41 |     void MultSwappedLoops(const double* __restrict a, const double* __restrict b, double* __restrict c, int n)
 42 |     {
 43 |         for (int i = 0; i < n; ++i) {
 44 |             for (int j = 0; j < n; ++j) {
 45 |                 c[i * n + j] = 0.0;
 46 |             }
 47 | 
 48 |             for (int k = 0; k < n; ++k) {
 49 |                 for (int j = 0; j < n; ++j) {
 50 |                     c[i * n + j] += a[i * n + k] * b[k * n + j];
 51 |                 }
 52 |             }
 53 |         }
 54 |     }
 55 | 
 56 |     void FillRandom(double* a, int n)
 57 |     {
 58 |         std::default_random_engine eng;
 59 |         std::uniform_real_distribution<double> dist;
 60 | 
 61 |         for (int i = 0; i < n; ++i) {
 62 |             for (int j = 0; j < n; ++j) {
 63 |                 a[i * n + j] = dist(eng);
 64 |             }
 65 |         }
 66 |     }
 67 | }
 68 | 
 69 | int main(int argc, char* argv[])
 70 | {
 71 |     const int n = atoi(argv[1]);
 72 |     //std::cerr << "n = " << n << std::endl;
 73 |     
 74 |     double* a = new double[n * n];
 75 |     double* b = new double[n * n];
 76 |     double* c = new double[n * n];
 77 | 
 78 |     FillRandom(a, n);
 79 |     FillRandom(b, n);
 80 |     
 81 |     {
 82 |         const auto startTime = std::clock();
 83 |         MultSimple(a, b, c, n);
 84 |         const auto endTime = std::clock();
 85 | 
 86 |        std::cerr << "timeSimple: " << double(endTime - startTime) / CLOCKS_PER_SEC << '\n';
 87 |     }
 88 | 
 89 |     {
 90 |         const auto startTime = std::clock();
 91 |         MultSimpleBlock(a, b, c, n);
 92 |         const auto endTime = std::clock();
 93 | 
 94 |         std::cerr << "timeBlock: " << double(endTime - startTime) / CLOCKS_PER_SEC << '\n';
 95 |     }
 96 | 
 97 |     {
 98 |         const auto startTime = std::clock();
 99 |         MultSwappedLoops(a, b, c, n);
100 |         const auto endTime = std::clock();
101 | 
102 |         std::cerr << "timeSwapped: " << double(endTime - startTime) / CLOCKS_PER_SEC << '\n';
103 |     }
104 | 
105 |     delete[] a;
106 |     delete[] b;
107 |     delete[] c;
108 | }
109 | 
110 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/web_graph/GraphGenerator.java:
--------------------------------------------------------------------------------
  1 | import java.util.*;
  2 | import java.util.stream.Collectors;
  3 | import java.util.stream.IntStream;
  4 | import java.util.stream.Stream;
  5 | 
  6 | public class GraphGenerator {
  7 | 
  8 |     private int n;
  9 |     private int m;
 10 | 
 11 |     private List<Set<Integer>> graph;
 12 |     private int[] degrees;
 13 |     private Set<BitSet> components;
 14 |     private int[][] distances;
 15 |     private double avgDistance;
 16 |     private int diameter;
 17 |     private int[] degreeDistribution;
 18 | 
 19 |     public static void main(String[] args) throws Exception {
 20 | 
 21 |         int diameterSum = 0;
 22 |         double avgDistanceSum = 0;
 23 |         int N = 100;
 24 |         int degreesCount = 100;
 25 |         double[] degreeDistributionSum = new double[degreesCount];
 26 | 
 27 |         for (int i = 1; i < N+1; i++) {
 28 |             GraphGenerator graphGenerator = new GraphGenerator();
 29 |             graphGenerator.generate(4000, 16);
 30 |             graphGenerator.calculateCharacteristics();
 31 | 
 32 |             diameterSum += graphGenerator.getDiameter();
 33 |             avgDistanceSum += graphGenerator.getAvgDistance();
 34 |             for (int j = 0; j < degreeDistributionSum.length; j++) {
 35 |                 degreeDistributionSum[j] += graphGenerator.getDegreeDistribution()[j];
 36 |             }
 37 |             System.out.println("iteration:" + i);
 38 | 
 39 |             if (i % 100 == 0) {
 40 |                 double factor = 1.0 / i;
 41 |                 System.out.println("Diameter:");
 42 |                 System.out.println(diameterSum * factor);
 43 |                 System.out.println("Average distance:");
 44 |                 System.out.println(avgDistanceSum * factor);
 45 |                 System.out.println("Degrees:");
 46 |                 for (int j = 0; j < degreeDistributionSum.length; j++) {
 47 |                     System.out.println(j + "\t" + degreeDistributionSum[j] * factor);
 48 |                 }
 49 |             }
 50 |         }
 51 | 
 52 | 
 53 | 
 54 |         /*GraphGenerator graphGenerator = new GraphGenerator();
 55 |         graphGenerator.generate(4000, 16);
 56 |         graphGenerator.calculateCharacteristics();
 57 | 
 58 |         System.out.println("Graph:");
 59 |         System.out.println("Diameter: " + graphGenerator.getDiameter());
 60 |         System.out.println("Average distance:" + graphGenerator.getAvgDistance());
 61 | 
 62 |         System.out.println("Degrees: ");
 63 |         Arrays.stream(graphGenerator.getDegrees()).forEach(el -> System.out.print(el + ",\t"));
 64 |         System.out.println();
 65 | 
 66 |         System.out.println("\nDegrees distribution:");
 67 |         int[] degreeDistribution = graphGenerator.getDegreeDistribution();
 68 |         for (int i = 0; i < degreeDistribution.length; i++) {
 69 |             System.out.print(i + ": " + degreeDistribution[i] + ",\t");
 70 |         }
 71 |         System.out.println();*/
 72 |     }
 73 | 
 74 |     public void generate(int n, int m) {
 75 |         clearData();
 76 | 
 77 |         this.n = n;
 78 |         this.m = m;
 79 | 
 80 |         List<Set<Integer>> initGraph = Stream.generate(HashSet<Integer>::new)
 81 |                 .limit(n * m)
 82 |                 .collect(Collectors.toList());
 83 | 
 84 |         int[] initDegrees = new int[n * m];
 85 |         Random random = new Random();
 86 | 
 87 |         //1st step of generation
 88 |         for (int i = 0; i < n * m; i++) {
 89 |             double chance = random.nextDouble();
 90 |             double factor = 1.0 / (2 * (i + 1) - 1);
 91 |             double leftBorder;
 92 |             double rightBorder = factor;
 93 | 
 94 |             if (i == 0 || chance < rightBorder) {
 95 |                 initDegrees[i] = 2;
 96 |                 initGraph.get(i).add(i);
 97 |             } else {
 98 |                 for (int j = 0; j < i; j++) {
 99 |                     leftBorder = rightBorder;
100 |                     rightBorder += initDegrees[j] * factor;
101 |                     if (leftBorder <= chance && chance < rightBorder) {
102 |                         initDegrees[i] = 1;
103 |                         initDegrees[j] += 1;
104 |                         initGraph.get(i).add(j);
105 |                         initGraph.get(j).add(i);
106 |                         break;
107 |                     }
108 |                 }
109 |             }
110 |         }
111 | 
112 |         //2nd step of generation
113 |         graph = Stream.generate(HashSet<Integer>::new)
114 |                 .limit(n)
115 |                 .collect(Collectors.toList());
116 |         degrees = new int[n];
117 | 
118 |         for (int v = 0; v < initGraph.size(); v += m) {
119 |             Set<Integer> newAdjVertices = initGraph.get(v).stream()
120 |                     .map(el -> el / m).collect(Collectors.toSet());
121 |             degrees[v / m] = initDegrees[v];
122 | 
123 |             for (int nextV = v + 1; nextV < v + m; nextV++) {
124 |                 newAdjVertices.addAll(
125 |                         initGraph.get(nextV).stream()
126 |                                 .map(el -> el / m).collect(Collectors.toSet()));
127 | 
128 |                 degrees[v / m] += initDegrees[nextV];
129 |             }
130 |             graph.set(v / m, newAdjVertices);
131 |         }
132 |     }
133 | 
134 |     public void calculateCharacteristics() {
135 |         findComponentsAndDistances();
136 | 
137 |         calculateAvgDistancesAndDiameter();
138 | 
139 |         calculateDegreeDistribution();
140 |     }
141 | 
142 | 
143 |     public Double getAvgDistance() {
144 |         return avgDistance;
145 |     }
146 | 
147 |     public int getDiameter() {
148 |         return diameter;
149 |     }
150 | 
151 |     public int[] getDegreeDistribution() {
152 |         return degreeDistribution;
153 |     }
154 | 
155 |     private void findComponentsAndDistances() {
156 |         distances = new int[n][n];
157 |         components = new HashSet<>();
158 | 
159 |         for (int i = 0; i < n; i++) {
160 |             ArrayDeque<Integer> q = new ArrayDeque<>(n);
161 |             q.push(i);
162 | 
163 |             BitSet used = new BitSet(n);
164 |             used.set(i);
165 | 
166 |             while (!q.isEmpty()) {
167 |                 int v = q.pop();
168 |                 for (Integer to : graph.get(v)) {
169 |                     if (!used.get(to)) {
170 |                         used.set(to);
171 |                         q.addLast(to);
172 |                         distances[i][to] = distances[i][v] + 1;
173 |                     }
174 |                 }
175 |             }
176 |             components.add(used);
177 |         }
178 |     }
179 | 
180 |     private void calculateDegreeDistribution() {
181 |         degreeDistribution = new int[n * m];
182 |         IntStream.of(degrees).forEach(degree -> degreeDistribution[degree]++);
183 |     }
184 | 
185 |     private void calculateAvgDistancesAndDiameter() {
186 |         int max_distance = 0;
187 |         List<Double> avgDistances = new ArrayList<>();
188 | 
189 |         for (BitSet component : components) {
190 |             int sum = 0;
191 |             for (int i = 0; i < n; i++) {
192 |                 if (component.get(i)) {
193 |                     for (int j = i + 1; j < n; j++) {
194 |                         if (component.get(j) && max_distance < distances[i][j]) {
195 |                             max_distance = distances[i][j];
196 |                         }
197 |                         sum += distances[i][j];
198 |                     }
199 |                 }
200 |             }
201 |             int amount = component.cardinality();
202 |             double avgDistance = sum * 1.0 / (amount * (amount - 1) / 2);
203 |             avgDistances.add(avgDistance);
204 |         }
205 | 
206 |         diameter = (components.size() == 1) ? max_distance : n;
207 |         avgDistance = avgDistances.stream().mapToDouble(el -> el).average().getAsDouble();
208 |     }
209 | 
210 |     private void clearData() {
211 |         n = m = diameter = 0;
212 |         avgDistance = 0;
213 |         graph = null;
214 |         degrees = null;
215 |         degreeDistribution = null;
216 |         components = null;
217 |         distances = null;
218 |     }
219 | 
220 |     public int[] getDegrees() {
221 |         return degrees;
222 |     }
223 | }
224 | 


--------------------------------------------------------------------------------
/internal_memory_algorithms/web_graph/gasnikov-raigorodskii.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/internal_memory_algorithms/web_graph/gasnikov-raigorodskii.pdf


--------------------------------------------------------------------------------
/internal_memory_algorithms/web_graph/web-graph.pptx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/internal_memory_algorithms/web_graph/web-graph.pptx


--------------------------------------------------------------------------------
/literature/Exam materials.docx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/literature/Exam materials.docx


--------------------------------------------------------------------------------
/literature/ExternalMemory.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/literature/ExternalMemory.pdf


--------------------------------------------------------------------------------
/literature/materials.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/literature/materials.pdf


--------------------------------------------------------------------------------
/literature/matmul.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/literature/matmul.pdf


--------------------------------------------------------------------------------
/literature/matrix.pptx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mikhail-kukuyev/Big-Data-Processing-Algorithms/a29f22f171c68cfaf80d03c6a00ad75016976a87/literature/matrix.pptx


--------------------------------------------------------------------------------