├── .gitattributes
├── .gitignore
├── Dockerfile
├── LICENSE
├── README.md
├── aoi.geojson
├── bbox_generate
├── cdl_total_dst.csv
├── chips_df_to_sample.csv
├── chips_df_to_sample_sampled.csv
├── data
│ ├── cdl_classes.csv
│ ├── chip_bbox_task_1.geojson
│ └── chip_bbox_task_1_5070.geojson
└── gen_chip_bbox.ipynb
├── cal_weights.ipynb
├── calc_mean_sd.ipynb
├── data
├── cdl_freq.csv
├── cdl_total_dst.csv
├── chip_bbox.geojson
├── chip_bbox_5070.geojson
├── chip_df.csv
├── chip_tracker.csv
├── selected_tiles.csv
├── sentinel_tile_grid.kml
├── tile_df.csv
├── tile_tracker.csv
└── track_df.csv
├── gen_dist.ipynb
├── requirements.txt
├── train_ids.csv
├── training_dst.png
├── val_ids.csv
├── validation_dst.png
└── workflow.ipynb
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.kml filter=lfs diff=lfs merge=lfs -text
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Ignore raw CDL data
2 | /raw_cdl
3 | .DS_Store
4 | .ipynb_checkpoints/
5 | data/.DS_Store
6 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM osgeo/gdal:ubuntu-full-3.6.3
2 |
3 | ENV LC_ALL=C.UTF-8
4 | ENV LANG=C.UTF-8
5 |
6 | RUN apt-get update
7 | RUN apt-get install sudo python3-pip python3-venv git -y
8 | RUN pip3 install rasterio==1.3.8 rio-cogeo --no-binary rasterio
9 | RUN pip3 install git+https://github.com/benmack/nasa_hls.git
10 | RUN python3 -m pip install -U pip
11 | RUN python3 -m pip install jupyterlab
12 | RUN python3 -m pip install pystac-client==0.7.1
13 |
14 | COPY ./requirements.txt ./requirements.txt
15 | RUN pip3 install -r ./requirements.txt
16 |
17 | RUN mkdir ./data
18 | RUN mkdir ./notebook
19 |
20 | EXPOSE 8888
21 |
22 | ENTRYPOINT ["jupyter", "lab", "--ip=0.0.0.0", "--allow-root"]
23 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Apache License
2 | Version 2.0, January 2004
3 | http://www.apache.org/licenses/
4 |
5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
6 |
7 | 1. Definitions.
8 |
9 | "License" shall mean the terms and conditions for use, reproduction,
10 | and distribution as defined by Sections 1 through 9 of this document.
11 |
12 | "Licensor" shall mean the copyright owner or entity authorized by
13 | the copyright owner that is granting the License.
14 |
15 | "Legal Entity" shall mean the union of the acting entity and all
16 | other entities that control, are controlled by, or are under common
17 | control with that entity. For the purposes of this definition,
18 | "control" means (i) the power, direct or indirect, to cause the
19 | direction or management of such entity, whether by contract or
20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
21 | outstanding shares, or (iii) beneficial ownership of such entity.
22 |
23 | "You" (or "Your") shall mean an individual or Legal Entity
24 | exercising permissions granted by this License.
25 |
26 | "Source" form shall mean the preferred form for making modifications,
27 | including but not limited to software source code, documentation
28 | source, and configuration files.
29 |
30 | "Object" form shall mean any form resulting from mechanical
31 | transformation or translation of a Source form, including but
32 | not limited to compiled object code, generated documentation,
33 | and conversions to other media types.
34 |
35 | "Work" shall mean the work of authorship, whether in Source or
36 | Object form, made available under the License, as indicated by a
37 | copyright notice that is included in or attached to the work
38 | (an example is provided in the Appendix below).
39 |
40 | "Derivative Works" shall mean any work, whether in Source or Object
41 | form, that is based on (or derived from) the Work and for which the
42 | editorial revisions, annotations, elaborations, or other modifications
43 | represent, as a whole, an original work of authorship. For the purposes
44 | of this License, Derivative Works shall not include works that remain
45 | separable from, or merely link (or bind by name) to the interfaces of,
46 | the Work and Derivative Works thereof.
47 |
48 | "Contribution" shall mean any work of authorship, including
49 | the original version of the Work and any modifications or additions
50 | to that Work or Derivative Works thereof, that is intentionally
51 | submitted to Licensor for inclusion in the Work by the copyright owner
52 | or by an individual or Legal Entity authorized to submit on behalf of
53 | the copyright owner. For the purposes of this definition, "submitted"
54 | means any form of electronic, verbal, or written communication sent
55 | to the Licensor or its representatives, including but not limited to
56 | communication on electronic mailing lists, source code control systems,
57 | and issue tracking systems that are managed by, or on behalf of, the
58 | Licensor for the purpose of discussing and improving the Work, but
59 | excluding communication that is conspicuously marked or otherwise
60 | designated in writing by the copyright owner as "Not a Contribution."
61 |
62 | "Contributor" shall mean Licensor and any individual or Legal Entity
63 | on behalf of whom a Contribution has been received by Licensor and
64 | subsequently incorporated within the Work.
65 |
66 | 2. Grant of Copyright License. Subject to the terms and conditions of
67 | this License, each Contributor hereby grants to You a perpetual,
68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
69 | copyright license to reproduce, prepare Derivative Works of,
70 | publicly display, publicly perform, sublicense, and distribute the
71 | Work and such Derivative Works in Source or Object form.
72 |
73 | 3. Grant of Patent License. Subject to the terms and conditions of
74 | this License, each Contributor hereby grants to You a perpetual,
75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
76 | (except as stated in this section) patent license to make, have made,
77 | use, offer to sell, sell, import, and otherwise transfer the Work,
78 | where such license applies only to those patent claims licensable
79 | by such Contributor that are necessarily infringed by their
80 | Contribution(s) alone or by combination of their Contribution(s)
81 | with the Work to which such Contribution(s) was submitted. If You
82 | institute patent litigation against any entity (including a
83 | cross-claim or counterclaim in a lawsuit) alleging that the Work
84 | or a Contribution incorporated within the Work constitutes direct
85 | or contributory patent infringement, then any patent licenses
86 | granted to You under this License for that Work shall terminate
87 | as of the date such litigation is filed.
88 |
89 | 4. Redistribution. You may reproduce and distribute copies of the
90 | Work or Derivative Works thereof in any medium, with or without
91 | modifications, and in Source or Object form, provided that You
92 | meet the following conditions:
93 |
94 | (a) You must give any other recipients of the Work or
95 | Derivative Works a copy of this License; and
96 |
97 | (b) You must cause any modified files to carry prominent notices
98 | stating that You changed the files; and
99 |
100 | (c) You must retain, in the Source form of any Derivative Works
101 | that You distribute, all copyright, patent, trademark, and
102 | attribution notices from the Source form of the Work,
103 | excluding those notices that do not pertain to any part of
104 | the Derivative Works; and
105 |
106 | (d) If the Work includes a "NOTICE" text file as part of its
107 | distribution, then any Derivative Works that You distribute must
108 | include a readable copy of the attribution notices contained
109 | within such NOTICE file, excluding those notices that do not
110 | pertain to any part of the Derivative Works, in at least one
111 | of the following places: within a NOTICE text file distributed
112 | as part of the Derivative Works; within the Source form or
113 | documentation, if provided along with the Derivative Works; or,
114 | within a display generated by the Derivative Works, if and
115 | wherever such third-party notices normally appear. The contents
116 | of the NOTICE file are for informational purposes only and
117 | do not modify the License. You may add Your own attribution
118 | notices within Derivative Works that You distribute, alongside
119 | or as an addendum to the NOTICE text from the Work, provided
120 | that such additional attribution notices cannot be construed
121 | as modifying the License.
122 |
123 | You may add Your own copyright statement to Your modifications and
124 | may provide additional or different license terms and conditions
125 | for use, reproduction, or distribution of Your modifications, or
126 | for any such Derivative Works as a whole, provided Your use,
127 | reproduction, and distribution of the Work otherwise complies with
128 | the conditions stated in this License.
129 |
130 | 5. Submission of Contributions. Unless You explicitly state otherwise,
131 | any Contribution intentionally submitted for inclusion in the Work
132 | by You to the Licensor shall be under the terms and conditions of
133 | this License, without any additional terms or conditions.
134 | Notwithstanding the above, nothing herein shall supersede or modify
135 | the terms of any separate license agreement you may have executed
136 | with Licensor regarding such Contributions.
137 |
138 | 6. Trademarks. This License does not grant permission to use the trade
139 | names, trademarks, service marks, or product names of the Licensor,
140 | except as required for reasonable and customary use in describing the
141 | origin of the Work and reproducing the content of the NOTICE file.
142 |
143 | 7. Disclaimer of Warranty. Unless required by applicable law or
144 | agreed to in writing, Licensor provides the Work (and each
145 | Contributor provides its Contributions) on an "AS IS" BASIS,
146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 | implied, including, without limitation, any warranties or conditions
148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 | PARTICULAR PURPOSE. You are solely responsible for determining the
150 | appropriateness of using or redistributing the Work and assume any
151 | risks associated with Your exercise of permissions under this License.
152 |
153 | 8. Limitation of Liability. In no event and under no legal theory,
154 | whether in tort (including negligence), contract, or otherwise,
155 | unless required by applicable law (such as deliberate and grossly
156 | negligent acts) or agreed to in writing, shall any Contributor be
157 | liable to You for damages, including any direct, indirect, special,
158 | incidental, or consequential damages of any character arising as a
159 | result of this License or out of the use or inability to use the
160 | Work (including but not limited to damages for loss of goodwill,
161 | work stoppage, computer failure or malfunction, or any and all
162 | other commercial damages or losses), even if such Contributor
163 | has been advised of the possibility of such damages.
164 |
165 | 9. Accepting Warranty or Additional Liability. While redistributing
166 | the Work or Derivative Works thereof, You may choose to offer,
167 | and charge a fee for, acceptance of support, warranty, indemnity,
168 | or other liability obligations and/or rights consistent with this
169 | License. However, in accepting such obligations, You may act only
170 | on Your own behalf and on Your sole responsibility, not on behalf
171 | of any other Contributor, and only if You agree to indemnify,
172 | defend, and hold each Contributor harmless for any liability
173 | incurred by, or claims asserted against, such Contributor by reason
174 | of your accepting any such warranty or additional liability.
175 |
176 | END OF TERMS AND CONDITIONS
177 |
178 | APPENDIX: How to apply the Apache License to your work.
179 |
180 | To apply the Apache License to your work, attach the following
181 | boilerplate notice, with the fields enclosed by brackets "[]"
182 | replaced with your own identifying information. (Don't include
183 | the brackets!) The text should be enclosed in the appropriate
184 | comment syntax for the file format. We also recommend that a
185 | file or class name and description of purpose be included on the
186 | same "printed page" as the copyright notice for easier
187 | identification within third-party archives.
188 |
189 | Copyright [yyyy] [name of copyright owner]
190 |
191 | Licensed under the Apache License, Version 2.0 (the "License");
192 | you may not use this file except in compliance with the License.
193 | You may obtain a copy of the License at
194 |
195 | http://www.apache.org/licenses/LICENSE-2.0
196 |
197 | Unless required by applicable law or agreed to in writing, software
198 | distributed under the License is distributed on an "AS IS" BASIS,
199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 | See the License for the specific language governing permissions and
201 | limitations under the License.
202 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Generating Multi-Temporal Crop Classification Training Data
2 |
3 | This repository contains the pipeline for generating a training dataset for land cover and crop type segmentation using USDA CDL data. The dataset will curate labels from USDA CDL data and input imagery from NASA HLS dataset (three cloud free scenes across the growing season). The dataset is published as part of the [Prithvi 100M](https://arxiv.org/abs/2310.18660) foundation model release on [HuggingFace](https://huggingface.co/datasets/ibm-nasa-geospatial/multi-temporal-crop-classification) and [Source Cooperative](https://beta.source.coop/repositories/clarkcga/multi-temporal-crop-classification/) with an open-access license.
4 |
5 | ## Repo Structure
6 |
7 |
8 | The main notebook `workflow.ipynb`, runs through 5 main steps, tracking the completion of each step at the HLS tile level. The completion of these steps is tracked in `tile_tracker.csv`. Chip level statistics are tracked in `chip_tracker.csv`.
9 |
10 | The `calc_mean_sd.ipynb` notebook calculates per band mean and standard deviation for HLS bands in the final dataset. It also calculates CDL per-class counts across all chips.
11 |
12 |
13 |
14 | ## Assumptions
15 |
16 |
17 | Here are the 5 steps for chip generation in `workflow.ipynb`.
18 | - Download HLS files in HDF format (downloaded for specified months, for all HLS tiles with chips)
19 | - Convert HDF files to GeoTIFF (3 images converted per HLS tile based on cloud cover and spatial coverage criteria. The cloud cover criteria is < 5% total cloud in entire image. The spatial coverage criteria first attempts to find 3 candidate images with 100% spatial coverage, and then decreases this threshold to 90%, 80%... 50%.) Of the candidate images that meet these criteria, the algorithm converts the first, last, and middle image.
20 | - Reproject GeoTIFF files to CDL projection
21 | - Chip HLS and CDL images based on chip geojson. (Outputs are in `chips`, `chips_binary`, and `chips_multi`. Each of these folders contains a 12 band HLS image e.g. `chip_000_000_merged.tif` per chip, with band order R, G, B, NIR for first date, then second date, then third date. The CDL chip is contained in the `chip_000_000.mask` file. In the `chips` directory, the CDL chip contains raw CDL values. In `chips_binary`, the CDL values are reclassified so 0 is non-crop and 1 is crop. In `chips_multi`, the CDL values are reclassified to 13 classes as in `cdl_freq.csv`)
22 | - Filter chips based on QA values and NA values. (The QA values and NA values per chip are tracked in `chip_tracker.csv`. The filter logic excludes chips that have >5% image values (for any of the three HLS image dates) for a single bad QA class. The filter logic also excludes any chips that have 1 or more NA pixels in any HLS image, in any band.)
23 |
24 | Also, when first determining which HLS tiles to use in the pipeline, please check that there are erroneous HLS tiles (see step 0a in `workflow.ipynb`). In our use case, we found that certain chips in southern CONUS were associated with HLS tile `01SBU`, which is wrong.
25 |
26 |
27 |
28 | ## Build/Run Docker Environment
29 |
30 |
31 | Build the Docker image as following:
32 | ```
33 | docker build -t cdl-data .
34 | ```
35 |
36 | Run the Docker as following (this should run after changing folder to the current folder of the repo):
37 | ```
38 | docker run -it -v :/data/ -v "$(pwd)":/cdl_training_data/ -p 8888:8888 cdl-data
39 | ```
40 | The IP to jupyterlab would be displayed automatically.
41 |
42 | *Notice: If running from EC2 you should replace the ip address by the public DNS of the EC2*
43 |
44 |
45 | ## Requirements
46 |
47 |
48 | Docker should be installed in your machine.
49 |
50 | The `workflow.ipynb` notebook requires 4 external files.
51 | - the file `data/2022_30m_cdls_clipped.tif` and this should be generated using the code in `clip.ipynb`. You need to include the raw CDL data for this code. The raw data can be downloaded from [here](https://www.nass.usda.gov/Research_and_Science/Cropland/Release) (the 2022 version).
52 | - the file `data/chip_bbox.geojson` which contains chip boundaries (in CDL crs), and attributes for chip centroids (in long, lat coordinates). The chip centroids are needed to associate each chip to an HLS tile.
53 | - the file `data/sentinel_tile_grid.kml` for associating chips to HLS tiles.
54 | - the file `data/chip_freq.csv` for reclassifying the original ~200 CDL values to 13 values (e.g. grass, forest, corn, cotton...)
55 |
--------------------------------------------------------------------------------
/aoi.geojson:
--------------------------------------------------------------------------------
1 | {
2 | "type": "FeatureCollection",
3 | "features": [
4 | {
5 | "type": "Feature",
6 | "properties": {},
7 | "geometry": {
8 | "coordinates": [
9 | [
10 | [
11 | -98.46590447567304,
12 | 28.482161633569604
13 | ],
14 | [
15 | -98.46590447567304,
16 | 28.491892282407136
17 | ],
18 | [
19 | -98.49284409933445,
20 | 28.491892282407136
21 | ],
22 | [
23 | -98.49284409933445,
24 | 28.482161633569604
25 | ],
26 | [
27 | -98.46590447567304,
28 | 28.482161633569604
29 | ]
30 | ]
31 | ],
32 | "type": "Polygon"
33 | }
34 | }
35 | ]
36 | }
--------------------------------------------------------------------------------
/bbox_generate/cdl_total_dst.csv:
--------------------------------------------------------------------------------
1 | old_class_value,total_count,percentage,cum_percentage,class_name,new_class_value_16,new_class_value
2 | 152,2063333477,0.240438461,0.240438461,Shrubland,1,1
3 | 176,1482158236,0.172714614,0.413153076,Grassland/Pasture,2,1
4 | 142,1087546299,0.126730827,0.539883903,Evergreen Forest,3,2
5 | 141,898140095,0.104659487,0.64454339,Deciduous Forest,3,2
6 | 1,409599488,0.047730273,0.692273663,Corn,4,3
7 | 5,399644525,0.04657023,0.738843893,Soybeans,5,4
8 | 190,381270149,0.04442908,0.783272973,Woody Wetlands,6,5
9 | 143,254963448,0.02971067,0.812983643,Mixed Forest,3,2
10 | 121,235937778,0.027493625,0.840477268,Developed/Open Space,7,6
11 | 111,147580217,0.017197395,0.857674663,Open Water,8,7
12 | 122,143906418,0.01676929,0.874443953,Developed/Low Intensity,7,6
13 | 24,125713120,0.014649241,0.889093194,Winter Wheat,9,8
14 | 37,123121151,0.014347201,0.903440395,Other Hay/Non Alfalfa,10,1
15 | 195,110102692,0.012830171,0.916270566,Herbaceous Wetlands,6,5
16 | 36,92198712,0.010743836,0.927014402,Alfalfa,11,9
17 | 61,87070549,0.010146256,0.937160657,Fallow/Idle Cropland,12,10
18 | 131,85538350,0.00996771,0.947128367,Barren,13,6
19 | 123,83797654,0.009764868,0.956893235,Developed/Med Intensity,7,6
20 | 2,68184110,0.007945435,0.96483867,Cotton,14,11
21 | 23,55358735,0.006450905,0.971289576,Spring Wheat,16,13
22 | 4,37035573,0.004315723,0.975605299,Sorghum,15,12
23 | 124,29135611,0.003395147,0.979000447,Developed/High Intensity,7,6
24 | 26,17366266,0.002023676,0.981024122,Dbl Crop WinWht/Soybeans,16,13
25 | 21,11778073,0.001372489,0.982396611,Barley,16,13
26 | 3,10557196,0.001230221,0.983626832,Rice,16,13
27 | 31,9813943,0.00114361,0.984770442,Canola,16,13
28 | 75,8421500,0.00098135,0.985751792,Almonds,16,13
29 | 28,7699542,0.000897221,0.986649013,Oats,16,13
30 | 6,6947944,0.000809638,0.987458651,Sunflower,16,13
31 | 10,6647579,0.000774637,0.988233288,Peanuts,16,13
32 | 22,6614531,0.000770786,0.989004073,Durum Wheat,16,13
33 | 59,5724785,0.000667104,0.989671178,Sod/Grass Seed,16,13
34 | 42,5663396,0.000659951,0.990331128,Dry Beans,16,13
35 | 45,5188141,0.00060457,0.990935698,Sugarcane,16,13
36 | 41,5111809,0.000595675,0.991531372,Sugarbeets,16,13
37 | 53,4990859,0.00058158,0.992112953,Peas,16,13
38 | 69,4870658,0.000567574,0.992680527,Grapes,16,13
39 | 43,4588956,0.000534747,0.993215274,Potatoes,16,13
40 | 29,4305524,0.000501719,0.993716993,Millet,16,13
41 | 205,3721143,0.000433622,0.994150614,Triticale,16,13
42 | 27,3289706,0.000383347,0.994533961,Rye,16,13
43 | 212,3147860,0.000366817,0.994900778,Oranges,16,13
44 | 52,3021952,0.000352145,0.995252924,Lentils,16,13
45 | 204,2974708,0.00034664,0.995599564,Pistachios,16,13
46 | 236,2340687,0.000272758,0.995872322,Dbl Crop WinWht/Sorghum,16,13
47 | 92,2309432,0.000269116,0.996141438,Aquaculture,16,13
48 | 76,2295688,0.000267515,0.996408953,Walnuts,16,13
49 | 74,2157328,0.000251392,0.996660344,Pecans,16,13
50 | 68,2074192,0.000241704,0.996902048,Apples,16,13
51 | 72,1950228,0.000227258,0.997129306,Citrus,16,13
52 | 51,1712708,0.00019958,0.997328887,Chick Peas,16,13
53 | 238,1566542,0.000182548,0.997511434,Dbl Crop WinWht/Cotton,16,13
54 | 225,1339057,0.000156039,0.997667474,Dbl Crop WinWht/Corn,16,13
55 | 54,1260950,0.000146937,0.997814411,Tomatoes,16,13
56 | 35,1200015,0.000139837,0.997954248,Mustard,16,13
57 | 32,1139213,0.000132752,0.998086999,Flaxseed,16,13
58 | 12,1126998,0.000131328,0.998218327,Sweet Corn,16,13
59 | 66,1014301,0.000118196,0.998336523,Cherries,16,13
60 | 242,1005848,0.000117211,0.998453734,Blueberries,16,13
61 | 33,987435,0.000115065,0.998568799,Safflower,16,13
62 | 228,877818,0.000102291,0.99867109,Dbl Crop Triticale/Corn,16,13
63 | 58,829673,9.67E-05,0.998767771,Clover/Wildflowers,16,13
64 | 112,769484,8.97E-05,0.998857438,Perennial Ice/Snow ,16,13
65 | 13,757270,8.82E-05,0.998945682,Pop or Orn Corn,16,13
66 | 71,723261,8.43E-05,0.999029963,Other Tree Crops,16,13
67 | 49,600910,7.00E-05,0.999099987,Onions,16,13
68 | 46,506246,5.90E-05,0.999158979,Sweet Potatoes,16,13
69 | 11,452380,5.27E-05,0.999211695,Tobacco,16,13
70 | 67,446577,5.20E-05,0.999263734,Peaches,16,13
71 | 70,401521,4.68E-05,0.999310523,Christmas Trees,16,13
72 | 56,355934,4.15E-05,0.999351999,Hops,16,13
73 | 226,337423,3.93E-05,0.999391319,Dbl Crop Oats/Corn,16,13
74 | 211,326907,3.81E-05,0.999429413,Olives,16,13
75 | 210,320923,3.74E-05,0.99946681,Prunes,16,13
76 | 47,313035,3.65E-05,0.999503288,Misc Vegs & Fruits,16,13
77 | 44,309625,3.61E-05,0.999539368,Other Crops,16,13
78 | 220,297567,3.47E-05,0.999574043,Plums,16,13
79 | 77,240253,2.80E-05,0.99960204,Pears,16,13
80 | 14,238986,2.78E-05,0.999629889,Mint,16,13
81 | 50,223026,2.60E-05,0.999655878,Cucumbers,16,13
82 | 215,214953,2.50E-05,0.999680926,Avocados,16,13
83 | 227,214938,2.50E-05,0.999705973,Lettuce,16,13
84 | 57,208988,2.44E-05,0.999730326,Herbs,16,13
85 | 206,193907,2.26E-05,0.999752922,Carrots,16,13
86 | 254,190177,2.22E-05,0.999775083,Dbl Crop Barley/Soybeans,16,13
87 | 208,162057,1.89E-05,0.999793967,Garlic,16,13
88 | 217,150035,1.75E-05,0.999811451,Pomegranates,16,13
89 | 48,142960,1.67E-05,0.99982811,Watermelons,16,13
90 | 219,138410,1.61E-05,0.999844238,Greens,16,13
91 | 229,136210,1.59E-05,0.999860111,Pumpkins,16,13
92 | 240,109040,1.27E-05,0.999872817,Dbl Crop Soybeans/Oats,16,13
93 | 243,102473,1.19E-05,0.999884758,Cabbage,16,13
94 | 237,91744,1.07E-05,0.999895449,Dbl Crop Barley/Corn,16,13
95 | 39,88874,1.04E-05,0.999905805,Buckwheat,16,13
96 | 216,83896,9.78E-06,0.999915582,Peppers,16,13
97 | 222,78828,9.19E-06,0.999924768,Squash,16,13
98 | 221,67953,7.92E-06,0.999932686,Strawberries,16,13
99 | 25,66598,7.76E-06,0.999940447,Other Small Grains,16,13
100 | 214,60268,7.02E-06,0.99994747,Broccoli,16,13
101 | 209,51235,5.97E-06,0.99995344,Cantaloupes,16,13
102 | 38,45729,5.33E-06,0.999958769,Camelina,16,13
103 | 246,44337,5.17E-06,0.999963935,Radishes,16,13
104 | 55,41258,4.81E-06,0.999968743,Caneberries,16,13
105 | 207,40047,4.67E-06,0.99997341,Asparagus,16,13
106 | 232,32816,3.82E-06,0.999977234,Dbl Crop Lettuce/Cotton,16,13
107 | 60,31618,3.68E-06,0.999980918,Switchgrass,16,13
108 | 250,25416,2.96E-06,0.99998388,Cranberries,16,13
109 | 247,19755,2.30E-06,0.999986182,Turnips,16,13
110 | 30,18951,2.21E-06,0.99998839,Speltz,16,13
111 | 244,18422,2.15E-06,0.999990537,Cauliflower,16,13
112 | 231,17228,2.01E-06,0.999992544,Dbl Crop Lettuce/Cantaloupe,16,13
113 | 241,15738,1.83E-06,0.999994378,Dbl Crop Corn/Soybeans,16,13
114 | 213,12093,1.41E-06,0.999995788,Honeydew Melons,16,13
115 | 218,11613,1.35E-06,0.999997141,Nectarines,16,13
116 | 224,7010,8.17E-07,0.999997958,Vetch,16,13
117 | 245,3955,4.61E-07,0.999998419,Celery,16,13
118 | 34,3771,4.39E-07,0.999998858,Rape Seed,16,13
119 | 248,3550,4.14E-07,0.999999272,Eggplants,16,13
120 | 223,2087,2.43E-07,0.999999515,Apricots,16,13
121 | 233,1805,2.10E-07,0.999999725,Dbl Crop Lettuce/Barley,16,13
122 | 249,1610,1.88E-07,0.999999913,Gourds,16,13
123 | 230,748,8.72E-08,1,Dbl Crop Lettuce/Durum Wht,16,13
124 | 88,0,0,1,Nonag/Undefined,16,13
125 | 239,0,0,1,Dbl Crop Soybeans/Cotton,16,13
126 | 235,0,0,1,Dbl Crop Barley/Sorghum,16,13
127 | 234,0,0,1,Dbl Crop Durum Wht/Sorghum,16,13
128 | 87,0,0,1,Wetlands,16,13
129 | 62,0,0,1,Pasture/Grass,16,13
130 | 63,0,0,1,Forest,16,13
131 | 64,0,0,1,Shrubland,16,13
132 | 65,0,0,1,Barren,16,13
133 | 81,0,0,1,Clouds/No Data,16,13
134 | 83,0,0,1,Water,16,13
135 | 82,0,0,1,Developed,16,13
--------------------------------------------------------------------------------
/bbox_generate/data/cdl_classes.csv:
--------------------------------------------------------------------------------
1 | class,value
2 | 1,Corn
3 | 2,Cotton
4 | 3,Rice
5 | 4,Sorghum
6 | 5,Soybeans
7 | 6,Sunflower
8 | 7,
9 | 8,
10 | 9,
11 | 10,Peanuts
12 | 11,Tobacco
13 | 12,Sweet Corn
14 | 13,Pop or Orn Corn
15 | 14,Mint
16 | 15,
17 | 16,
18 | 17,
19 | 18,
20 | 19,
21 | 20,
22 | 21,Barley
23 | 22,Durum Wheat
24 | 23,Spring Wheat
25 | 24,Winter Wheat
26 | 25,Other Small Grains
27 | 26,Dbl Crop WinWht/Soybeans
28 | 27,Rye
29 | 28,Oats
30 | 29,Millet
31 | 30,Speltz
32 | 31,Canola
33 | 32,Flaxseed
34 | 33,Safflower
35 | 34,Rape Seed
36 | 35,Mustard
37 | 36,Alfalfa
38 | 37,Other Hay/Non Alfalfa
39 | 38,Camelina
40 | 39,Buckwheat
41 | 40,
42 | 41,Sugarbeets
43 | 42,Dry Beans
44 | 43,Potatoes
45 | 44,Other Crops
46 | 45,Sugarcane
47 | 46,Sweet Potatoes
48 | 47,Misc Vegs & Fruits
49 | 48,Watermelons
50 | 49,Onions
51 | 50,Cucumbers
52 | 51,Chick Peas
53 | 52,Lentils
54 | 53,Peas
55 | 54,Tomatoes
56 | 55,Caneberries
57 | 56,Hops
58 | 57,Herbs
59 | 58,Clover/Wildflowers
60 | 59,Sod/Grass Seed
61 | 60,Switchgrass
62 | 61,Fallow/Idle Cropland
63 | 62,Pasture/Grass
64 | 63,Forest
65 | 64,Shrubland
66 | 65,Barren
67 | 66,Cherries
68 | 67,Peaches
69 | 68,Apples
70 | 69,Grapes
71 | 70,Christmas Trees
72 | 71,Other Tree Crops
73 | 72,Citrus
74 | 73,
75 | 74,Pecans
76 | 75,Almonds
77 | 76,Walnuts
78 | 77,Pears
79 | 78,
80 | 79,
81 | 80,
82 | 81,Clouds/No Data
83 | 82,Developed
84 | 83,Water
85 | 84,
86 | 85,
87 | 86,
88 | 87,Wetlands
89 | 88,Nonag/Undefined
90 | 89,
91 | 90,
92 | 91,
93 | 92,Aquaculture
94 | 93,
95 | 94,
96 | 95,
97 | 96,
98 | 97,
99 | 98,
100 | 99,
101 | 100,
102 | 101,
103 | 102,
104 | 103,
105 | 104,
106 | 105,
107 | 106,
108 | 107,
109 | 108,
110 | 109,
111 | 110,
112 | 111,Open Water
113 | 112,Perennial Ice/Snow
114 | 113,
115 | 114,
116 | 115,
117 | 116,
118 | 117,
119 | 118,
120 | 119,
121 | 120,
122 | 121,Developed/Open Space
123 | 122,Developed/Low Intensity
124 | 123,Developed/Med Intensity
125 | 124,Developed/High Intensity
126 | 125,
127 | 126,
128 | 127,
129 | 128,
130 | 129,
131 | 130,
132 | 131,Barren
133 | 132,
134 | 133,
135 | 134,
136 | 135,
137 | 136,
138 | 137,
139 | 138,
140 | 139,
141 | 140,
142 | 141,Deciduous Forest
143 | 142,Evergreen Forest
144 | 143,Mixed Forest
145 | 144,
146 | 145,
147 | 146,
148 | 147,
149 | 148,
150 | 149,
151 | 150,
152 | 151,
153 | 152,Shrubland
154 | 153,
155 | 154,
156 | 155,
157 | 156,
158 | 157,
159 | 158,
160 | 159,
161 | 160,
162 | 161,
163 | 162,
164 | 163,
165 | 164,
166 | 165,
167 | 166,
168 | 167,
169 | 168,
170 | 169,
171 | 170,
172 | 171,
173 | 172,
174 | 173,
175 | 174,
176 | 175,
177 | 176,Grassland/Pasture
178 | 177,
179 | 178,
180 | 179,
181 | 180,
182 | 181,
183 | 182,
184 | 183,
185 | 184,
186 | 185,
187 | 186,
188 | 187,
189 | 188,
190 | 189,
191 | 190,Woody Wetlands
192 | 191,
193 | 192,
194 | 193,
195 | 194,
196 | 195,Herbaceous Wetlands
197 | 196,
198 | 197,
199 | 198,
200 | 199,
201 | 200,
202 | 201,
203 | 202,
204 | 203,
205 | 204,Pistachios
206 | 205,Triticale
207 | 206,Carrots
208 | 207,Asparagus
209 | 208,Garlic
210 | 209,Cantaloupes
211 | 210,Prunes
212 | 211,Olives
213 | 212,Oranges
214 | 213,Honeydew Melons
215 | 214,Broccoli
216 | 215,Avocados
217 | 216,Peppers
218 | 217,Pomegranates
219 | 218,Nectarines
220 | 219,Greens
221 | 220,Plums
222 | 221,Strawberries
223 | 222,Squash
224 | 223,Apricots
225 | 224,Vetch
226 | 225,Dbl Crop WinWht/Corn
227 | 226,Dbl Crop Oats/Corn
228 | 227,Lettuce
229 | 228,Dbl Crop Triticale/Corn
230 | 229,Pumpkins
231 | 230,Dbl Crop Lettuce/Durum Wht
232 | 231,Dbl Crop Lettuce/Cantaloupe
233 | 232,Dbl Crop Lettuce/Cotton
234 | 233,Dbl Crop Lettuce/Barley
235 | 234,Dbl Crop Durum Wht/Sorghum
236 | 235,Dbl Crop Barley/Sorghum
237 | 236,Dbl Crop WinWht/Sorghum
238 | 237,Dbl Crop Barley/Corn
239 | 238,Dbl Crop WinWht/Cotton
240 | 239,Dbl Crop Soybeans/Cotton
241 | 240,Dbl Crop Soybeans/Oats
242 | 241,Dbl Crop Corn/Soybeans
243 | 242,Blueberries
244 | 243,Cabbage
245 | 244,Cauliflower
246 | 245,Celery
247 | 246,Radishes
248 | 247,Turnips
249 | 248,Eggplants
250 | 249,Gourds
251 | 250,Cranberries
252 | 251,
253 | 252,
254 | 253,
255 | 254,Dbl Crop Barley/Soybeans
256 | 255,
--------------------------------------------------------------------------------
/bbox_generate/gen_chip_bbox.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "id": "65bda451-6919-4a51-9147-f50e0209a283",
6 | "metadata": {},
7 | "source": [
8 | "# Generate Bbox definition for the dataset"
9 | ]
10 | },
11 | {
12 | "cell_type": "markdown",
13 | "id": "da909d15-1b0e-4476-a20a-0db1cce98a80",
14 | "metadata": {},
15 | "source": [
16 | "This notebook loads the USDA CDL dataset, and generates chip bbox across the CONUS to have a diverse distribution of classes. \n",
17 | "The notebook is designed for projects run at Clark University's Center for Geospatial Analytics; hence, it has certain features that are shared across multiple projects. For example, we merge all CDL classes to 13 classes before sampling. "
18 | ]
19 | },
20 | {
21 | "cell_type": "code",
22 | "execution_count": null,
23 | "id": "1f05c11e-196d-434e-b2bc-4512413f7356",
24 | "metadata": {},
25 | "outputs": [],
26 | "source": [
27 | "import rioxarray \n",
28 | "import rasterio\n",
29 | "import numpy as np\n",
30 | "import pandas as pd\n",
31 | "import pyproj\n",
32 | "import json\n",
33 | "from shapely.geometry import Polygon"
34 | ]
35 | },
36 | {
37 | "cell_type": "markdown",
38 | "id": "a7ed7a86-7e2e-4310-86dd-f63ff3a4c34e",
39 | "metadata": {},
40 | "source": [
41 | "### Load CDL Data\n",
42 | "You need to download the CDL .tiff file from USDA' website [here](https://www.nass.usda.gov/Research_and_Science/Cropland/Release/index.php) and store it under `data/`."
43 | ]
44 | },
45 | {
46 | "cell_type": "code",
47 | "execution_count": null,
48 | "id": "5f350978-2693-4ef9-905f-b74dd59e217c",
49 | "metadata": {},
50 | "outputs": [],
51 | "source": [
52 | "cdl_class_df = pd.read_csv(\"data/cdl_classes.csv\", encoding = \"ISO-8859-1\")"
53 | ]
54 | },
55 | {
56 | "cell_type": "code",
57 | "execution_count": null,
58 | "id": "243ad12a-2287-4658-96e5-c1968d0b409b",
59 | "metadata": {},
60 | "outputs": [],
61 | "source": [
62 | "# Drop class values that do not represent any real class (not all 256 values represent a class)\n",
63 | "cdl_class_valid = cdl_class_df[~cdl_class_df['value'].isna()][\"class\"].values"
64 | ]
65 | },
66 | {
67 | "cell_type": "code",
68 | "execution_count": null,
69 | "id": "5c11d9f7-6145-4094-8010-91a9650433d3",
70 | "metadata": {},
71 | "outputs": [],
72 | "source": [
73 | "cdl_file = \"data/2022_30m_cdls.tif\""
74 | ]
75 | },
76 | {
77 | "cell_type": "code",
78 | "execution_count": null,
79 | "id": "0d8a6a1d-6236-4d16-bb46-aa8043bf7f50",
80 | "metadata": {},
81 | "outputs": [],
82 | "source": [
83 | "xds = rioxarray.open_rasterio(cdl_file, cache=False)"
84 | ]
85 | },
86 | {
87 | "cell_type": "markdown",
88 | "id": "294c8842-e309-4c6e-9bdf-3eddddea914e",
89 | "metadata": {},
90 | "source": [
91 | "### Chipping"
92 | ]
93 | },
94 | {
95 | "cell_type": "code",
96 | "execution_count": null,
97 | "id": "3333ca0a-d81e-4436-a6d9-0fcbe1c64c56",
98 | "metadata": {},
99 | "outputs": [],
100 | "source": [
101 | "# Define chip specs\n",
102 | "chip_dim_x = 224\n",
103 | "chip_dim_y = 224\n",
104 | "x0 = xds.x.data[0]\n",
105 | "y0 = xds.y.data[0]\n",
106 | "res = 30 # meters"
107 | ]
108 | },
109 | {
110 | "cell_type": "code",
111 | "execution_count": null,
112 | "id": "8473856f-9be0-4fff-8325-5c9148ed5ade",
113 | "metadata": {},
114 | "outputs": [],
115 | "source": [
116 | "# Create an empty df to load chips\n",
117 | "df_columns = [\"chip_id\"]\n",
118 | "df_columns.append(0)\n",
119 | "for i in cdl_class_df[\"class\"]:\n",
120 | " df_columns.append(i)\n",
121 | "df_columns.append(\"chip_coordinate_x\")\n",
122 | "df_columns.append(\"chip_coordinate_y\")\n",
123 | "chips_df = pd.DataFrame(columns = df_columns)"
124 | ]
125 | },
126 | {
127 | "cell_type": "code",
128 | "execution_count": null,
129 | "id": "9c6f4603-d6f7-4b5e-9dcb-cd5ad3f790f6",
130 | "metadata": {
131 | "scrolled": true
132 | },
133 | "outputs": [],
134 | "source": [
135 | "# Iterate over non-overlapping chips across all of CONUS and load their properties into the df\n",
136 | "for idx in range(0, int(np.floor(xds.shape[2] / chip_dim_x))):\n",
137 | " for idy in range(0, int(np.floor(xds.shape[1] / chip_dim_y))):\n",
138 | " chip_id = f\"chip_{str(idy).zfill(3)}_{str(idx).zfill(3)}\"\n",
139 | " chip_x0 = x0 + idx * chip_dim_x * res\n",
140 | " chip_y0 = y0 - idy * chip_dim_y * res\n",
141 | " # Note: the following line uses xarray da.sel() method which uses inclusive bounds for data selection. Hence\n",
142 | " # we use (chip_dim_y -1) and (chip_dim_x -1). \n",
143 | " chip = xds.rio.slice_xy(chip_x0, chip_y0 - (chip_dim_y -1) * res, chip_x0 + (chip_dim_x - 1) * res, chip_y0)\n",
144 | " classes, class_counts = np.unique(chip.data, return_counts=True)\n",
145 | " # Drop chips that contain no-data value (0)\n",
146 | " if 0 not in classes: \n",
147 | " counts = np.zeros(256)\n",
148 | " counts[classes] = class_counts\n",
149 | " chips_df.loc[len(chips_df.index)] = [chip_id] + counts.tolist() + [chip_x0, chip_y0]"
150 | ]
151 | },
152 | {
153 | "cell_type": "markdown",
154 | "id": "fee32d6e-49e8-45b3-bee6-b99ff977dac7",
155 | "metadata": {},
156 | "source": [
157 | "### Inspect Data Distribution"
158 | ]
159 | },
160 | {
161 | "cell_type": "code",
162 | "execution_count": null,
163 | "id": "bbe60377-8e93-4a30-a9d2-32a313047529",
164 | "metadata": {},
165 | "outputs": [],
166 | "source": [
167 | "# Plot distribution of classes across the dataset. \n",
168 | "import matplotlib.pyplot as plt\n",
169 | "plt.bar(cdl_class_valid, chips_df.loc[:, cdl_class_valid].sum())"
170 | ]
171 | },
172 | {
173 | "cell_type": "code",
174 | "execution_count": null,
175 | "id": "5a9d02ad-4813-489f-a9a4-5fbe869357bf",
176 | "metadata": {},
177 | "outputs": [],
178 | "source": [
179 | "# Inspecting total class distribution and define classes to merge. \n",
180 | "cdl_total_dst = pd.DataFrame(chips_df.loc[:, cdl_class_valid].sum(), columns=[\"total_count\"])\n",
181 | "cdl_total_dst[\"percentage\"] = cdl_total_dst[\"total_count\"] / cdl_total_dst[\"total_count\"].sum()\n",
182 | "cdl_total_dst.sort_values(by = \"percentage\", ascending = False, inplace = True)\n",
183 | "cdl_total_dst[\"cum_percentage\"] = cdl_total_dst[\"percentage\"].cumsum(axis = 0)\n",
184 | "cdl_total_dst[\"class_name\"] = np.nan\n",
185 | "for i in cdl_total_dst.iterrows():\n",
186 | " cdl_total_dst.loc[i[0], \"class_name\"] = cdl_class_df[cdl_class_df[\"class\"] == i[0]][\"value\"].values[0]"
187 | ]
188 | },
189 | {
190 | "cell_type": "code",
191 | "execution_count": null,
192 | "id": "cc3e9cbd-b101-44f1-bfc9-daff59a51c1d",
193 | "metadata": {},
194 | "outputs": [],
195 | "source": [
196 | "# Do NOT uncomment this. This will overwrite the csv file which contains the column \"new_class_value\". \n",
197 | "# This column is added manually by inspecting the class distribution, and defines the new class values. \n",
198 | "# cdl_total_dst.to_csv('cdl_total_dst.csv')"
199 | ]
200 | },
201 | {
202 | "cell_type": "markdown",
203 | "id": "ff7391e3-dafe-4a6f-8020-29cc99966ced",
204 | "metadata": {},
205 | "source": [
206 | "Reviewed the csv manually, and defined new classes to merge. \n",
207 | "In the following we will load the csv again, and create a new df with new classes. "
208 | ]
209 | },
210 | {
211 | "cell_type": "code",
212 | "execution_count": null,
213 | "id": "caffa62d-c823-4e72-ab8f-5818eeae5eab",
214 | "metadata": {},
215 | "outputs": [],
216 | "source": [
217 | "cdl_total_dst = pd.read_csv(\"cdl_total_dst.csv\")\n",
218 | "n_classes = cdl_total_dst[\"new_class_value\"].max()"
219 | ]
220 | },
221 | {
222 | "cell_type": "code",
223 | "execution_count": null,
224 | "id": "9a0472a8-729b-4503-a9d6-f484ece57f57",
225 | "metadata": {
226 | "scrolled": true
227 | },
228 | "outputs": [],
229 | "source": [
230 | "chips_df_to_sample = pd.DataFrame(np.zeros((chips_df.shape[0], n_classes)), columns = np.arange(1, n_classes+1))\n",
231 | "for row in cdl_total_dst.iterrows():\n",
232 | " chips_df_to_sample.loc[:, row[1][\"new_class_value\"]] = chips_df_to_sample.loc[:, row[1][\"new_class_value\"]] + chips_df.loc[:, row[1][\"old_class_value\"]]\n",
233 | "chips_df_to_sample[\"chip_id\"] = chips_df[\"chip_id\"]\n",
234 | "chips_df_to_sample[\"chip_coordinate_x\"] = chips_df[\"chip_coordinate_x\"]\n",
235 | "chips_df_to_sample[\"chip_coordinate_y\"] = chips_df[\"chip_coordinate_y\"]"
236 | ]
237 | },
238 | {
239 | "cell_type": "code",
240 | "execution_count": null,
241 | "id": "4002d4e8-5e4c-4bf9-9666-d09b0cffb9cc",
242 | "metadata": {},
243 | "outputs": [],
244 | "source": [
245 | "#The following confirms that the re-classification (merging of similar classes hasn't resulted in any mistake). \n",
246 | "print(np.sum(chips_df_to_sample[[col for col in chips_df_to_sample.columns if col not in [\"chip_coordinate_y\", \"chip_coordinate_x\", \"chip_id\"]]].sum()))\n",
247 | "\n",
248 | "print(np.sum(chips_df[[col for col in chips_df.columns if col not in [\"chip_coordinate_y\", \"chip_coordinate_x\", \"chip_id\"]]].sum()))"
249 | ]
250 | },
251 | {
252 | "cell_type": "code",
253 | "execution_count": null,
254 | "id": "629fb633-e727-4f91-bea8-cb7a66efaf51",
255 | "metadata": {},
256 | "outputs": [],
257 | "source": [
258 | "# chips_df_to_sample.to_csv(\"chips_df_to_sample.csv\")"
259 | ]
260 | },
261 | {
262 | "cell_type": "markdown",
263 | "id": "d3b467a1-3ea9-4105-b3fa-69f3a4c8c53a",
264 | "metadata": {},
265 | "source": [
266 | "### Sampling"
267 | ]
268 | },
269 | {
270 | "cell_type": "code",
271 | "execution_count": null,
272 | "id": "fb2f68b1-bf1a-4817-a762-771c8798cf76",
273 | "metadata": {},
274 | "outputs": [],
275 | "source": [
276 | "def chip_weight(row):\n",
277 | " \"\"\"\n",
278 | " This function return a weight for each chip (defined as a row in the df). \n",
279 | " The weight is 1 - the difference between the max and min of the portion of the chip that is covered \n",
280 | " by each class. e.g. if the dominant class takes 0.5 of the chip area and the least dominant one takes 0.1\n",
281 | " the weight would be 1 - (0.5 - 0.1). In this way, chips that have more even distribution get higher weight. \n",
282 | " In addition to avoid having chips that only contain certain dominant classes, any chip with less than 8 classes\n",
283 | " will be punished with a weight of zero. \n",
284 | " \"\"\"\n",
285 | " weight = 1 - (np.max(row) - np.min(row)) / np.sum(row)\n",
286 | " if np.count_nonzero(row) < 8:\n",
287 | " wegiht = weight - 1\n",
288 | " if weight < 0:\n",
289 | " weight = 0\n",
290 | " return weight"
291 | ]
292 | },
293 | {
294 | "cell_type": "code",
295 | "execution_count": null,
296 | "id": "06af36fb-65f8-4527-8324-cc354c166f15",
297 | "metadata": {},
298 | "outputs": [],
299 | "source": [
300 | "chips_df_to_sample[\"chip_weight\"] = chips_df_to_sample.loc[:, 1:n_classes].apply(chip_weight, axis = 1)"
301 | ]
302 | },
303 | {
304 | "cell_type": "code",
305 | "execution_count": null,
306 | "id": "beacffe7-d294-4045-9ce7-7d32d0fa9821",
307 | "metadata": {},
308 | "outputs": [],
309 | "source": [
310 | "chips_df_to_sample[\"chip_weight\"] = chips_df_to_sample[\"chip_weight\"] / chips_df_to_sample[\"chip_weight\"].sum()"
311 | ]
312 | },
313 | {
314 | "cell_type": "code",
315 | "execution_count": null,
316 | "id": "e7000d1e-af99-4b0b-887f-ad68c9e81820",
317 | "metadata": {},
318 | "outputs": [],
319 | "source": [
320 | "from numpy.random import choice\n",
321 | "samples = choice(chips_df_to_sample.index, 100000000, p = chips_df_to_sample[\"chip_weight\"])"
322 | ]
323 | },
324 | {
325 | "cell_type": "code",
326 | "execution_count": null,
327 | "id": "2c82c4fb-63d6-40d8-b40c-9dab457f9135",
328 | "metadata": {},
329 | "outputs": [],
330 | "source": [
331 | "uniqe_sample, sample_count = np.unique(samples, return_counts=True)\n",
332 | "sample_counts = np.zeros(chips_df_to_sample.index.shape)\n",
333 | "sample_counts[uniqe_sample] = sample_count"
334 | ]
335 | },
336 | {
337 | "cell_type": "code",
338 | "execution_count": null,
339 | "id": "8a95a568-6a78-46b1-84c6-ba712b11d96d",
340 | "metadata": {},
341 | "outputs": [],
342 | "source": [
343 | "chips_df_to_sample['total_samples_count'] = sample_counts\n",
344 | "\n",
345 | "chips_df_to_sample.sort_values(by=['total_samples_count'], ascending=False, inplace=True)"
346 | ]
347 | },
348 | {
349 | "cell_type": "code",
350 | "execution_count": null,
351 | "id": "5d09eaae-8384-4247-ba1d-bc876b2f5973",
352 | "metadata": {},
353 | "outputs": [],
354 | "source": [
355 | "# chips_df_to_sample.to_csv(\"chips_df_to_sample_sampled.csv\")\n",
356 | "chips_df_to_sample = pd.read_csv(\"chips_df_to_sample_sampled.csv\", index_col=\"index\")"
357 | ]
358 | },
359 | {
360 | "cell_type": "markdown",
361 | "id": "bf83d13c-3cfb-491b-b0a0-331cb70cab7a",
362 | "metadata": {},
363 | "source": [
364 | "### Generate BBoxes"
365 | ]
366 | },
367 | {
368 | "cell_type": "code",
369 | "execution_count": null,
370 | "id": "045904ae-a26e-40c4-9992-610b32fd92c2",
371 | "metadata": {},
372 | "outputs": [],
373 | "source": [
374 | "import matplotlib.pyplot as plt\n",
375 | "plt.bar(range(1, n_classes+1), chips_df_to_sample.iloc[:5000, range(0, n_classes)].sum()/np.sum(chips_df_to_sample.iloc[:5000, range(0, n_classes)].sum()))"
376 | ]
377 | },
378 | {
379 | "cell_type": "code",
380 | "execution_count": null,
381 | "id": "f5fda2d2-46c3-4305-87d5-87376892a5de",
382 | "metadata": {},
383 | "outputs": [],
384 | "source": [
385 | "class_per = chips_df_to_sample.iloc[:5000, range(0, n_classes)].sum()/np.sum(chips_df_to_sample.iloc[:5000, range(0, n_classes)].sum())"
386 | ]
387 | },
388 | {
389 | "cell_type": "code",
390 | "execution_count": null,
391 | "id": "67d67d13-859f-4a77-b4ed-c6438c0aab77",
392 | "metadata": {},
393 | "outputs": [],
394 | "source": [
395 | "class_weights = (1./class_per) / np.sum((1./class_per))"
396 | ]
397 | },
398 | {
399 | "cell_type": "code",
400 | "execution_count": null,
401 | "id": "164237ab-0fa1-4208-9e82-ae6adde93a6a",
402 | "metadata": {},
403 | "outputs": [],
404 | "source": [
405 | "class_weights.to_csv(\"task_1_class_weights.csv\")"
406 | ]
407 | },
408 | {
409 | "cell_type": "code",
410 | "execution_count": null,
411 | "id": "3631cc48-c2a1-4f6e-93cd-37c8e111bf4b",
412 | "metadata": {},
413 | "outputs": [],
414 | "source": [
415 | "plt.plot(chips_df_to_sample.iloc[:5000, 17], chips_df_to_sample.iloc[:5000, 18], \"*\") "
416 | ]
417 | },
418 | {
419 | "cell_type": "code",
420 | "execution_count": null,
421 | "id": "b0a080dc-2d1b-4338-aaec-6252e120a456",
422 | "metadata": {},
423 | "outputs": [],
424 | "source": [
425 | "features = []\n",
426 | "proj = pyproj.Transformer.from_crs(5070, 4326, always_xy=True)\n",
427 | "chip_height = chip_dim_y * res\n",
428 | "chip_width = chip_dim_x * res\n",
429 | "for row in chips_df_to_sample.iloc[:5000, :].iterrows():\n",
430 | " feature_template = [{'type': 'Feature',\n",
431 | " 'properties': {'id': 0,\n",
432 | " 'center': [0, 0]},\n",
433 | " 'geometry': {'type': 'MultiPolygon',\n",
434 | " 'coordinates': [[[[-281415, 1533615],\n",
435 | " [256185, 1533615],\n",
436 | " [256185, 996015],\n",
437 | " [-281415, 996015],\n",
438 | " [-281415, 1533615]]]]\n",
439 | " }\n",
440 | " }]\n",
441 | " # The subtraction/addition of `res` is needed in the following to change the x and y from center of the pixel to the edge\n",
442 | " chip_x0 = row[1][\"chip_coordinate_x\"] - res / 2 \n",
443 | " chip_y0 = row[1][\"chip_coordinate_y\"] + res / 2\n",
444 | "\n",
445 | " coor = [[[[proj.transform(chip_x0, chip_y0)[0], proj.transform(chip_x0, chip_y0)[1]],\n",
446 | " [proj.transform(chip_x0, chip_y0 - chip_height)[0], proj.transform(chip_x0, chip_y0 - chip_height)[1]],\n",
447 | " [proj.transform(chip_x0 + chip_width, chip_y0 - chip_height)[0], proj.transform(chip_x0 + chip_width, chip_y0 - chip_height)[1]],\n",
448 | " [proj.transform(chip_x0 + chip_width, chip_y0)[0], proj.transform(chip_x0 + chip_width, chip_y0)[1]],\n",
449 | " [proj.transform(chip_x0, chip_y0)[0], proj.transform(chip_x0, chip_y0)[1]]\n",
450 | " ]]]\n",
451 | " chip_cen = [chip_x0 + chip_width / 2, chip_y0 - chip_height / 2]\n",
452 | " chip_cen_lon, chip_cen_lat = proj.transform(chip_cen[0], chip_cen[1])\n",
453 | " \n",
454 | " feature_template[0]['geometry']['coordinates'] = coor\n",
455 | " feature_template[0]['properties']['id'] = row[1][\"chip_id\"]\n",
456 | " feature_template[0]['properties']['center'] = [chip_cen_lon, chip_cen_lat] \n",
457 | " features.extend(feature_template)\n",
458 | "\n",
459 | "feature_collection = {\n",
460 | "'type': 'FeatureCollection',\n",
461 | " 'name': 'chips',\n",
462 | " 'crs': {'type': 'name', 'properties': {'name': 'urn:ogc:def:crs:EPSG::4326'}},\n",
463 | " 'features': features\n",
464 | "}"
465 | ]
466 | },
467 | {
468 | "cell_type": "code",
469 | "execution_count": null,
470 | "id": "1cb65742-ba3d-4af9-b45a-c0330992bd1c",
471 | "metadata": {},
472 | "outputs": [],
473 | "source": [
474 | "with open(\"data/chip_bbox_task_1.geojson\", 'w') as outfile:\n",
475 | " json.dump(feature_collection, outfile)"
476 | ]
477 | },
478 | {
479 | "cell_type": "markdown",
480 | "id": "6996ca06-2148-45f7-a258-8a8a063ca351",
481 | "metadata": {},
482 | "source": [
483 | "Generating the same geojson files but with coordinates in 5070 for chipping"
484 | ]
485 | },
486 | {
487 | "cell_type": "code",
488 | "execution_count": null,
489 | "id": "d30442fc-b578-41d7-88c1-6ba00eaf5903",
490 | "metadata": {},
491 | "outputs": [],
492 | "source": [
493 | "features = []\n",
494 | "chip_height = chip_dim_y * res\n",
495 | "chip_width = chip_dim_x * res\n",
496 | "for row in chips_df_to_sample.iloc[:5000, :].iterrows():\n",
497 | " feature_template = [{'type': 'Feature',\n",
498 | " 'properties': {'id': 0,\n",
499 | " 'center': [0, 0]},\n",
500 | " 'geometry': {'type': 'MultiPolygon',\n",
501 | " 'coordinates': [[[[-281415, 1533615],\n",
502 | " [256185, 1533615],\n",
503 | " [256185, 996015],\n",
504 | " [-281415, 996015],\n",
505 | " [-281415, 1533615]]]]\n",
506 | " }\n",
507 | " }]\n",
508 | " # The subtraction/addition of `res` is needed in the following to change the x and y from center of the pixel to the edge\n",
509 | " chip_x0 = row[1][\"chip_coordinate_x\"] - res / 2\n",
510 | " chip_y0 = row[1][\"chip_coordinate_y\"] + res / 2\n",
511 | "\n",
512 | " coor = [[[[chip_x0, chip_y0],\n",
513 | " [chip_x0, chip_y0 - chip_height],\n",
514 | " [chip_x0 + chip_width, chip_y0 - chip_height],\n",
515 | " [chip_x0 + chip_width, chip_y0],\n",
516 | " [chip_x0, chip_y0]\n",
517 | " ]]]\n",
518 | " chip_cen = [chip_x0 + chip_width / 2, chip_y0 - chip_height / 2]\n",
519 | " chip_cen_lon, chip_cen_lat = [chip_cen[0], chip_cen[1]]\n",
520 | " \n",
521 | " feature_template[0]['geometry']['coordinates'] = coor\n",
522 | " feature_template[0]['properties']['id'] = row[1][\"chip_id\"]\n",
523 | " feature_template[0]['properties']['center'] = [chip_cen_lon, chip_cen_lat] \n",
524 | " features.extend(feature_template)\n",
525 | "\n",
526 | "feature_collection = {\n",
527 | "'type': 'FeatureCollection',\n",
528 | " 'name': 'chips',\n",
529 | " 'crs': {'type': 'name', 'properties': {'name': 'urn:ogc:def:crs:EPSG::5070'}},\n",
530 | " 'features': features\n",
531 | "}"
532 | ]
533 | },
534 | {
535 | "cell_type": "code",
536 | "execution_count": null,
537 | "id": "78903c75-184f-4991-b982-2a68812a262d",
538 | "metadata": {},
539 | "outputs": [],
540 | "source": [
541 | "with open(\"data/chip_bbox_task_1_5070.geojson\", 'w') as outfile:\n",
542 | " json.dump(feature_collection, outfile)"
543 | ]
544 | }
545 | ],
546 | "metadata": {
547 | "kernelspec": {
548 | "display_name": "Python 3 (ipykernel)",
549 | "language": "python",
550 | "name": "python3"
551 | },
552 | "language_info": {
553 | "codemirror_mode": {
554 | "name": "ipython",
555 | "version": 3
556 | },
557 | "file_extension": ".py",
558 | "mimetype": "text/x-python",
559 | "name": "python",
560 | "nbconvert_exporter": "python",
561 | "pygments_lexer": "ipython3",
562 | "version": "3.10.6"
563 | }
564 | },
565 | "nbformat": 4,
566 | "nbformat_minor": 5
567 | }
568 |
--------------------------------------------------------------------------------
/cal_weights.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "id": "4837010a-4e11-4182-9b7b-7a80a714124f",
7 | "metadata": {
8 | "tags": []
9 | },
10 | "outputs": [],
11 | "source": [
12 | "import glob\n",
13 | "import os\n",
14 | "import numpy as np\n",
15 | "import sklearn"
16 | ]
17 | },
18 | {
19 | "cell_type": "markdown",
20 | "id": "a7d149ef-9a52-43d8-817e-d1f1f1de2804",
21 | "metadata": {
22 | "tags": []
23 | },
24 | "source": [
25 | "## Get distribution"
26 | ]
27 | },
28 | {
29 | "cell_type": "code",
30 | "execution_count": null,
31 | "id": "3a9d63ae-dd2a-4de3-8676-7011cea708b7",
32 | "metadata": {
33 | "tags": []
34 | },
35 | "outputs": [],
36 | "source": [
37 | "files = glob.glob(\"/data/chips_filtered/**mask**\")\n",
38 | "files[0:10]"
39 | ]
40 | },
41 | {
42 | "cell_type": "code",
43 | "execution_count": null,
44 | "id": "5756b368-d286-4edb-becd-c725e43cf518",
45 | "metadata": {
46 | "tags": []
47 | },
48 | "outputs": [],
49 | "source": [
50 | "import rasterio\n",
51 | "\n",
52 | "def open_tiff(fname):\n",
53 | " \n",
54 | " with rasterio.open(fname, \"r\") as src:\n",
55 | " \n",
56 | " data = src.read()\n",
57 | " \n",
58 | " return data\n"
59 | ]
60 | },
61 | {
62 | "cell_type": "code",
63 | "execution_count": null,
64 | "id": "8b44450b-58f9-4380-9e45-9d9fe3c91f39",
65 | "metadata": {
66 | "tags": []
67 | },
68 | "outputs": [],
69 | "source": [
70 | "y = []\n",
71 | "for file in files:\n",
72 | " \n",
73 | " data = open_tiff(file)\n",
74 | " y.append(data.flatten())"
75 | ]
76 | },
77 | {
78 | "cell_type": "code",
79 | "execution_count": null,
80 | "id": "bce00c2f-43fd-4386-8871-3a158e61ef78",
81 | "metadata": {},
82 | "outputs": [],
83 | "source": [
84 | "y_stack = np.vstack(y)"
85 | ]
86 | },
87 | {
88 | "cell_type": "code",
89 | "execution_count": null,
90 | "id": "cc1295ba-f95e-4022-8de9-73e383faef4f",
91 | "metadata": {
92 | "tags": []
93 | },
94 | "outputs": [],
95 | "source": [
96 | "y_flatten = y_stack.flatten()"
97 | ]
98 | },
99 | {
100 | "cell_type": "code",
101 | "execution_count": null,
102 | "id": "4fb6f0fa-df8f-4734-a878-1971a7e540aa",
103 | "metadata": {
104 | "tags": []
105 | },
106 | "outputs": [],
107 | "source": [
108 | "class_weights=sklearn.utils.class_weight.compute_class_weight('balanced',classes=np.unique(np.ravel(y_flatten,order='C')),y=np.ravel(y_flatten,order='C'))"
109 | ]
110 | },
111 | {
112 | "cell_type": "code",
113 | "execution_count": null,
114 | "id": "09330241-f7cb-4c4a-a212-cc386612ba93",
115 | "metadata": {
116 | "tags": []
117 | },
118 | "outputs": [],
119 | "source": [
120 | "class_weights"
121 | ]
122 | },
123 | {
124 | "cell_type": "code",
125 | "execution_count": null,
126 | "id": "3235db77-d8d7-4fad-ac50-508d155c0a4f",
127 | "metadata": {},
128 | "outputs": [],
129 | "source": [
130 | "class_weights / np.sum(class_weights)"
131 | ]
132 | },
133 | {
134 | "cell_type": "code",
135 | "execution_count": null,
136 | "id": "e46c057b-3898-4ee8-9394-fd463e946430",
137 | "metadata": {},
138 | "outputs": [],
139 | "source": []
140 | }
141 | ],
142 | "metadata": {
143 | "kernelspec": {
144 | "display_name": "Python 3 (ipykernel)",
145 | "language": "python",
146 | "name": "python3"
147 | },
148 | "language_info": {
149 | "codemirror_mode": {
150 | "name": "ipython",
151 | "version": 3
152 | },
153 | "file_extension": ".py",
154 | "mimetype": "text/x-python",
155 | "name": "python",
156 | "nbconvert_exporter": "python",
157 | "pygments_lexer": "ipython3",
158 | "version": "3.10.6"
159 | }
160 | },
161 | "nbformat": 4,
162 | "nbformat_minor": 5
163 | }
164 |
--------------------------------------------------------------------------------
/calc_mean_sd.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": 1,
6 | "id": "679dcc20-c510-4ccc-a1fa-54e901e24659",
7 | "metadata": {},
8 | "outputs": [],
9 | "source": [
10 | "import rasterio\n",
11 | "from glob import glob\n",
12 | "import numpy as np\n",
13 | "import pandas as pd\n",
14 | "import pickle\n",
15 | "import os\n",
16 | "import multiprocessing as mp"
17 | ]
18 | },
19 | {
20 | "cell_type": "code",
21 | "execution_count": 2,
22 | "id": "0c5c2f1e-c73c-4b3f-9018-ce2540381754",
23 | "metadata": {},
24 | "outputs": [],
25 | "source": [
26 | "def calc_mean_std(kwargs):\n",
27 | " i_band = kwargs[\"i_band\"]\n",
28 | " band = kwargs[\"band\"]\n",
29 | " clip_val = 1.5\n",
30 | " \n",
31 | " chips = glob(\"/data/chips_filtered/*_merged.tif\")\n",
32 | " means = []\n",
33 | " stds = []\n",
34 | " \n",
35 | " print(\"Bands:\", band)\n",
36 | " vals_all = np.ndarray([0])\n",
37 | " for k in range(len(chips)):\n",
38 | " file = chips[k]\n",
39 | " with rasterio.open(file) as src:\n",
40 | " vals = src.read(band).flatten()\n",
41 | " vals_all = np.concatenate([vals_all, vals])\n",
42 | " band_mean = vals_all.mean()\n",
43 | " band_std = vals_all.std()\n",
44 | " vals_all_clipped = np.clip(vals_all, np.nanpercentile(vals_all, clip_val), \n",
45 | " np.nanpercentile(vals_all, 100 - clip_val))\n",
46 | " band_min = np.min(vals_all_clipped)\n",
47 | " band_max = np.max(vals_all_clipped)\n",
48 | " f= open(\"/data/band_stats/band_values_\" + str(i_band) + \".txt\",\"w+\")\n",
49 | " f.write(\"Mean: %f\\r\\n\" % (band_mean))\n",
50 | " f.write(\"Std: %f\\r\\n\" % (band_std))\n",
51 | " f.write(\"Min: %f\\r\\n\" % (band_min))\n",
52 | " f.write(\"Max: %f\\r\\n\" % (band_max))\n",
53 | " f.close\n",
54 | " \n",
55 | " with open(\"/data/band_stats/band_values_\" + str(i_band) + '.list', 'wb') as file:\n",
56 | " pickle.dump(vals_all, file)"
57 | ]
58 | },
59 | {
60 | "cell_type": "code",
61 | "execution_count": 3,
62 | "id": "9b0cf5d7-ed7d-48a2-ac56-8856cfb6ad6b",
63 | "metadata": {},
64 | "outputs": [],
65 | "source": [
66 | "i_band = list(range(1, 7))\n",
67 | "band_ids = [[1,7,13], [2,8,14], [3,9,15], [4,10,16], [5,11,17], [6,12,18]]\n",
68 | "df = pd.DataFrame({\"i_band\": i_band, \"band\": band_ids})"
69 | ]
70 | },
71 | {
72 | "cell_type": "code",
73 | "execution_count": 4,
74 | "id": "722807aa-92eb-4270-b612-83c4e3ae513e",
75 | "metadata": {},
76 | "outputs": [
77 | {
78 | "name": "stdout",
79 | "output_type": "stream",
80 | "text": [
81 | "Bands:Bands:Bands:Bands:Bands:Bands: [1, 7, 13][3, 9, 15][2, 8, 14][4, 10, 16][5, 11, 17][6, 12, 18]\n",
82 | "\n",
83 | "\n",
84 | "\n",
85 | "\n",
86 | "\n"
87 | ]
88 | }
89 | ],
90 | "source": [
91 | "with mp.Pool(processes=6) as pool:\n",
92 | " pool.map(calc_mean_std, df.to_dict('records'))"
93 | ]
94 | },
95 | {
96 | "cell_type": "code",
97 | "execution_count": null,
98 | "id": "b30d0b71-93b0-48dc-b868-3eae8b7ffacc",
99 | "metadata": {},
100 | "outputs": [],
101 | "source": []
102 | }
103 | ],
104 | "metadata": {
105 | "kernelspec": {
106 | "display_name": "Python 3 (ipykernel)",
107 | "language": "python",
108 | "name": "python3"
109 | },
110 | "language_info": {
111 | "codemirror_mode": {
112 | "name": "ipython",
113 | "version": 3
114 | },
115 | "file_extension": ".py",
116 | "mimetype": "text/x-python",
117 | "name": "python",
118 | "nbconvert_exporter": "python",
119 | "pygments_lexer": "ipython3",
120 | "version": "3.10.6"
121 | }
122 | },
123 | "nbformat": 4,
124 | "nbformat_minor": 5
125 | }
126 |
--------------------------------------------------------------------------------
/data/cdl_freq.csv:
--------------------------------------------------------------------------------
1 | ,CDL_val,count,pct,class,cumulative sum,new_class_value
2 | 47,176,64894733,41.3472537,grassland,41.3472537,1
3 | 43,141,32217747,20.52732629,decid forest,61.87457999,2
4 | 44,142,10946138,6.974260067,evergreen forest,68.84884006,2
5 | 11,24,9692821,6.175717357,winter wheat,75.02455741,7
6 | 45,143,7523517,4.793559535,mixed forest,79.81811695,2
7 | 46,152,6401382,4.078598576,shrubs,83.89671553,3
8 | 38,121,4787456,3.050296205,deveoped/open,86.94701173,4
9 | 37,111,4776016,3.043007284,open water,89.99001901,5
10 | 48,190,3655213,2.328895001,woody wetlands,92.31891402,6
11 | 39,122,2434906,1.551384395,developed/low,93.87029841,4
12 | 18,37,2191127,1.396062204,other hay/non alfalfa,95.26636061,8
13 | 40,123,1104208,0.703538888,developed/mid,95.9698995,4
14 | 0,1,1101765,0.701982347,corn,96.67188185,9
15 | 4,5,1069963,0.681719911,soybeans,97.35360176,10
16 | 1,2,821831,0.523624234,cotton,97.87722599,11
17 | 3,4,717610,0.457220507,sorghum,98.3344465,12
18 | 12,26,486041,0.309677837,dbl crop winter wheat soybeans,98.64412434,13
19 | 31,61,333869,0.212722445,fallow,98.85684678,13
20 | 17,36,309904,0.197453302,alfalfa,99.05430009,13
21 | 41,124,259954,0.165627987,developed/high,99.21992807,13
22 | 49,195,221153,0.140906184,herbaceous wetlands,99.36083426,13
23 | 13,27,211067,0.134479955,rye,99.49531421,13
24 | 42,131,206421,0.131519787,barren,99.626834,13
25 | 14,28,156346,0.099614829,oats,99.72644883,13
26 | 50,205,91803,0.05849168,triticale,99.78494051,13
27 | 29,59,91528,0.058316465,sod/grass seed,99.84325697,13
28 | 35,74,59830,0.038120292,pecans,99.88137726,13
29 | 59,236,37599,0.023955956,dbl crop winter wheat/sorghm,99.90533322,13
30 | 2,3,35231,0.022447201,rice,99.92778042,13
31 | 8,21,31645,0.020162404,barley,99.94794283,13
32 | 61,238,18871,0.012023534,dbl crop winter wheat/cotton,99.95996636,13
33 | 27,57,17325,0.01103851,herbs,99.97100487,13
34 | 55,225,8088,0.005153216,dbl crop winter wheat/corn,99.97615809,13
35 | 16,31,7657,0.004878607,canola,99.98103669,13
36 | 6,10,7234,0.004609096,peanuts,99.98564579,13
37 | 15,29,6908,0.004401387,millet,99.99004718,13
38 | 21,44,4437,0.002827005,other crops,99.99287418,13
39 | 5,6,2031,0.001294038,sunflower,99.99416822,13
40 | 26,53,2025,0.001290215,peas,99.99545843,13
41 | 10,23,1463,0.000932141,spring wheat,99.99639057,13
42 | 36,92,1156,0.000736538,aquaculture,99.99712711,13
43 | 28,58,670,0.000426886,clover/wildflowers,99.997554,13
44 | 9,22,583,0.000371455,durum wheat,99.99792545,13
45 | 32,67,518,0.00033004,peaches,99.99825549,13
46 | 24,48,481,0.000306466,watermelons,99.99856196,13
47 | 57,228,436,0.000277795,dbl crop triticale/corn,99.99883975,13
48 | 56,226,426,0.000271423,dbl crop oats/corn,99.99911118,13
49 | 7,12,375,0.000238929,tobacco,99.99935011,13
50 | 60,237,166,0.000105766,dbl crop barley/corn,99.99945587,13
51 | 62,240,155,9.88E-05,dbl crop soybeans/oats,99.99955467,13
52 | 20,42,125,7.96E-05,dry beans,99.99963427,13
53 | 22,45,119,7.58E-05,sugarcane,99.99971007,13
54 | 30,60,111,7.07E-05,switchgrass,99.99978077,13
55 | 34,72,100,6.37E-05,citrus,99.99984447,13
56 | 63,241,59,3.76E-05,dbl crop corn/soybeans,99.99988207,13
57 | 64,242,39,2.48E-05,blueberries,99.99990687,13
58 | 66,254,38,2.42E-05,dbl crop barley/soybeans,99.99993107,13
59 | 25,49,34,2.17E-05,onions,99.99995277,13
60 | 33,69,30,1.91E-05,grapes,99.99997187,13
61 | 51,211,18,1.15E-05,olives,99.99998337,13
62 | 23,47,14,8.92E-06,misc vegs and fruits,99.99999229,13
63 | 52,216,3,1.91E-06,peppers,99.9999942,13
64 | 58,229,3,1.91E-06,pumpkins,99.99999611,13
65 | 19,41,2,1.27E-06,sugarbeets,99.99999738,13
66 | 53,221,2,1.27E-06,strawberries,99.99999865,13
67 | 54,222,1,6.37E-07,squash,99.99999929,13
68 | 65,243,1,6.37E-07,cabbage,99.99999993,13
--------------------------------------------------------------------------------
/data/cdl_total_dst.csv:
--------------------------------------------------------------------------------
1 | old_class_value,total_count,percentage,cum_percentage,class_name,new_class_value_16,new_class_value
2 | 152,2063333477,0.240438461,0.240438461,Shrubland,1,1
3 | 176,1482158236,0.172714614,0.413153076,Grassland/Pasture,2,1
4 | 142,1087546299,0.126730827,0.539883903,Evergreen Forest,3,2
5 | 141,898140095,0.104659487,0.64454339,Deciduous Forest,3,2
6 | 1,409599488,0.047730273,0.692273663,Corn,4,3
7 | 5,399644525,0.04657023,0.738843893,Soybeans,5,4
8 | 190,381270149,0.04442908,0.783272973,Woody Wetlands,6,5
9 | 143,254963448,0.02971067,0.812983643,Mixed Forest,3,2
10 | 121,235937778,0.027493625,0.840477268,Developed/Open Space,7,6
11 | 111,147580217,0.017197395,0.857674663,Open Water,8,7
12 | 122,143906418,0.01676929,0.874443953,Developed/Low Intensity,7,6
13 | 24,125713120,0.014649241,0.889093194,Winter Wheat,9,8
14 | 37,123121151,0.014347201,0.903440395,Other Hay/Non Alfalfa,10,1
15 | 195,110102692,0.012830171,0.916270566,Herbaceous Wetlands,6,5
16 | 36,92198712,0.010743836,0.927014402,Alfalfa,11,9
17 | 61,87070549,0.010146256,0.937160657,Fallow/Idle Cropland,12,10
18 | 131,85538350,0.00996771,0.947128367,Barren,13,6
19 | 123,83797654,0.009764868,0.956893235,Developed/Med Intensity,7,6
20 | 2,68184110,0.007945435,0.96483867,Cotton,14,11
21 | 23,55358735,0.006450905,0.971289576,Spring Wheat,16,13
22 | 4,37035573,0.004315723,0.975605299,Sorghum,15,12
23 | 124,29135611,0.003395147,0.979000447,Developed/High Intensity,7,6
24 | 26,17366266,0.002023676,0.981024122,Dbl Crop WinWht/Soybeans,16,13
25 | 21,11778073,0.001372489,0.982396611,Barley,16,13
26 | 3,10557196,0.001230221,0.983626832,Rice,16,13
27 | 31,9813943,0.00114361,0.984770442,Canola,16,13
28 | 75,8421500,0.00098135,0.985751792,Almonds,16,13
29 | 28,7699542,0.000897221,0.986649013,Oats,16,13
30 | 6,6947944,0.000809638,0.987458651,Sunflower,16,13
31 | 10,6647579,0.000774637,0.988233288,Peanuts,16,13
32 | 22,6614531,0.000770786,0.989004073,Durum Wheat,16,13
33 | 59,5724785,0.000667104,0.989671178,Sod/Grass Seed,16,13
34 | 42,5663396,0.000659951,0.990331128,Dry Beans,16,13
35 | 45,5188141,0.00060457,0.990935698,Sugarcane,16,13
36 | 41,5111809,0.000595675,0.991531372,Sugarbeets,16,13
37 | 53,4990859,0.00058158,0.992112953,Peas,16,13
38 | 69,4870658,0.000567574,0.992680527,Grapes,16,13
39 | 43,4588956,0.000534747,0.993215274,Potatoes,16,13
40 | 29,4305524,0.000501719,0.993716993,Millet,16,13
41 | 205,3721143,0.000433622,0.994150614,Triticale,16,13
42 | 27,3289706,0.000383347,0.994533961,Rye,16,13
43 | 212,3147860,0.000366817,0.994900778,Oranges,16,13
44 | 52,3021952,0.000352145,0.995252924,Lentils,16,13
45 | 204,2974708,0.00034664,0.995599564,Pistachios,16,13
46 | 236,2340687,0.000272758,0.995872322,Dbl Crop WinWht/Sorghum,16,13
47 | 92,2309432,0.000269116,0.996141438,Aquaculture,16,13
48 | 76,2295688,0.000267515,0.996408953,Walnuts,16,13
49 | 74,2157328,0.000251392,0.996660344,Pecans,16,13
50 | 68,2074192,0.000241704,0.996902048,Apples,16,13
51 | 72,1950228,0.000227258,0.997129306,Citrus,16,13
52 | 51,1712708,0.00019958,0.997328887,Chick Peas,16,13
53 | 238,1566542,0.000182548,0.997511434,Dbl Crop WinWht/Cotton,16,13
54 | 225,1339057,0.000156039,0.997667474,Dbl Crop WinWht/Corn,16,13
55 | 54,1260950,0.000146937,0.997814411,Tomatoes,16,13
56 | 35,1200015,0.000139837,0.997954248,Mustard,16,13
57 | 32,1139213,0.000132752,0.998086999,Flaxseed,16,13
58 | 12,1126998,0.000131328,0.998218327,Sweet Corn,16,13
59 | 66,1014301,0.000118196,0.998336523,Cherries,16,13
60 | 242,1005848,0.000117211,0.998453734,Blueberries,16,13
61 | 33,987435,0.000115065,0.998568799,Safflower,16,13
62 | 228,877818,0.000102291,0.99867109,Dbl Crop Triticale/Corn,16,13
63 | 58,829673,9.67E-05,0.998767771,Clover/Wildflowers,16,13
64 | 112,769484,8.97E-05,0.998857438,Perennial Ice/Snow ,16,13
65 | 13,757270,8.82E-05,0.998945682,Pop or Orn Corn,16,13
66 | 71,723261,8.43E-05,0.999029963,Other Tree Crops,16,13
67 | 49,600910,7.00E-05,0.999099987,Onions,16,13
68 | 46,506246,5.90E-05,0.999158979,Sweet Potatoes,16,13
69 | 11,452380,5.27E-05,0.999211695,Tobacco,16,13
70 | 67,446577,5.20E-05,0.999263734,Peaches,16,13
71 | 70,401521,4.68E-05,0.999310523,Christmas Trees,16,13
72 | 56,355934,4.15E-05,0.999351999,Hops,16,13
73 | 226,337423,3.93E-05,0.999391319,Dbl Crop Oats/Corn,16,13
74 | 211,326907,3.81E-05,0.999429413,Olives,16,13
75 | 210,320923,3.74E-05,0.99946681,Prunes,16,13
76 | 47,313035,3.65E-05,0.999503288,Misc Vegs & Fruits,16,13
77 | 44,309625,3.61E-05,0.999539368,Other Crops,16,13
78 | 220,297567,3.47E-05,0.999574043,Plums,16,13
79 | 77,240253,2.80E-05,0.99960204,Pears,16,13
80 | 14,238986,2.78E-05,0.999629889,Mint,16,13
81 | 50,223026,2.60E-05,0.999655878,Cucumbers,16,13
82 | 215,214953,2.50E-05,0.999680926,Avocados,16,13
83 | 227,214938,2.50E-05,0.999705973,Lettuce,16,13
84 | 57,208988,2.44E-05,0.999730326,Herbs,16,13
85 | 206,193907,2.26E-05,0.999752922,Carrots,16,13
86 | 254,190177,2.22E-05,0.999775083,Dbl Crop Barley/Soybeans,16,13
87 | 208,162057,1.89E-05,0.999793967,Garlic,16,13
88 | 217,150035,1.75E-05,0.999811451,Pomegranates,16,13
89 | 48,142960,1.67E-05,0.99982811,Watermelons,16,13
90 | 219,138410,1.61E-05,0.999844238,Greens,16,13
91 | 229,136210,1.59E-05,0.999860111,Pumpkins,16,13
92 | 240,109040,1.27E-05,0.999872817,Dbl Crop Soybeans/Oats,16,13
93 | 243,102473,1.19E-05,0.999884758,Cabbage,16,13
94 | 237,91744,1.07E-05,0.999895449,Dbl Crop Barley/Corn,16,13
95 | 39,88874,1.04E-05,0.999905805,Buckwheat,16,13
96 | 216,83896,9.78E-06,0.999915582,Peppers,16,13
97 | 222,78828,9.19E-06,0.999924768,Squash,16,13
98 | 221,67953,7.92E-06,0.999932686,Strawberries,16,13
99 | 25,66598,7.76E-06,0.999940447,Other Small Grains,16,13
100 | 214,60268,7.02E-06,0.99994747,Broccoli,16,13
101 | 209,51235,5.97E-06,0.99995344,Cantaloupes,16,13
102 | 38,45729,5.33E-06,0.999958769,Camelina,16,13
103 | 246,44337,5.17E-06,0.999963935,Radishes,16,13
104 | 55,41258,4.81E-06,0.999968743,Caneberries,16,13
105 | 207,40047,4.67E-06,0.99997341,Asparagus,16,13
106 | 232,32816,3.82E-06,0.999977234,Dbl Crop Lettuce/Cotton,16,13
107 | 60,31618,3.68E-06,0.999980918,Switchgrass,16,13
108 | 250,25416,2.96E-06,0.99998388,Cranberries,16,13
109 | 247,19755,2.30E-06,0.999986182,Turnips,16,13
110 | 30,18951,2.21E-06,0.99998839,Speltz,16,13
111 | 244,18422,2.15E-06,0.999990537,Cauliflower,16,13
112 | 231,17228,2.01E-06,0.999992544,Dbl Crop Lettuce/Cantaloupe,16,13
113 | 241,15738,1.83E-06,0.999994378,Dbl Crop Corn/Soybeans,16,13
114 | 213,12093,1.41E-06,0.999995788,Honeydew Melons,16,13
115 | 218,11613,1.35E-06,0.999997141,Nectarines,16,13
116 | 224,7010,8.17E-07,0.999997958,Vetch,16,13
117 | 245,3955,4.61E-07,0.999998419,Celery,16,13
118 | 34,3771,4.39E-07,0.999998858,Rape Seed,16,13
119 | 248,3550,4.14E-07,0.999999272,Eggplants,16,13
120 | 223,2087,2.43E-07,0.999999515,Apricots,16,13
121 | 233,1805,2.10E-07,0.999999725,Dbl Crop Lettuce/Barley,16,13
122 | 249,1610,1.88E-07,0.999999913,Gourds,16,13
123 | 230,748,8.72E-08,1,Dbl Crop Lettuce/Durum Wht,16,13
124 | 88,0,0,1,Nonag/Undefined,16,13
125 | 239,0,0,1,Dbl Crop Soybeans/Cotton,16,13
126 | 235,0,0,1,Dbl Crop Barley/Sorghum,16,13
127 | 234,0,0,1,Dbl Crop Durum Wht/Sorghum,16,13
128 | 87,0,0,1,Wetlands,16,13
129 | 62,0,0,1,Pasture/Grass,16,13
130 | 63,0,0,1,Forest,16,13
131 | 64,0,0,1,Shrubland,16,13
132 | 65,0,0,1,Barren,16,13
133 | 81,0,0,1,Clouds/No Data,16,13
134 | 83,0,0,1,Water,16,13
135 | 82,0,0,1,Developed,16,13
--------------------------------------------------------------------------------
/data/sentinel_tile_grid.kml:
--------------------------------------------------------------------------------
1 | version https://git-lfs.github.com/spec/v1
2 | oid sha256:ffcd58f1443f01f1c6f96baeef83a96239f86b50911729ed12195690ebc4de61
3 | size 108817408
4 |
--------------------------------------------------------------------------------
/data/tile_tracker.csv:
--------------------------------------------------------------------------------
1 | tile,exclude,tif_download,tif_reproject,chip,filter_chips
2 | 13TDE,False,False,False,False,False
3 | 16SDD,False,False,False,False,False
4 | 13SFV,False,False,False,False,False
5 | 14TNS,False,False,False,False,False
6 | 14UMU,False,False,False,False,False
7 | 17TKG,False,False,False,False,False
8 | 16SGA,False,False,False,False,False
9 | 17TPH,False,False,False,False,False
10 | 14SLG,False,False,False,False,False
11 | 18TWN,False,False,False,False,False
12 | 18STE,False,False,False,False,False
13 | 16TCN,False,False,False,False,False
14 | 14SLF,False,False,False,False,False
15 | 14TNR,False,False,False,False,False
16 | 15TVK,False,False,False,False,False
17 | 16SED,False,False,False,False,False
18 | 13SGA,False,False,False,False,False
19 | 16RFV,False,False,False,False,False
20 | 11TPH,False,False,False,False,False
21 | 18TUN,False,False,False,False,False
22 | 14SPH,False,False,False,False,False
23 | 11TLM,False,False,False,False,False
24 | 12TWT,False,False,False,False,False
25 | 16TFN,False,False,False,False,False
26 | 18SVH,False,False,False,False,False
27 | 15TUL,False,False,False,False,False
28 | 13TEN,False,False,False,False,False
29 | 12TYR,False,False,False,False,False
30 | 16TEM,False,False,False,False,False
31 | 15STT,False,False,False,False,False
32 | 14SKG,False,False,False,False,False
33 | 16RGV,False,False,False,False,False
34 | 15TYK,False,False,False,False,False
35 | 14UNU,False,False,False,False,False
36 | 14SPG,False,False,False,False,False
37 | 16SCE,False,False,False,False,False
38 | 15TVL,False,False,False,False,False
39 | 17SNT,False,False,False,False,False
40 | 13SGU,False,False,False,False,False
41 | 14SKE,False,False,False,False,False
42 | 16SCG,False,False,False,False,False
43 | 13TEE,False,False,False,False,False
44 | 17SQV,False,False,False,False,False
45 | 16SCF,False,False,False,False,False
46 | 13SGV,False,False,False,False,False
47 | 16TDP,False,False,False,False,False
48 | 17TLJ,False,False,False,False,False
49 | 16TGN,False,False,False,False,False
50 | 16TEN,False,False,False,False,False
51 | 15SUV,False,False,False,False,False
52 | 14TQL,False,False,False,False,False
53 | 16TDQ,False,False,False,False,False
54 | 14SPF,False,False,False,False,False
55 | 11TNM,False,False,False,False,False
56 | 15TXE,False,False,False,False,False
57 | 16SDF,False,False,False,False,False
58 | 15TWL,False,False,False,False,False
59 | 14SNH,False,False,False,False,False
60 | 14TQS,False,False,False,False,False
61 | 15SVC,False,False,False,False,False
62 | 14TMP,False,False,False,False,False
63 | 16TDL,False,False,False,False,False
64 | 16TCQ,False,False,False,False,False
65 | 14TMR,False,False,False,False,False
66 | 14SLH,False,False,False,False,False
67 | 14TPT,False,False,False,False,False
68 | 15TWG,False,False,False,False,False
69 | 18TWQ,False,False,False,False,False
70 | 12UYU,False,False,False,False,False
71 | 14TMS,False,False,False,False,False
72 | 14SNA,False,False,False,False,False
73 | 16TFP,False,False,False,False,False
74 | 18TVK,False,False,False,False,False
75 | 17SQU,False,False,False,False,False
76 | 17TLH,False,False,False,False,False
77 | 12TXR,False,False,False,False,False
78 | 14TPR,False,False,False,False,False
79 | 18TWK,False,False,False,False,False
80 | 18SVJ,False,False,False,False,False
81 | 15SYA,False,False,False,False,False
82 | 13TGF,False,False,False,False,False
83 | 10TER,False,False,False,False,False
84 | 18SUJ,False,False,False,False,False
85 | 16SCH,False,False,False,False,False
86 | 15SYD,False,False,False,False,False
87 | 17TQH,False,False,False,False,False
88 | 16TCR,False,False,False,False,False
89 | 16TGP,False,False,False,False,False
90 | 15RTN,False,False,False,False,False
91 | 18TXQ,False,False,False,False,False
92 | 16TBM,False,False,False,False,False
93 | 13TFF,False,False,False,False,False
94 | 16TFM,False,False,False,False,False
95 | 14TPQ,False,False,False,False,False
96 | 14TMT,False,False,False,False,False
97 | 16TEP,False,False,False,False,False
98 | 14SMH,False,False,False,False,False
99 | 14TNL,False,False,False,False,False
100 | 15TYH,False,False,False,False,False
101 | 16SDE,False,False,False,False,False
102 | 16TDN,False,False,False,False,False
103 | 14SMG,False,False,False,False,False
104 | 15TXK,False,False,False,False,False
105 | 14RQT,False,False,False,False,False
106 | 10TGR,False,False,False,False,False
107 | 17TQJ,False,False,False,False,False
108 | 16TGK,False,False,False,False,False
109 | 18TVL,False,False,False,False,False
110 | 18SUH,False,False,False,False,False
111 | 16SDA,False,False,False,False,False
112 | 12TVR,False,False,False,False,False
113 | 10UEU,False,False,False,False,False
114 | 16TBK,False,False,False,False,False
115 | 16SBG,False,False,False,False,False
116 | 17SKR,False,False,False,False,False
117 | 16TGQ,False,False,False,False,False
118 | 14TQT,False,False,False,False,False
119 | 14SKH,False,False,False,False,False
120 | 17TMF,False,False,False,False,False
121 | 15TUM,False,False,False,False,False
122 | 14TLP,False,False,False,False,False
123 | 14RPP,False,False,False,False,False
124 | 16TGM,False,False,False,False,False
125 | 16RCU,False,False,False,False,False
126 | 11TLL,False,False,False,False,False
127 | 16TGR,False,False,False,False,False
128 | 14TQR,False,False,False,False,False
129 | 14SNJ,False,False,False,False,False
130 | 15SUC,False,False,False,False,False
131 | 14UPU,False,False,False,False,False
132 | 15SXA,False,False,False,False,False
133 | 12TUN,False,False,False,False,False
134 | 16SDH,False,False,False,False,False
135 | 16SBH,False,False,False,False,False
136 | 15TVJ,False,False,False,False,False
137 | 16SEF,False,False,False,False,False
138 | 17SNU,False,False,False,False,False
139 | 12SVB,False,False,False,False,False
140 | 14TNT,False,False,False,False,False
141 | 11TNJ,False,False,False,False,False
142 | 12TVT,False,False,False,False,False
143 | 14TLT,False,False,False,False,False
144 | 14SMJ,False,False,False,False,False
145 | 16TEL,False,False,False,False,False
146 | 11TQN,False,False,False,False,False
147 | 15TTE,False,False,False,False,False
148 | 17SLR,False,False,False,False,False
149 | 18STD,False,False,False,False,False
150 | 16TCP,False,False,False,False,False
151 | 13SFC,False,False,False,False,False
152 | 13TCM,False,False,False,False,False
153 | 12TYS,False,False,False,False,False
154 | 18TUK,False,False,False,False,False
155 | 15TWK,False,False,False,False,False
156 | 15SYU,False,False,False,False,False
157 | 16SEH,False,False,False,False,False
158 | 10SGF,False,False,False,False,False
159 | 14SPA,False,False,False,False,False
160 | 15TXJ,False,False,False,False,False
161 | 16TFL,False,False,False,False,False
162 | 18SUF,False,False,False,False,False
163 | 15SUD,False,False,False,False,False
164 | 13UDP,False,False,False,False,False
165 | 15SXU,False,False,False,False,False
166 | 16SDJ,False,False,False,False,False
167 | 18TTK,False,False,False,False,False
168 | 14RPS,False,False,False,False,False
169 | 14TLS,False,False,False,False,False
170 | 10SGE,False,False,False,False,False
171 | 15SWU,False,False,False,False,False
172 | 13UEP,False,False,False,False,False
173 | 11UQP,False,False,False,False,False
174 | 13TFM,False,False,False,False,False
175 | 15TVM,False,False,False,False,False
176 | 12TVK,False,False,False,False,False
177 | 14RPQ,False,False,False,False,False
178 | 13SFA,False,False,False,False,False
179 | 14ULU,False,False,False,False,False
180 | 11TNN,False,False,False,False,False
181 | 16SBF,False,False,False,False,False
182 | 11TMN,False,False,False,False,False
183 | 16RDU,False,False,False,False,False
184 | 14RNQ,False,False,False,False,False
185 | 14SQB,False,False,False,False,False
186 | 16SBC,False,False,False,False,False
187 | 15TUK,False,False,False,False,False
188 | 14TLQ,False,False,False,False,False
189 | 15SXD,False,False,False,False,False
190 | 15TXL,False,False,False,False,False
191 | 14TMK,False,False,False,False,False
192 | 18TWL,False,False,False,False,False
193 | 17SMS,False,False,False,False,False
194 | 01SBS,False,False,False,False,False
195 | 15SVS,False,False,False,False,False
196 | 10TFQ,False,False,False,False,False
197 | 14RPU,False,False,False,False,False
198 | 15SYC,False,False,False,False,False
199 | 15SVR,False,False,False,False,False
200 | 14SKD,False,False,False,False,False
201 | 14TLK,False,False,False,False,False
202 | 13SFU,False,False,False,False,False
203 | 14SKF,False,False,False,False,False
204 | 14SQA,False,False,False,False,False
205 | 17RLQ,False,False,False,False,False
206 | 17TLF,False,False,False,False,False
207 | 13SGC,False,False,False,False,False
208 | 15TXF,False,False,False,False,False
209 | 18STJ,False,False,False,False,False
210 | 15TWE,False,False,False,False,False
211 | 18TVP,False,False,False,False,False
212 | 16TCL,False,False,False,False,False
213 | 10SFF,False,False,False,False,False
214 | 15TTF,False,False,False,False,False
215 | 13TFE,False,False,False,False,False
216 | 15TVF,False,False,False,False,False
217 | 17TKE,False,False,False,False,False
218 | 14ULV,False,False,False,False,False
219 | 17SPC,False,False,False,False,False
220 | 17SQD,False,False,False,False,False
221 | 17RKQ,False,False,False,False,False
222 | 14TNK,False,False,False,False,False
223 | 17TLE,False,False,False,False,False
224 | 14RQU,False,False,False,False,False
225 | 12TYQ,False,False,False,False,False
226 | 16SEE,False,False,False,False,False
227 | 14SPJ,False,False,False,False,False
228 | 16SEG,False,False,False,False,False
229 | 13SFD,False,False,False,False,False
230 | 16SDG,False,False,False,False,False
231 | 11UPP,False,False,False,False,False
232 | 13TGH,False,False,False,False,False
233 | 11TML,False,False,False,False,False
234 | 14TQK,False,False,False,False,False
235 | 15TVG,False,False,False,False,False
236 | 15SYT,False,False,False,False,False
237 | 15SYS,False,False,False,False,False
238 | 14SLJ,False,False,False,False,False
239 | 13TGN,False,False,False,False,False
240 | 14UMV,False,False,False,False,False
241 | 13TCK,False,False,False,False,False
242 | 14SLE,False,False,False,False,False
243 | 14TNQ,False,False,False,False,False
244 | 14RPV,False,False,False,False,False
245 | 15TWJ,False,False,False,False,False
246 | 15TYE,False,False,False,False,False
247 | 13SGB,False,False,False,False,False
248 | 15TYL,False,False,False,False,False
249 | 16SBJ,False,False,False,False,False
250 | 12TVP,False,False,False,False,False
251 | 15SWC,False,False,False,False,False
252 | 11TQH,False,False,False,False,False
253 | 11TMM,False,False,False,False,False
254 | 14TPN,False,False,False,False,False
255 | 16SBE,False,False,False,False,False
256 | 13TFG,False,False,False,False,False
257 | 12TVM,False,False,False,False,False
258 | 15TYJ,False,False,False,False,False
259 | 13TEK,False,False,False,False,False
260 | 16SEA,False,False,False,False,False
261 | 12TUM,False,False,False,False,False
262 | 17SMR,False,False,False,False,False
263 | 15TXG,False,False,False,False,False
264 | 12UXU,False,False,False,False,False
265 | 11TMJ,False,False,False,False,False
266 | 17TPJ,False,False,False,False,False
267 | 14TPL,False,False,False,False,False
268 | 14SMB,False,False,False,False,False
269 | 11TMK,False,False,False,False,False
270 | 14RPR,False,False,False,False,False
271 | 17TPE,False,False,False,False,False
272 | 10TDT,False,False,False,False,False
273 | 17RKP,False,False,False,False,False
274 | 17SMT,False,False,False,False,False
275 | 14SQC,False,False,False,False,False
276 | 16RDV,False,False,False,False,False
277 | 15SWD,False,False,False,False,False
278 | 15SXV,False,False,False,False,False
279 | 12TXT,False,False,False,False,False
280 | 16TFQ,False,False,False,False,False
281 | 10TEM,False,False,False,False,False
282 | 14TQQ,False,False,False,False,False
283 | 15RVP,False,False,False,False,False
284 | 17RMK,False,False,False,False,False
285 | 15STD,False,False,False,False,False
286 | 15RXQ,False,False,False,False,False
287 | 10TFR,False,False,False,False,False
288 | 17SLS,False,False,False,False,False
289 | 16TDR,False,False,False,False,False
290 | 10TFS,False,False,False,False,False
291 | 15TUJ,False,False,False,False,False
292 | 18STF,False,False,False,False,False
293 | 16SCB,False,False,False,False,False
294 | 14RNR,False,False,False,False,False
295 | 15SUB,False,False,False,False,False
296 | 12TWS,False,False,False,False,False
297 | 14TNP,False,False,False,False,False
298 | 13SGD,False,False,False,False,False
299 | 14TPS,False,False,False,False,False
300 | 15TWH,False,False,False,False,False
301 | 18TUL,False,False,False,False,False
302 | 14UQU,False,False,False,False,False
303 | 11TNL,False,False,False,False,False
304 | 15SXC,False,False,False,False,False
305 | 16SCJ,False,False,False,False,False
306 | 10SGD,False,False,False,False,False
307 | 12UUU,False,False,False,False,False
308 | 13UCP,False,False,False,False,False
309 | 15STC,False,False,False,False,False
310 | 16REV,False,False,False,False,False
311 | 15TWF,False,False,False,False,False
312 | 13UFQ,False,False,False,False,False
313 | 11SPS,False,False,False,False,False
314 | 13SCR,False,False,False,False,False
315 | 16TCM,False,False,False,False,False
316 | 15TUH,False,False,False,False,False
317 | 14TMQ,False,False,False,False,False
318 | 17SKD,False,False,False,False,False
319 | 15SVD,False,False,False,False,False
320 | 10SFH,False,False,False,False,False
321 | 15TTG,False,False,False,False,False
322 | 15SVU,False,False,False,False,False
323 | 14TNN,False,False,False,False,False
324 | 14SME,False,False,False,False,False
325 | 12UVU,False,False,False,False,False
326 | 13TFN,False,False,False,False,False
327 | 13SFB,False,False,False,False,False
328 | 16TFK,False,False,False,False,False
329 | 10TDS,False,False,False,False,False
330 | 15TXH,False,False,False,False,False
331 | 18TVN,False,False,False,False,False
332 | 14TKK,False,False,False,False,False
333 | 16SFG,False,False,False,False,False
334 | 15TUE,False,False,False,False,False
335 | 17SLD,False,False,False,False,False
336 | 12SYF,False,False,False,False,False
337 | 13TEG,False,False,False,False,False
338 | 17SPU,False,False,False,False,False
339 | 15TVE,False,False,False,False,False
340 | 16TBL,False,False,False,False,False
341 | 14RNT,False,False,False,False,False
342 | 18TTL,False,False,False,False,False
343 | 16SFJ,False,False,False,False,False
344 | 10TGS,False,False,False,False,False
345 | 13SBC,False,False,False,False,False
346 | 16SGJ,False,False,False,False,False
347 | 15RWQ,False,False,False,False,False
348 | 14TLL,False,False,False,False,False
349 | 16SDB,False,False,False,False,False
350 | 15SWR,False,False,False,False,False
351 | 10UGU,False,False,False,False,False
352 | 15SXT,False,False,False,False,False
353 | 14UQV,False,False,False,False,False
354 | 15RXP,False,False,False,False,False
355 | 10UEV,False,False,False,False,False
356 | 16SCC,False,False,False,False,False
357 | 16SEJ,False,False,False,False,False
358 | 15TUF,False,False,False,False,False
359 | 13UFP,False,False,False,False,False
360 | 16SCD,False,False,False,False,False
361 | 14TMN,False,False,False,False,False
362 | 16SFH,False,False,False,False,False
363 | 11TLN,False,False,False,False,False
364 | 15SVT,False,False,False,False,False
365 | 15SYB,False,False,False,False,False
366 | 14UNV,False,False,False,False,False
367 | 12SUB,False,False,False,False,False
368 | 14TPK,False,False,False,False,False
369 | 16TDM,False,False,False,False,False
370 | 17RML,False,False,False,False,False
371 | 16TEK,False,False,False,False,False
372 | 11UMP,False,False,False,False,False
373 | 18TVQ,False,False,False,False,False
374 | 10TFM,False,False,False,False,False
375 | 14SQJ,False,False,False,False,False
376 | 17RMN,False,False,False,False,False
377 | 12TVN,False,False,False,False,False
378 | 13TDM,False,False,False,False,False
379 | 13TDG,False,False,False,False,False
380 | 15RTP,False,False,False,False,False
381 | 17TNF,False,False,False,False,False
382 | 13TEF,False,False,False,False,False
383 | 12SXG,False,False,False,False,False
384 | 17RLN,False,False,False,False,False
385 | 10TGL,False,False,False,False,False
386 | 15STB,False,False,False,False,False
387 | 12TXQ,False,False,False,False,False
388 | 16RFU,False,False,False,False,False
389 | 11SKD,False,False,False,False,False
390 | 15SUT,False,False,False,False,False
391 | 14SKJ,False,False,False,False,False
392 | 12TUP,False,False,False,False,False
393 | 14SMC,False,False,False,False,False
394 | 14RMT,False,False,False,False,False
395 | 18STH,False,False,False,False,False
396 | 10UFU,False,False,False,False,False
397 | 12TTM,False,False,False,False,False
398 | 17SQC,False,False,False,False,False
399 | 15SXR,False,False,False,False,False
400 | 14SLB,False,False,False,False,False
401 | 15RYQ,False,False,False,False,False
402 | 13TDF,False,False,False,False,False
403 | 13TFL,False,False,False,False,False
404 | 12UWV,False,False,False,False,False
405 | 15RYP,False,False,False,False,False
406 | 15SVV,False,False,False,False,False
407 | 14SNG,False,False,False,False,False
408 | 14RQS,False,False,False,False,False
409 | 10TDM,False,False,False,False,False
410 | 10SGG,False,False,False,False,False
411 | 15TUG,False,False,False,False,False
412 | 15STA,False,False,False,False,False
413 | 10SEH,False,False,False,False,False
414 | 13UDQ,False,False,False,False,False
415 | 17SKC,False,False,False,False,False
416 | 17SKS,False,False,False,False,False
417 | 14TML,False,False,False,False,False
418 | 15TYF,False,False,False,False,False
419 | 17RLP,False,False,False,False,False
420 | 17SPV,False,False,False,False,False
421 | 12TVL,False,False,False,False,False
422 | 11TNK,False,False,False,False,False
423 | 11TQJ,False,False,False,False,False
424 | 15TWN,False,False,False,False,False
425 | 14TPP,False,False,False,False,False
426 | 16TDK,False,False,False,False,False
427 | 13TEL,False,False,False,False,False
428 | 16TEQ,False,False,False,False,False
429 | 14SLD,False,False,False,False,False
430 | 16SEB,False,False,False,False,False
431 | 12TXS,False,False,False,False,False
432 | 14SNE,False,False,False,False,False
433 | 12TUL,False,False,False,False,False
434 | 17SPT,False,False,False,False,False
435 | 13TEM,False,False,False,False,False
436 | 12TYT,False,False,False,False,False
437 | 14SMD,False,False,False,False,False
438 | 12TVS,False,False,False,False,False
439 | 10SFE,False,False,False,False,False
440 | 11SLD,False,False,False,False,False
441 | 13TGE,False,False,False,False,False
442 | 14UPV,False,False,False,False,False
443 | 14TQP,False,False,False,False,False
444 | 14SNF,False,False,False,False,False
445 | 15SXS,False,False,False,False,False
446 | 17RLM,False,False,False,False,False
447 | 17SNA,False,False,False,False,False
448 | 18TUM,False,False,False,False,False
449 | 13SGT,False,False,False,False,False
450 | 12TYP,False,False,False,False,False
451 | 19TEM,False,False,False,False,False
452 | 14SNC,False,False,False,False,False
453 | 15TUN,False,False,False,False,False
454 | 14SPE,False,False,False,False,False
455 | 15SYV,False,False,False,False,False
456 | 15RWP,False,False,False,False,False
457 | 18SUE,False,False,False,False,False
458 | 15SUR,False,False,False,False,False
459 | 10TES,False,False,False,False,False
460 | 13SEC,False,False,False,False,False
461 | 15RUP,False,False,False,False,False
462 | 14RQV,False,False,False,False,False
463 | 10SFG,False,False,False,False,False
464 | 12TXK,False,False,False,False,False
465 | 18TXN,False,False,False,False,False
466 | 16TCK,False,False,False,False,False
467 | 10TGT,False,False,False,False,False
468 | 16SBD,False,False,False,False,False
469 | 14TQN,False,False,False,False,False
470 | 17TNG,False,False,False,False,False
471 | 15STR,False,False,False,False,False
472 | 13TGK,False,False,False,False,False
473 | 10SDJ,False,False,False,False,False
474 | 17SNS,False,False,False,False,False
475 | 18TUP,False,False,False,False,False
476 | 14TPM,False,False,False,False,False
477 | 12UVV,False,False,False,False,False
478 | 11TNH,False,False,False,False,False
479 | 11TQG,False,False,False,False,False
480 | 15SVB,False,False,False,False,False
481 | 12TUT,False,False,False,False,False
482 | 14SND,False,False,False,False,False
483 | 11ULP,False,False,False,False,False
484 | 18TXP,False,False,False,False,False
485 | 13SER,False,False,False,False,False
486 | 13SBB,False,False,False,False,False
487 | 12SUJ,False,False,False,False,False
488 | 11UNQ,False,False,False,False,False
489 | 10TDQ,False,False,False,False,False
490 | 10TFT,False,False,False,False,False
491 | 16TFR,False,False,False,False,False
492 | 12SXA,False,False,False,False,False
493 |
--------------------------------------------------------------------------------
/gen_dist.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "code",
5 | "execution_count": null,
6 | "id": "70329e02-4cf2-44e3-af5b-3ff8cffb1342",
7 | "metadata": {},
8 | "outputs": [],
9 | "source": [
10 | "import pandas as pd\n",
11 | "import rasterio as rio\n",
12 | "import numpy as np"
13 | ]
14 | },
15 | {
16 | "cell_type": "code",
17 | "execution_count": null,
18 | "id": "8778c7d4-619a-4ca6-a337-7cb547433cc5",
19 | "metadata": {},
20 | "outputs": [],
21 | "source": [
22 | "class_names = {\n",
23 | " 0 : \"No Data\",\n",
24 | " 1\t:\t\"Natural Vegetation\"\t,\n",
25 | " 2\t:\t\"Forest\"\t,\n",
26 | " 3\t:\t\"Corn\"\t,\n",
27 | " 4\t:\t\"Soybeans\"\t,\n",
28 | " 5\t:\t\"Wetlands\"\t,\n",
29 | " 6\t:\t\"Developed/Barren\"\t,\n",
30 | " 7\t:\t\"Open Water\"\t,\n",
31 | " 8\t:\t\"Winter Wheat\"\t,\n",
32 | " 9\t:\t\"Alfalfa\"\t,\n",
33 | " 10\t:\t\"Fallow/Idle Cropland\"\t,\n",
34 | " 11\t:\t\"Cotton\"\t,\n",
35 | " 12\t:\t\"Sorghum\"\t,\n",
36 | " 13\t:\t\"Other\"\t\n",
37 | " }"
38 | ]
39 | },
40 | {
41 | "cell_type": "code",
42 | "execution_count": null,
43 | "id": "c27e6a7d-04b4-4c11-8fa4-be6627db34b4",
44 | "metadata": {},
45 | "outputs": [],
46 | "source": [
47 | "val_df = pd.read_csv(\"val_ids.csv\")\n",
48 | "train_df = pd.read_csv(\"train_ids.csv\")"
49 | ]
50 | },
51 | {
52 | "cell_type": "code",
53 | "execution_count": null,
54 | "id": "e8c2d788-e05b-47cf-b279-ac7b8de87be8",
55 | "metadata": {},
56 | "outputs": [],
57 | "source": [
58 | "# Create an empty df to load chips\n",
59 | "df_columns = [\"chip_id\"]\n",
60 | "for i in range(14):\n",
61 | " df_columns.append(i)\n",
62 | "val_chips_df = pd.DataFrame(columns = df_columns)\n",
63 | "\n",
64 | "for id in val_df.ids:\n",
65 | " filepath = \"/data/chips_filtered/chip_\"+id+\".mask.tif\"\n",
66 | " with rio.open(filepath, \"r\") as src:\n",
67 | " img = src.read(1)\n",
68 | " classes, class_counts = np.unique(img, return_counts=True)\n",
69 | " counts = np.zeros(14)\n",
70 | " counts[classes] = class_counts\n",
71 | " val_chips_df.loc[len(val_chips_df.index)] = [id] + counts.tolist()\n",
72 | " "
73 | ]
74 | },
75 | {
76 | "cell_type": "code",
77 | "execution_count": null,
78 | "id": "ced4707c-d05d-4292-86ac-15352ad3bfa2",
79 | "metadata": {},
80 | "outputs": [],
81 | "source": [
82 | "import matplotlib.pyplot as plt\n",
83 | "plt.bar(range(14), val_chips_df.loc[:, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]].sum()/np.sum(val_chips_df.loc[:, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]].sum()))\n",
84 | "plt.xticks(range(14), list(class_names.values()), rotation = 90)\n",
85 | "plt.draw()\n",
86 | "plt.savefig('validation_dst.png',\n",
87 | " dpi=150,\n",
88 | " format='png',\n",
89 | " bbox_inches='tight'\n",
90 | " )"
91 | ]
92 | },
93 | {
94 | "cell_type": "code",
95 | "execution_count": null,
96 | "id": "851fb5ae-cc8b-4e13-a468-0fadf9b78308",
97 | "metadata": {},
98 | "outputs": [],
99 | "source": [
100 | "# Create an empty df to load chips\n",
101 | "df_columns = [\"chip_id\"]\n",
102 | "for i in range(14):\n",
103 | " df_columns.append(i)\n",
104 | "train_chips_df = pd.DataFrame(columns = df_columns)\n",
105 | "\n",
106 | "for id in train_df.ids:\n",
107 | " filepath = \"/data/chips_filtered/chip_\"+id+\".mask.tif\"\n",
108 | " with rio.open(filepath, \"r\") as src:\n",
109 | " img = src.read(1)\n",
110 | " classes, class_counts = np.unique(img, return_counts=True)\n",
111 | " counts = np.zeros(14)\n",
112 | " counts[classes] = class_counts\n",
113 | " train_chips_df.loc[len(train_chips_df.index)] = [id] + counts.tolist()\n",
114 | " "
115 | ]
116 | },
117 | {
118 | "cell_type": "code",
119 | "execution_count": null,
120 | "id": "939c2d36-90a4-4bb3-8913-83b015fee5a6",
121 | "metadata": {},
122 | "outputs": [],
123 | "source": [
124 | "import matplotlib.pyplot as plt\n",
125 | "plt.bar(range(14), train_chips_df.loc[:, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]].sum()/np.sum(train_chips_df.loc[:, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]].sum()))\n",
126 | "plt.xticks(range(14), list(class_names.values()), rotation = 90)\n",
127 | "plt.draw()\n",
128 | "plt.savefig('training_dst.png',\n",
129 | " dpi=150,\n",
130 | " format='png',\n",
131 | " bbox_inches='tight'\n",
132 | " )"
133 | ]
134 | },
135 | {
136 | "cell_type": "code",
137 | "execution_count": null,
138 | "id": "414c3380-e272-4eb3-9866-174475eb096c",
139 | "metadata": {},
140 | "outputs": [],
141 | "source": []
142 | }
143 | ],
144 | "metadata": {
145 | "kernelspec": {
146 | "display_name": "Python 3 (ipykernel)",
147 | "language": "python",
148 | "name": "python3"
149 | },
150 | "language_info": {
151 | "codemirror_mode": {
152 | "name": "ipython",
153 | "version": 3
154 | },
155 | "file_extension": ".py",
156 | "mimetype": "text/x-python",
157 | "name": "python",
158 | "nbconvert_exporter": "python",
159 | "pygments_lexer": "ipython3",
160 | "version": "3.10.6"
161 | }
162 | },
163 | "nbformat": 4,
164 | "nbformat_minor": 5
165 | }
166 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy
2 | matplotlib
3 | fiona
4 | rioxarray==0.3.1
5 | geopandas==0.9.0
6 | boto3==1.27.0
7 | requests==2.31.0
8 | xmltodict==0.13.0
9 |
--------------------------------------------------------------------------------
/train_ids.csv:
--------------------------------------------------------------------------------
1 | ids
2 | 257_266
3 | 328_501
4 | 171_477
5 | 236_281
6 | 134_482
7 | 120_493
8 | 161_390
9 | 102_442
10 | 129_508
11 | 213_402
12 | 417_328
13 | 198_322
14 | 114_283
15 | 109_419
16 | 108_418
17 | 237_362
18 | 152_478
19 | 236_291
20 | 108_414
21 | 352_543
22 | 387_331
23 | 228_292
24 | 129_472
25 | 257_275
26 | 130_475
27 | 134_440
28 | 111_416
29 | 250_584
30 | 141_474
31 | 241_442
32 | 228_329
33 | 248_592
34 | 344_537
35 | 169_516
36 | 116_114
37 | 231_302
38 | 158_523
39 | 216_335
40 | 358_324
41 | 139_473
42 | 257_267
43 | 237_189
44 | 148_472
45 | 147_470
46 | 095_349
47 | 246_593
48 | 321_532
49 | 103_320
50 | 347_398
51 | 137_437
52 | 193_406
53 | 246_584
54 | 323_314
55 | 238_286
56 | 039_108
57 | 051_166
58 | 320_534
59 | 245_593
60 | 131_445
61 | 107_415
62 | 131_506
63 | 322_533
64 | 240_332
65 | 319_516
66 | 147_444
67 | 136_478
68 | 212_296
69 | 194_313
70 | 122_510
71 | 356_567
72 | 279_551
73 | 173_468
74 | 234_370
75 | 208_392
76 | 126_133
77 | 234_435
78 | 197_261
79 | 122_444
80 | 063_354
81 | 100_102
82 | 210_390
83 | 155_413
84 | 300_543
85 | 240_273
86 | 311_469
87 | 173_585
88 | 305_300
89 | 159_468
90 | 269_420
91 | 191_367
92 | 386_324
93 | 349_542
94 | 131_443
95 | 282_561
96 | 148_470
97 | 196_362
98 | 132_487
99 | 147_486
100 | 085_254
101 | 220_306
102 | 217_315
103 | 130_170
104 | 320_544
105 | 256_445
106 | 257_441
107 | 124_123
108 | 252_261
109 | 132_440
110 | 150_495
111 | 076_351
112 | 235_599
113 | 205_338
114 | 069_196
115 | 218_328
116 | 118_446
117 | 036_147
118 | 159_593
119 | 189_403
120 | 142_474
121 | 172_561
122 | 158_618
123 | 115_318
124 | 319_529
125 | 181_511
126 | 243_335
127 | 172_588
128 | 159_481
129 | 182_400
130 | 199_324
131 | 307_525
132 | 272_266
133 | 239_283
134 | 135_472
135 | 184_239
136 | 175_400
137 | 313_236
138 | 233_267
139 | 203_455
140 | 119_447
141 | 176_238
142 | 163_478
143 | 182_238
144 | 171_586
145 | 111_417
146 | 332_505
147 | 087_201
148 | 105_427
149 | 234_373
150 | 150_481
151 | 119_323
152 | 145_413
153 | 370_350
154 | 151_418
155 | 130_504
156 | 121_445
157 | 108_402
158 | 133_483
159 | 169_612
160 | 233_332
161 | 258_445
162 | 038_107
163 | 130_480
164 | 219_317
165 | 329_393
166 | 132_439
167 | 059_119
168 | 227_301
169 | 196_333
170 | 240_587
171 | 146_393
172 | 214_292
173 | 139_421
174 | 134_364
175 | 250_259
176 | 116_448
177 | 272_259
178 | 245_443
179 | 149_489
180 | 302_301
181 | 217_328
182 | 099_102
183 | 160_532
184 | 252_274
185 | 055_144
186 | 179_520
187 | 152_404
188 | 228_300
189 | 194_310
190 | 109_414
191 | 111_421
192 | 123_508
193 | 127_482
194 | 319_525
195 | 174_503
196 | 185_392
197 | 066_347
198 | 115_438
199 | 128_492
200 | 180_023
201 | 218_324
202 | 219_476
203 | 167_411
204 | 138_615
205 | 107_334
206 | 153_469
207 | 140_493
208 | 313_519
209 | 258_266
210 | 105_402
211 | 126_124
212 | 106_406
213 | 189_493
214 | 135_494
215 | 062_084
216 | 149_503
217 | 182_606
218 | 350_328
219 | 084_359
220 | 239_281
221 | 003_062
222 | 281_449
223 | 142_485
224 | 228_286
225 | 174_521
226 | 036_149
227 | 106_249
228 | 137_496
229 | 269_477
230 | 225_438
231 | 089_369
232 | 242_471
233 | 193_431
234 | 198_016
235 | 064_269
236 | 417_321
237 | 331_502
238 | 118_437
239 | 241_280
240 | 330_508
241 | 139_436
242 | 128_322
243 | 239_595
244 | 255_272
245 | 276_565
246 | 236_443
247 | 330_497
248 | 317_528
249 | 324_509
250 | 125_492
251 | 075_347
252 | 104_320
253 | 107_410
254 | 183_512
255 | 251_264
256 | 271_395
257 | 140_486
258 | 374_298
259 | 370_343
260 | 267_437
261 | 192_325
262 | 170_601
263 | 134_423
264 | 116_492
265 | 242_472
266 | 236_277
267 | 266_276
268 | 231_497
269 | 106_411
270 | 332_504
271 | 140_488
272 | 186_380
273 | 111_595
274 | 202_451
275 | 092_322
276 | 249_587
277 | 265_582
278 | 095_327
279 | 357_370
280 | 152_494
281 | 265_482
282 | 230_282
283 | 128_435
284 | 330_512
285 | 332_508
286 | 268_373
287 | 153_473
288 | 239_333
289 | 205_293
290 | 344_460
291 | 146_501
292 | 196_278
293 | 109_424
294 | 060_070
295 | 058_192
296 | 164_405
297 | 130_472
298 | 185_516
299 | 268_410
300 | 210_391
301 | 246_582
302 | 150_468
303 | 151_489
304 | 204_288
305 | 136_485
306 | 168_407
307 | 304_545
308 | 232_265
309 | 319_540
310 | 245_333
311 | 139_390
312 | 210_466
313 | 109_321
314 | 333_497
315 | 156_389
316 | 108_336
317 | 131_471
318 | 152_240
319 | 173_579
320 | 215_309
321 | 103_102
322 | 202_439
323 | 168_580
324 | 223_446
325 | 119_554
326 | 233_288
327 | 323_522
328 | 064_355
329 | 219_444
330 | 316_523
331 | 253_271
332 | 064_117
333 | 255_447
334 | 055_199
335 | 216_337
336 | 104_342
337 | 191_432
338 | 147_440
339 | 153_401
340 | 106_412
341 | 070_097
342 | 315_519
343 | 237_334
344 | 319_542
345 | 190_349
346 | 166_436
347 | 106_409
348 | 421_325
349 | 228_279
350 | 231_293
351 | 141_493
352 | 162_592
353 | 229_592
354 | 180_611
355 | 306_425
356 | 180_262
357 | 256_443
358 | 097_334
359 | 238_329
360 | 137_432
361 | 151_474
362 | 240_475
363 | 213_432
364 | 283_448
365 | 218_432
366 | 069_353
367 | 223_284
368 | 252_273
369 | 317_517
370 | 252_446
371 | 198_027
372 | 099_319
373 | 237_591
374 | 287_453
375 | 161_469
376 | 063_268
377 | 219_311
378 | 144_488
379 | 231_275
380 | 256_446
381 | 206_293
382 | 244_333
383 | 219_336
384 | 070_349
385 | 331_495
386 | 034_097
387 | 105_335
388 | 181_264
389 | 086_203
390 | 080_600
391 | 250_269
392 | 127_320
393 | 132_444
394 | 230_329
395 | 344_469
396 | 086_216
397 | 221_441
398 | 116_444
399 | 185_404
400 | 291_450
401 | 122_507
402 | 217_437
403 | 109_435
404 | 128_317
405 | 134_484
406 | 117_112
407 | 207_466
408 | 256_278
409 | 113_298
410 | 222_293
411 | 168_385
412 | 283_431
413 | 286_301
414 | 208_372
415 | 215_310
416 | 115_040
417 | 107_417
418 | 341_499
419 | 089_215
420 | 214_315
421 | 220_437
422 | 352_420
423 | 136_435
424 | 254_259
425 | 056_198
426 | 135_471
427 | 106_106
428 | 092_320
429 | 065_192
430 | 219_432
431 | 110_434
432 | 330_509
433 | 249_270
434 | 130_426
435 | 105_404
436 | 127_504
437 | 328_510
438 | 147_443
439 | 166_524
440 | 161_521
441 | 281_402
442 | 132_508
443 | 209_449
444 | 365_319
445 | 282_263
446 | 317_532
447 | 112_417
448 | 193_368
449 | 149_492
450 | 104_333
451 | 168_588
452 | 051_235
453 | 305_343
454 | 128_431
455 | 227_279
456 | 197_393
457 | 150_490
458 | 130_507
459 | 159_515
460 | 036_113
461 | 211_428
462 | 185_433
463 | 254_271
464 | 263_287
465 | 129_425
466 | 238_276
467 | 050_114
468 | 176_027
469 | 192_409
470 | 181_484
471 | 318_523
472 | 177_408
473 | 121_466
474 | 230_302
475 | 131_421
476 | 336_533
477 | 274_479
478 | 093_103
479 | 315_516
480 | 310_542
481 | 297_429
482 | 086_361
483 | 171_049
484 | 040_101
485 | 177_604
486 | 131_440
487 | 165_467
488 | 226_282
489 | 277_405
490 | 131_470
491 | 183_235
492 | 135_434
493 | 234_375
494 | 125_436
495 | 216_603
496 | 146_495
497 | 342_534
498 | 148_492
499 | 244_268
500 | 167_522
501 | 167_340
502 | 270_409
503 | 114_423
504 | 101_330
505 | 148_477
506 | 084_361
507 | 106_410
508 | 316_517
509 | 248_589
510 | 217_434
511 | 200_465
512 | 154_478
513 | 131_302
514 | 108_353
515 | 118_326
516 | 285_432
517 | 160_533
518 | 102_104
519 | 111_356
520 | 226_308
521 | 331_494
522 | 183_446
523 | 119_503
524 | 117_504
525 | 128_478
526 | 191_585
527 | 266_275
528 | 279_256
529 | 086_336
530 | 030_042
531 | 032_079
532 | 098_308
533 | 229_286
534 | 010_063
535 | 194_329
536 | 180_264
537 | 112_405
538 | 113_331
539 | 214_424
540 | 220_479
541 | 199_450
542 | 105_293
543 | 218_457
544 | 158_462
545 | 112_246
546 | 171_338
547 | 147_489
548 | 225_371
549 | 246_435
550 | 187_405
551 | 145_493
552 | 270_316
553 | 129_381
554 | 320_318
555 | 083_208
556 | 138_490
557 | 219_334
558 | 171_292
559 | 188_365
560 | 183_283
561 | 157_388
562 | 160_615
563 | 311_519
564 | 046_044
565 | 064_067
566 | 067_292
567 | 095_343
568 | 067_282
569 | 247_584
570 | 217_285
571 | 076_349
572 | 112_440
573 | 247_420
574 | 129_509
575 | 105_104
576 | 032_080
577 | 193_583
578 | 224_294
579 | 198_311
580 | 119_448
581 | 258_443
582 | 354_327
583 | 165_476
584 | 134_361
585 | 071_352
586 | 136_495
587 | 037_102
588 | 293_548
589 | 149_490
590 | 155_412
591 | 255_438
592 | 337_506
593 | 157_597
594 | 334_501
595 | 281_270
596 | 113_329
597 | 183_517
598 | 213_282
599 | 063_115
600 | 123_323
601 | 045_229
602 | 253_260
603 | 144_496
604 | 329_497
605 | 248_588
606 | 222_330
607 | 097_356
608 | 220_307
609 | 185_244
610 | 214_466
611 | 240_276
612 | 319_522
613 | 151_265
614 | 127_480
615 | 164_420
616 | 298_351
617 | 242_448
618 | 199_317
619 | 256_442
620 | 145_472
621 | 180_244
622 | 251_288
623 | 119_340
624 | 064_191
625 | 254_286
626 | 149_265
627 | 137_493
628 | 282_558
629 | 221_324
630 | 096_303
631 | 249_290
632 | 111_414
633 | 200_441
634 | 242_335
635 | 321_338
636 | 128_426
637 | 117_447
638 | 319_524
639 | 322_313
640 | 138_480
641 | 079_095
642 | 173_586
643 | 179_341
644 | 047_172
645 | 189_406
646 | 154_484
647 | 196_258
648 | 062_082
649 | 149_501
650 | 124_504
651 | 174_402
652 | 115_503
653 | 326_500
654 | 242_590
655 | 128_051
656 | 333_498
657 | 193_311
658 | 061_084
659 | 334_499
660 | 236_595
661 | 182_424
662 | 110_340
663 | 171_431
664 | 138_442
665 | 168_325
666 | 128_508
667 | 254_421
668 | 230_490
669 | 162_481
670 | 087_335
671 | 333_499
672 | 301_456
673 | 165_583
674 | 295_541
675 | 140_429
676 | 115_478
677 | 085_264
678 | 224_371
679 | 281_254
680 | 060_196
681 | 064_354
682 | 114_320
683 | 132_506
684 | 205_292
685 | 019_091
686 | 259_414
687 | 204_409
688 | 343_540
689 | 139_366
690 | 291_554
691 | 234_337
692 | 270_408
693 | 149_264
694 | 094_339
695 | 053_192
696 | 191_403
697 | 171_264
698 | 329_489
699 | 164_415
700 | 223_372
701 | 147_478
702 | 243_468
703 | 327_521
704 | 252_264
705 | 159_520
706 | 322_522
707 | 251_279
708 | 248_427
709 | 204_389
710 | 218_315
711 | 267_382
712 | 166_265
713 | 153_375
714 | 316_343
715 | 252_266
716 | 236_444
717 | 093_354
718 | 196_274
719 | 219_257
720 | 145_473
721 | 269_270
722 | 252_584
723 | 120_323
724 | 241_586
725 | 107_414
726 | 301_382
727 | 153_601
728 | 092_356
729 | 252_284
730 | 281_272
731 | 163_403
732 | 221_330
733 | 391_325
734 | 207_402
735 | 309_544
736 | 042_116
737 | 184_518
738 | 121_504
739 | 118_339
740 | 117_449
741 | 219_437
742 | 164_516
743 | 120_126
744 | 239_444
745 | 127_127
746 | 160_465
747 | 309_380
748 | 238_466
749 | 066_117
750 | 233_334
751 | 217_442
752 | 396_322
753 | 223_480
754 | 293_351
755 | 230_303
756 | 171_353
757 | 088_361
758 | 197_361
759 | 220_184
760 | 145_483
761 | 269_268
762 | 330_523
763 | 098_340
764 | 234_295
765 | 172_182
766 | 272_562
767 | 254_267
768 | 130_382
769 | 231_288
770 | 234_441
771 | 185_399
772 | 152_469
773 | 154_252
774 | 091_253
775 | 105_407
776 | 238_445
777 | 179_585
778 | 119_509
779 | 219_429
780 | 059_087
781 | 040_114
782 | 086_272
783 | 276_402
784 | 236_367
785 | 227_309
786 | 247_580
787 | 333_504
788 | 157_466
789 | 196_400
790 | 212_447
791 | 138_494
792 | 343_532
793 | 095_322
794 | 182_425
795 | 104_308
796 | 213_391
797 | 163_524
798 | 131_487
799 | 204_390
800 | 165_468
801 | 203_376
802 | 123_494
803 | 165_524
804 | 109_439
805 | 218_460
806 | 074_351
807 | 114_478
808 | 130_381
809 | 089_303
810 | 281_266
811 | 226_314
812 | 132_472
813 | 032_183
814 | 148_488
815 | 352_421
816 | 201_288
817 | 155_483
818 | 206_296
819 | 135_361
820 | 169_590
821 | 235_290
822 | 119_444
823 | 239_446
824 | 218_430
825 | 165_402
826 | 148_494
827 | 251_261
828 | 260_484
829 | 106_334
830 | 206_391
831 | 297_353
832 | 312_160
833 | 365_309
834 | 335_504
835 | 083_175
836 | 117_505
837 | 141_442
838 | 241_447
839 | 227_295
840 | 268_370
841 | 129_446
842 | 111_439
843 | 118_448
844 | 162_478
845 | 063_079
846 | 230_353
847 | 125_439
848 | 262_448
849 | 161_482
850 | 217_290
851 | 238_335
852 | 174_586
853 | 234_445
854 | 233_369
855 | 194_409
856 | 240_594
857 | 112_321
858 | 286_552
859 | 117_486
860 | 129_478
861 | 296_134
862 | 134_304
863 | 127_303
864 | 250_583
865 | 034_104
866 | 191_397
867 | 117_438
868 | 251_320
869 | 180_362
870 | 325_505
871 | 253_445
872 | 146_340
873 | 227_331
874 | 149_468
875 | 219_446
876 | 326_517
877 | 100_341
878 | 121_560
879 | 235_331
880 | 162_460
881 | 155_475
882 | 073_268
883 | 233_273
884 | 194_280
885 | 067_087
886 | 117_320
887 | 142_361
888 | 207_359
889 | 255_277
890 | 198_395
891 | 221_308
892 | 240_588
893 | 266_438
894 | 229_297
895 | 120_340
896 | 202_017
897 | 186_489
898 | 169_346
899 | 181_256
900 | 157_467
901 | 252_272
902 | 291_454
903 | 265_277
904 | 318_339
905 | 078_198
906 | 126_495
907 | 115_447
908 | 181_610
909 | 203_288
910 | 127_477
911 | 236_271
912 | 223_468
913 | 283_281
914 | 143_486
915 | 147_479
916 | 110_435
917 | 251_270
918 | 249_337
919 | 271_389
920 | 096_316
921 | 161_403
922 | 152_470
923 | 230_283
924 | 251_445
925 | 275_406
926 | 256_444
927 | 119_126
928 | 168_382
929 | 198_366
930 | 151_493
931 | 320_525
932 | 204_458
933 | 148_469
934 | 217_432
935 | 133_441
936 | 331_498
937 | 220_316
938 | 130_508
939 | 372_342
940 | 164_517
941 | 100_322
942 | 245_334
943 | 227_275
944 | 213_439
945 | 074_096
946 | 166_411
947 | 326_524
948 | 003_063
949 | 242_588
950 | 225_372
951 | 047_106
952 | 202_367
953 | 155_474
954 | 231_594
955 | 165_520
956 | 133_363
957 | 108_352
958 | 123_495
959 | 226_605
960 | 063_120
961 | 177_428
962 | 218_325
963 | 205_480
964 | 230_459
965 | 218_440
966 | 178_605
967 | 054_182
968 | 118_422
969 | 245_435
970 | 249_592
971 | 148_413
972 | 216_264
973 | 060_117
974 | 106_026
975 | 087_360
976 | 164_414
977 | 182_607
978 | 134_499
979 | 233_477
980 | 237_472
981 | 150_265
982 | 208_457
983 | 153_472
984 | 112_448
985 | 177_269
986 | 183_518
987 | 137_476
988 | 112_342
989 | 232_372
990 | 206_401
991 | 255_266
992 | 193_401
993 | 181_239
994 | 245_582
995 | 213_429
996 | 166_379
997 | 333_509
998 | 260_443
999 | 182_484
1000 | 345_460
1001 | 419_323
1002 | 152_479
1003 | 248_586
1004 | 208_383
1005 | 108_438
1006 | 170_595
1007 | 241_598
1008 | 181_605
1009 | 226_278
1010 | 254_587
1011 | 108_403
1012 | 189_402
1013 | 127_125
1014 | 164_583
1015 | 163_252
1016 | 123_482
1017 | 151_501
1018 | 139_437
1019 | 127_478
1020 | 141_475
1021 | 239_330
1022 | 262_583
1023 | 307_410
1024 | 217_287
1025 | 418_313
1026 | 264_482
1027 | 181_396
1028 | 048_231
1029 | 193_314
1030 | 279_403
1031 | 232_441
1032 | 169_271
1033 | 205_283
1034 | 254_443
1035 | 318_534
1036 | 026_110
1037 | 322_508
1038 | 114_420
1039 | 287_412
1040 | 218_336
1041 | 047_104
1042 | 035_148
1043 | 194_433
1044 | 219_436
1045 | 187_409
1046 | 194_364
1047 | 180_242
1048 | 196_325
1049 | 148_473
1050 | 389_324
1051 | 187_414
1052 | 269_370
1053 | 316_514
1054 | 223_483
1055 | 127_442
1056 | 209_311
1057 | 210_308
1058 | 214_428
1059 | 211_308
1060 | 248_591
1061 | 185_408
1062 | 152_584
1063 | 164_526
1064 | 092_336
1065 | 190_284
1066 | 109_436
1067 | 198_457
1068 | 070_294
1069 | 037_107
1070 | 250_329
1071 | 364_348
1072 | 173_580
1073 | 136_434
1074 | 072_095
1075 | 230_358
1076 | 318_540
1077 | 059_086
1078 | 234_332
1079 | 271_410
1080 | 093_104
1081 | 102_326
1082 | 197_311
1083 | 185_263
1084 | 125_479
1085 | 126_478
1086 | 202_463
1087 | 239_332
1088 | 195_314
1089 | 240_474
1090 | 132_507
1091 | 146_482
1092 | 142_487
1093 | 153_479
1094 | 255_417
1095 | 146_489
1096 | 252_279
1097 | 260_431
1098 | 140_436
1099 | 171_588
1100 | 238_463
1101 | 217_311
1102 | 041_103
1103 | 182_426
1104 | 301_454
1105 | 152_433
1106 | 123_491
1107 | 327_523
1108 | 214_311
1109 | 165_521
1110 | 309_470
1111 | 175_511
1112 | 251_271
1113 | 060_079
1114 | 180_342
1115 | 176_604
1116 | 218_423
1117 | 089_364
1118 | 126_442
1119 | 133_508
1120 | 106_413
1121 | 088_302
1122 | 214_429
1123 | 173_405
1124 | 132_470
1125 | 283_564
1126 | 333_503
1127 | 256_259
1128 | 113_415
1129 | 235_445
1130 | 052_241
1131 | 157_468
1132 | 038_109
1133 | 257_443
1134 | 218_429
1135 | 054_202
1136 | 129_507
1137 | 103_104
1138 | 180_238
1139 | 119_556
1140 | 138_435
1141 | 391_326
1142 | 099_321
1143 | 276_083
1144 | 169_409
1145 | 056_115
1146 | 194_308
1147 | 173_519
1148 | 313_518
1149 | 147_493
1150 | 097_327
1151 | 237_590
1152 | 137_495
1153 | 237_442
1154 | 246_583
1155 | 107_407
1156 | 119_321
1157 | 173_516
1158 | 098_357
1159 | 317_530
1160 | 115_602
1161 | 182_521
1162 | 338_432
1163 | 138_474
1164 | 377_349
1165 | 216_461
1166 | 169_410
1167 | 166_382
1168 | 200_415
1169 | 108_401
1170 | 183_237
1171 | 183_425
1172 | 196_330
1173 | 205_435
1174 | 191_419
1175 | 151_478
1176 | 161_525
1177 | 227_278
1178 | 194_385
1179 | 327_511
1180 | 119_446
1181 | 220_440
1182 | 230_285
1183 | 219_279
1184 | 111_407
1185 | 236_598
1186 | 038_239
1187 | 144_498
1188 | 159_406
1189 | 148_483
1190 | 186_265
1191 | 124_337
1192 | 255_444
1193 | 270_476
1194 | 175_586
1195 | 195_367
1196 | 139_585
1197 | 315_523
1198 | 100_332
1199 | 171_611
1200 | 251_446
1201 | 166_471
1202 | 285_307
1203 | 152_471
1204 | 113_358
1205 | 247_290
1206 | 194_257
1207 | 134_308
1208 | 123_393
1209 | 314_519
1210 | 203_268
1211 | 122_560
1212 | 131_472
1213 | 150_485
1214 | 164_475
1215 | 120_559
1216 | 222_268
1217 | 251_259
1218 | 379_328
1219 | 141_494
1220 | 269_563
1221 | 126_357
1222 | 314_515
1223 | 271_267
1224 | 237_457
1225 | 153_470
1226 | 049_114
1227 | 121_375
1228 | 190_419
1229 | 282_252
1230 | 132_302
1231 | 332_509
1232 | 059_117
1233 | 126_496
1234 | 200_030
1235 | 185_305
1236 | 160_483
1237 | 106_440
1238 | 258_419
1239 | 276_264
1240 | 151_539
1241 | 143_444
1242 | 238_281
1243 | 056_058
1244 | 221_283
1245 | 161_377
1246 | 201_457
1247 | 275_405
1248 | 185_368
1249 | 065_348
1250 | 174_406
1251 | 168_388
1252 | 266_434
1253 | 317_544
1254 | 234_371
1255 | 161_467
1256 | 198_478
1257 | 080_176
1258 | 344_415
1259 | 228_314
1260 | 126_438
1261 | 058_221
1262 | 206_197
1263 | 126_427
1264 | 125_129
1265 | 127_497
1266 | 257_446
1267 | 099_355
1268 | 252_590
1269 | 150_487
1270 | 150_470
1271 | 120_557
1272 | 105_329
1273 | 168_433
1274 | 103_318
1275 | 333_515
1276 | 121_509
1277 | 131_477
1278 | 196_350
1279 | 253_595
1280 | 080_230
1281 | 175_238
1282 | 159_403
1283 | 190_286
1284 | 203_407
1285 | 316_519
1286 | 176_579
1287 | 117_326
1288 | 155_484
1289 | 116_208
1290 | 151_484
1291 | 223_288
1292 | 260_571
1293 | 155_402
1294 | 119_125
1295 | 330_517
1296 | 175_236
1297 | 112_479
1298 | 244_334
1299 | 208_417
1300 | 087_268
1301 | 055_200
1302 | 390_324
1303 | 157_480
1304 | 179_506
1305 | 274_467
1306 | 226_269
1307 | 199_323
1308 | 143_469
1309 | 209_381
1310 | 130_422
1311 | 308_524
1312 | 282_267
1313 | 142_481
1314 | 125_447
1315 | 066_353
1316 | 219_266
1317 | 097_302
1318 | 034_148
1319 | 232_277
1320 | 121_482
1321 | 105_332
1322 | 220_314
1323 | 163_593
1324 | 231_294
1325 | 128_485
1326 | 182_361
1327 | 202_324
1328 | 114_421
1329 | 223_442
1330 | 213_449
1331 | 152_489
1332 | 200_391
1333 | 058_090
1334 | 081_338
1335 | 220_448
1336 | 111_401
1337 | 154_487
1338 | 129_444
1339 | 321_518
1340 | 131_381
1341 | 144_444
1342 | 097_322
1343 | 128_507
1344 | 175_588
1345 | 155_469
1346 | 256_557
1347 | 233_268
1348 | 237_593
1349 | 119_558
1350 | 228_294
1351 | 221_333
1352 | 169_398
1353 | 197_291
1354 | 133_473
1355 | 113_317
1356 | 150_500
1357 | 267_383
1358 | 134_510
1359 | 132_438
1360 | 227_332
1361 | 249_446
1362 | 248_585
1363 | 201_398
1364 | 249_291
1365 | 115_604
1366 | 272_429
1367 | 283_432
1368 | 343_469
1369 | 210_278
1370 | 175_411
1371 | 213_335
1372 | 116_504
1373 | 318_344
1374 | 127_471
1375 | 074_058
1376 | 146_485
1377 | 091_324
1378 | 193_582
1379 | 219_309
1380 | 213_426
1381 | 184_261
1382 | 302_382
1383 | 219_328
1384 | 221_426
1385 | 185_497
1386 | 326_317
1387 | 159_421
1388 | 107_413
1389 | 222_023
1390 | 234_288
1391 | 183_613
1392 | 122_374
1393 | 320_522
1394 | 141_488
1395 | 105_317
1396 | 299_360
1397 | 128_480
1398 | 320_523
1399 | 238_444
1400 | 225_357
1401 | 125_503
1402 | 080_095
1403 | 134_471
1404 | 163_519
1405 | 271_266
1406 | 232_332
1407 | 054_107
1408 | 326_516
1409 | 210_309
1410 | 136_471
1411 | 183_607
1412 | 120_436
1413 | 235_440
1414 | 296_135
1415 | 073_095
1416 | 319_339
1417 | 208_424
1418 | 214_329
1419 | 155_467
1420 | 174_397
1421 | 228_431
1422 | 243_446
1423 | 199_275
1424 | 183_356
1425 | 239_598
1426 | 141_443
1427 | 280_559
1428 | 164_376
1429 | 128_506
1430 | 065_286
1431 | 064_120
1432 | 105_617
1433 | 189_396
1434 | 101_342
1435 | 074_291
1436 | 113_441
1437 | 329_521
1438 | 329_498
1439 | 265_439
1440 | 142_483
1441 | 257_268
1442 | 256_280
1443 | 230_280
1444 | 207_331
1445 | 054_115
1446 | 151_488
1447 | 136_437
1448 | 209_445
1449 | 253_582
1450 | 188_349
1451 | 100_333
1452 | 329_502
1453 | 308_526
1454 | 246_333
1455 | 230_291
1456 | 130_425
1457 | 229_605
1458 | 148_593
1459 | 222_369
1460 | 171_347
1461 | 124_479
1462 | 103_417
1463 | 318_539
1464 | 129_506
1465 | 086_204
1466 | 222_333
1467 | 140_411
1468 | 117_448
1469 | 211_335
1470 | 059_191
1471 | 343_341
1472 | 233_478
1473 | 176_587
1474 | 226_280
1475 | 038_168
1476 | 330_502
1477 | 147_393
1478 | 317_545
1479 | 181_578
1480 | 163_454
1481 | 211_334
1482 | 203_401
1483 | 214_308
1484 | 284_276
1485 | 228_285
1486 | 318_528
1487 | 204_418
1488 | 175_401
1489 | 097_099
1490 | 165_377
1491 | 134_497
1492 | 063_357
1493 | 275_469
1494 | 076_300
1495 | 185_242
1496 | 093_331
1497 | 081_174
1498 | 326_501
1499 | 158_617
1500 | 228_276
1501 | 071_347
1502 | 113_442
1503 | 220_429
1504 | 237_466
1505 | 180_430
1506 | 089_250
1507 | 130_159
1508 | 123_383
1509 | 204_401
1510 | 047_089
1511 | 132_133
1512 | 142_498
1513 | 319_534
1514 | 144_497
1515 | 282_128
1516 | 255_445
1517 | 251_284
1518 | 220_328
1519 | 185_502
1520 | 194_315
1521 | 174_412
1522 | 325_527
1523 | 223_271
1524 | 304_541
1525 | 115_338
1526 | 174_152
1527 | 123_434
1528 | 107_440
1529 | 181_353
1530 | 179_263
1531 | 309_524
1532 | 247_590
1533 | 173_605
1534 | 136_484
1535 | 133_309
1536 | 154_408
1537 | 198_312
1538 | 171_606
1539 | 152_490
1540 | 145_498
1541 | 336_499
1542 | 059_193
1543 | 025_082
1544 | 097_316
1545 | 219_431
1546 | 317_531
1547 | 061_112
1548 | 388_323
1549 | 320_535
1550 | 138_427
1551 | 156_598
1552 | 244_586
1553 | 093_340
1554 | 155_346
1555 | 105_452
1556 | 142_490
1557 | 031_195
1558 | 145_406
1559 | 241_474
1560 | 203_335
1561 | 095_338
1562 | 184_410
1563 | 084_265
1564 | 212_445
1565 | 121_503
1566 | 135_321
1567 | 171_346
1568 | 224_326
1569 | 416_318
1570 | 134_494
1571 | 329_508
1572 | 092_253
1573 | 131_315
1574 | 225_325
1575 | 333_508
1576 | 044_085
1577 | 241_335
1578 | 165_403
1579 | 214_449
1580 | 130_509
1581 | 282_266
1582 | 135_497
1583 | 232_330
1584 | 242_336
1585 | 333_495
1586 | 277_403
1587 | 419_322
1588 | 296_540
1589 | 231_285
1590 | 101_355
1591 | 219_312
1592 | 105_331
1593 | 275_398
1594 | 308_293
1595 | 138_495
1596 | 119_492
1597 | 224_333
1598 | 179_462
1599 | 265_310
1600 | 282_262
1601 | 022_126
1602 | 163_469
1603 | 152_480
1604 | 057_065
1605 | 297_368
1606 | 218_293
1607 | 119_403
1608 | 184_363
1609 | 195_366
1610 | 097_103
1611 | 220_332
1612 | 067_352
1613 | 131_473
1614 | 239_591
1615 | 145_474
1616 | 134_360
1617 | 043_099
1618 | 111_418
1619 | 237_595
1620 | 124_483
1621 | 243_336
1622 | 218_311
1623 | 119_322
1624 | 270_317
1625 | 078_257
1626 | 226_272
1627 | 259_577
1628 | 184_404
1629 | 148_495
1630 | 191_511
1631 | 142_155
1632 | 267_566
1633 | 139_491
1634 | 167_425
1635 | 119_504
1636 | 133_470
1637 | 096_306
1638 | 164_469
1639 | 112_446
1640 | 175_466
1641 | 159_592
1642 | 191_466
1643 | 332_507
1644 | 310_543
1645 | 188_403
1646 | 365_348
1647 | 237_289
1648 | 126_477
1649 | 173_351
1650 | 220_333
1651 | 334_471
1652 | 345_468
1653 | 301_353
1654 | 201_410
1655 | 134_495
1656 | 125_124
1657 | 262_433
1658 | 206_356
1659 | 221_460
1660 | 324_316
1661 | 103_307
1662 | 041_099
1663 | 168_610
1664 | 334_507
1665 | 033_148
1666 | 155_255
1667 | 099_357
1668 | 194_261
1669 | 064_352
1670 | 144_469
1671 | 187_400
1672 | 136_487
1673 | 309_526
1674 | 144_478
1675 | 168_612
1676 | 079_220
1677 | 210_310
1678 | 207_424
1679 | 157_520
1680 | 185_483
1681 | 231_332
1682 | 167_580
1683 | 270_298
1684 | 208_324
1685 | 244_279
1686 | 087_364
1687 | 226_291
1688 | 112_357
1689 | 114_487
1690 | 376_346
1691 | 279_265
1692 | 039_167
1693 | 225_315
1694 | 124_302
1695 | 308_517
1696 | 068_352
1697 | 112_423
1698 | 259_576
1699 | 249_272
1700 | 093_353
1701 | 145_469
1702 | 168_392
1703 | 273_466
1704 | 125_445
1705 | 250_446
1706 | 217_289
1707 | 118_442
1708 | 177_264
1709 | 166_263
1710 | 130_510
1711 | 104_416
1712 | 149_494
1713 | 206_427
1714 | 140_485
1715 | 170_514
1716 | 082_204
1717 | 144_494
1718 | 223_330
1719 | 172_154
1720 | 224_440
1721 | 160_377
1722 | 351_539
1723 | 267_274
1724 | 145_393
1725 | 052_240
1726 | 137_494
1727 | 048_070
1728 | 275_407
1729 | 295_535
1730 | 123_507
1731 | 249_427
1732 | 358_325
1733 | 082_104
1734 | 110_429
1735 | 215_329
1736 | 330_333
1737 | 143_426
1738 | 369_343
1739 | 087_269
1740 | 166_386
1741 | 144_439
1742 | 129_475
1743 | 058_191
1744 | 386_323
1745 | 332_506
1746 | 184_238
1747 | 229_298
1748 | 113_420
1749 | 323_507
1750 | 138_360
1751 | 154_346
1752 | 065_196
1753 | 165_582
1754 | 321_521
1755 | 130_445
1756 | 175_585
1757 | 207_389
1758 | 132_426
1759 | 231_281
1760 | 188_404
1761 | 158_334
1762 | 030_076
1763 | 168_393
1764 | 258_413
1765 | 092_214
1766 | 176_585
1767 | 340_331
1768 | 206_287
1769 | 101_358
1770 | 317_529
1771 | 185_348
1772 | 060_258
1773 | 270_475
1774 | 167_589
1775 | 253_587
1776 | 106_421
1777 | 145_501
1778 | 343_535
1779 | 275_465
1780 | 032_197
1781 | 175_428
1782 | 183_501
1783 | 065_118
1784 | 264_434
1785 | 329_488
1786 | 157_481
1787 | 242_466
1788 | 115_493
1789 | 180_022
1790 | 162_600
1791 | 122_492
1792 | 039_111
1793 | 236_369
1794 | 230_286
1795 | 215_241
1796 | 274_468
1797 | 221_306
1798 | 143_476
1799 | 239_337
1800 | 160_401
1801 | 155_479
1802 | 192_347
1803 | 164_468
1804 | 175_605
1805 | 209_312
1806 | 132_443
1807 | 211_435
1808 | 128_326
1809 | 235_332
1810 | 135_316
1811 | 116_409
1812 | 167_598
1813 | 207_478
1814 | 193_330
1815 | 200_478
1816 | 190_400
1817 | 374_346
1818 | 146_479
1819 | 256_277
1820 | 132_488
1821 | 204_402
1822 | 167_433
1823 | 099_334
1824 | 239_285
1825 | 223_311
1826 | 076_616
1827 | 221_311
1828 | 282_264
1829 | 146_496
1830 | 118_322
1831 | 085_360
1832 | 057_192
1833 | 318_529
1834 | 134_493
1835 | 114_422
1836 | 227_431
1837 | 146_493
1838 | 138_426
1839 | 126_307
1840 | 319_510
1841 | 243_333
1842 | 221_321
1843 | 292_552
1844 | 150_493
1845 | 096_322
1846 | 115_507
1847 | 317_343
1848 | 251_584
1849 | 275_470
1850 | 239_588
1851 | 124_304
1852 | 180_610
1853 | 220_336
1854 | 094_329
1855 | 422_327
1856 | 272_264
1857 | 224_373
1858 | 224_439
1859 | 149_266
1860 | 346_328
1861 | 222_453
1862 | 332_503
1863 | 346_428
1864 | 154_480
1865 | 323_527
1866 | 226_274
1867 | 233_590
1868 | 147_392
1869 | 251_590
1870 | 103_347
1871 | 168_611
1872 | 236_449
1873 | 144_440
1874 | 132_500
1875 | 044_219
1876 | 145_488
1877 | 281_265
1878 | 327_500
1879 | 244_582
1880 | 133_481
1881 | 281_082
1882 | 226_333
1883 | 050_174
1884 | 282_562
1885 | 181_400
1886 | 165_265
1887 | 142_499
1888 | 162_469
1889 | 171_587
1890 | 080_231
1891 | 093_212
1892 | 133_318
1893 | 167_263
1894 | 210_272
1895 | 334_503
1896 | 164_266
1897 | 210_318
1898 | 127_446
1899 | 102_201
1900 | 061_094
1901 | 219_603
1902 | 075_350
1903 | 275_474
1904 | 131_444
1905 | 127_483
1906 | 388_324
1907 | 192_474
1908 | 254_270
1909 | 062_350
1910 | 326_388
1911 | 228_306
1912 | 140_443
1913 | 125_308
1914 | 240_336
1915 | 116_486
1916 | 183_520
1917 | 038_203
1918 | 274_464
1919 | 221_309
1920 | 257_559
1921 | 061_061
1922 | 059_091
1923 | 378_349
1924 | 112_039
1925 | 214_450
1926 | 166_587
1927 | 232_293
1928 | 244_594
1929 | 203_435
1930 | 122_501
1931 | 282_265
1932 | 227_304
1933 | 215_336
1934 | 091_336
1935 | 123_492
1936 | 132_441
1937 | 180_398
1938 | 222_432
1939 | 101_357
1940 | 123_483
1941 | 149_415
1942 | 200_319
1943 | 200_364
1944 | 287_391
1945 | 108_421
1946 | 228_469
1947 | 129_325
1948 | 174_408
1949 | 153_415
1950 | 373_294
1951 | 038_204
1952 | 175_432
1953 | 351_393
1954 | 252_281
1955 | 162_251
1956 | 049_086
1957 | 221_443
1958 | 080_229
1959 | 223_481
1960 | 214_305
1961 | 107_105
1962 | 105_319
1963 | 108_405
1964 | 314_342
1965 | 150_440
1966 | 128_311
1967 | 228_298
1968 | 219_250
1969 | 079_345
1970 | 162_377
1971 | 071_348
1972 | 244_592
1973 | 318_527
1974 | 219_314
1975 | 214_307
1976 | 082_339
1977 | 137_309
1978 | 120_447
1979 | 255_274
1980 | 107_106
1981 | 177_265
1982 | 124_441
1983 | 257_483
1984 | 221_440
1985 | 169_611
1986 | 124_497
1987 | 125_443
1988 | 146_497
1989 | 165_441
1990 | 212_334
1991 | 207_410
1992 | 081_176
1993 | 289_306
1994 | 120_558
1995 | 147_494
1996 | 170_579
1997 | 076_056
1998 | 215_438
1999 | 328_502
2000 | 224_438
2001 | 281_267
2002 | 226_591
2003 | 210_389
2004 | 181_263
2005 | 175_363
2006 | 134_426
2007 | 201_409
2008 | 274_408
2009 | 219_329
2010 | 206_481
2011 | 193_296
2012 | 036_117
2013 | 322_543
2014 | 126_502
2015 | 326_522
2016 | 213_313
2017 | 194_340
2018 | 220_311
2019 | 194_399
2020 | 126_301
2021 | 242_468
2022 | 241_475
2023 | 212_466
2024 | 241_599
2025 | 227_430
2026 | 269_478
2027 | 251_447
2028 | 330_499
2029 | 230_458
2030 | 153_487
2031 | 179_584
2032 | 087_197
2033 | 180_240
2034 | 215_293
2035 | 237_335
2036 | 106_107
2037 | 188_398
2038 | 124_508
2039 | 206_404
2040 | 176_510
2041 | 216_433
2042 | 194_314
2043 | 332_498
2044 | 163_592
2045 | 319_318
2046 | 174_587
2047 | 236_600
2048 | 308_540
2049 | 239_277
2050 | 163_537
2051 | 141_485
2052 | 128_126
2053 | 131_404
2054 | 229_292
2055 | 247_427
2056 | 142_492
2057 | 236_434
2058 | 207_334
2059 | 324_334
2060 | 220_268
2061 | 049_045
2062 | 155_254
2063 | 079_094
2064 | 065_073
2065 | 119_555
2066 | 247_336
2067 | 221_292
2068 | 217_428
2069 | 249_591
2070 | 100_358
2071 | 253_265
2072 | 196_345
2073 | 298_352
2074 | 292_547
2075 | 225_481
2076 | 054_098
2077 | 145_156
2078 | 134_316
2079 | 107_248
2080 | 173_370
2081 | 218_306
2082 | 169_610
2083 | 313_517
2084 | 226_309
2085 | 114_596
2086 | 234_442
2087 | 186_364
2088 | 151_240
2089 | 231_287
2090 | 219_439
2091 | 085_592
2092 | 264_467
2093 | 179_580
2094 | 219_447
2095 | 267_438
2096 | 087_200
2097 | 150_418
2098 | 219_440
2099 | 370_547
2100 | 165_522
2101 | 174_493
2102 | 137_478
2103 | 128_446
2104 | 115_448
2105 | 257_264
2106 | 242_429
2107 | 230_484
2108 | 252_275
2109 | 112_609
2110 | 203_266
2111 | 078_238
2112 | 332_499
2113 | 288_129
2114 | 251_585
2115 | 207_457
2116 | 233_331
2117 | 202_388
2118 | 368_360
2119 | 165_376
2120 | 258_483
2121 | 180_460
2122 | 268_269
2123 | 190_353
2124 | 232_024
2125 | 131_474
2126 | 194_493
2127 | 243_467
2128 | 053_114
2129 | 208_362
2130 | 193_318
2131 | 160_408
2132 | 257_273
2133 | 130_438
2134 | 183_354
2135 | 146_494
2136 | 170_346
2137 | 221_316
2138 | 157_522
2139 | 106_429
2140 | 157_483
2141 | 221_602
2142 | 143_489
2143 | 324_510
2144 | 196_458
2145 | 142_427
2146 | 221_436
2147 | 294_135
2148 | 251_587
2149 | 262_484
2150 | 229_282
2151 | 060_064
2152 | 331_501
2153 | 269_297
2154 | 194_366
2155 | 176_607
2156 | 317_368
2157 | 230_591
2158 | 222_314
2159 | 140_483
2160 | 225_482
2161 | 087_202
2162 | 322_544
2163 | 370_349
2164 | 142_501
2165 | 280_251
2166 | 236_592
2167 | 273_472
2168 | 191_406
2169 | 234_376
2170 | 060_108
2171 | 240_429
2172 | 166_459
2173 | 209_334
2174 | 311_380
2175 | 167_611
2176 | 195_407
2177 | 279_255
2178 | 193_459
2179 | 042_092
2180 | 228_281
2181 | 169_601
2182 | 131_476
2183 | 135_430
2184 | 234_265
2185 | 329_512
2186 | 248_595
2187 | 150_494
2188 | 169_387
2189 | 077_295
2190 | 147_501
2191 | 199_442
2192 | 178_023
2193 | 331_523
2194 | 129_311
2195 | 319_528
2196 | 327_507
2197 | 090_254
2198 | 110_335
2199 | 197_326
2200 | 172_413
2201 | 185_241
2202 | 210_429
2203 | 192_292
2204 | 119_505
2205 | 311_525
2206 | 059_116
2207 | 182_399
2208 | 137_492
2209 | 133_478
2210 | 415_315
2211 | 112_419
2212 | 180_579
2213 | 350_345
2214 | 129_483
2215 | 232_287
2216 | 217_423
2217 | 167_437
2218 | 053_116
2219 | 181_518
2220 | 211_283
2221 | 182_412
2222 | 253_264
2223 | 267_420
2224 | 344_459
2225 | 199_365
2226 | 065_354
2227 | 143_356
2228 | 119_559
2229 | 321_510
2230 | 033_097
2231 | 335_500
2232 | 041_115
2233 | 181_413
2234 | 137_156
2235 | 232_445
2236 | 239_284
2237 | 251_425
2238 | 163_528
2239 | 221_332
2240 | 227_280
2241 | 121_465
2242 | 210_461
2243 | 139_435
2244 | 198_365
2245 | 201_382
2246 | 291_134
2247 | 215_292
2248 | 192_583
2249 | 324_410
2250 | 165_258
2251 | 220_462
2252 | 323_417
2253 | 351_540
2254 | 236_448
2255 | 218_437
2256 | 294_348
2257 | 140_484
2258 | 243_589
2259 | 217_427
2260 | 037_148
2261 | 258_268
2262 | 226_330
2263 | 136_306
2264 | 331_499
2265 | 274_465
2266 | 091_320
2267 | 163_458
2268 | 222_337
2269 | 204_329
2270 | 084_207
2271 | 169_599
2272 | 371_349
2273 | 421_324
2274 | 206_337
2275 | 131_442
2276 | 216_429
2277 | 366_348
2278 | 201_361
2279 | 169_258
2280 | 217_283
2281 | 213_435
2282 | 124_446
2283 | 105_334
2284 | 226_289
2285 | 131_429
2286 | 148_501
2287 | 120_322
2288 | 155_408
2289 | 245_596
2290 | 306_543
2291 | 232_596
2292 | 132_484
2293 | 280_560
2294 | 145_338
2295 | 222_292
2296 | 101_306
2297 | 222_328
2298 | 208_299
2299 | 235_335
2300 | 197_319
2301 | 208_428
2302 | 267_582
2303 | 111_319
2304 | 216_442
2305 | 065_288
2306 | 170_267
2307 | 252_585
2308 | 162_483
2309 | 237_598
2310 | 116_427
2311 | 325_316
2312 | 209_430
2313 | 211_282
2314 | 051_172
2315 | 115_486
2316 | 121_506
2317 | 134_472
2318 | 205_329
2319 | 125_420
2320 | 145_443
2321 | 182_579
2322 | 292_553
2323 | 162_595
2324 | 244_451
2325 | 246_592
2326 | 059_190
2327 | 123_446
2328 | 193_407
2329 | 181_507
2330 | 200_323
2331 | 249_440
2332 | 234_280
2333 | 062_120
2334 | 099_331
2335 | 099_495
2336 | 274_472
2337 | 148_496
2338 | 230_273
2339 | 288_119
2340 | 093_358
2341 | 229_379
2342 | 120_509
2343 | 048_090
2344 | 247_596
2345 | 251_485
2346 | 147_434
2347 | 204_344
2348 | 189_395
2349 | 233_597
2350 | 160_591
2351 | 203_414
2352 | 144_474
2353 | 238_593
2354 | 196_323
2355 | 319_533
2356 | 190_285
2357 | 220_335
2358 | 214_304
2359 | 114_506
2360 | 242_591
2361 | 325_314
2362 | 235_456
2363 | 136_470
2364 | 154_422
2365 | 114_592
2366 | 157_523
2367 | 192_476
2368 | 215_314
2369 | 181_250
2370 | 257_445
2371 | 178_497
2372 | 205_429
2373 | 063_116
2374 | 106_408
2375 | 105_418
2376 | 249_594
2377 | 207_323
2378 | 236_596
2379 | 243_595
2380 | 127_470
2381 | 141_486
2382 | 170_586
2383 | 151_248
2384 | 357_363
2385 | 253_448
2386 | 121_505
2387 | 204_506
2388 | 166_260
2389 | 151_490
2390 | 272_450
2391 | 166_261
2392 | 131_507
2393 | 195_486
2394 | 222_480
2395 | 322_417
2396 | 180_606
2397 | 235_444
2398 | 353_327
2399 | 088_265
2400 | 063_352
2401 | 229_288
2402 | 227_330
2403 | 226_457
2404 | 233_289
2405 | 192_280
2406 | 125_504
2407 | 267_434
2408 | 109_334
2409 | 141_402
2410 | 124_129
2411 | 207_298
2412 | 219_468
2413 | 139_488
2414 | 066_090
2415 | 045_243
2416 | 126_497
2417 | 331_509
2418 | 225_484
2419 | 345_471
2420 | 139_480
2421 | 133_472
2422 | 212_411
2423 | 091_334
2424 | 127_445
2425 | 161_533
2426 | 094_357
2427 | 207_434
2428 | 127_503
2429 | 274_469
2430 | 222_332
2431 | 238_458
2432 | 150_469
2433 | 216_454
2434 | 273_474
2435 | 052_239
2436 | 330_394
2437 | 225_274
2438 | 318_517
2439 | 057_194
2440 | 197_398
2441 | 167_587
2442 | 188_428
2443 | 340_326
2444 | 276_473
2445 | 250_585
2446 | 259_444
2447 | 094_354
2448 | 177_243
2449 | 197_366
2450 | 179_603
2451 | 132_134
2452 | 127_479
2453 | 120_503
2454 | 071_257
2455 | 095_323
2456 | 253_286
2457 | 239_335
2458 | 101_102
2459 | 151_604
2460 | 333_500
2461 | 152_602
2462 | 165_530
2463 | 140_470
2464 | 062_257
2465 | 182_239
2466 | 124_492
2467 | 242_470
2468 | 208_272
2469 | 130_442
2470 | 032_111
2471 | 191_516
2472 | 191_471
2473 | 240_593
2474 | 094_336
2475 | 154_468
2476 | 299_548
2477 | 321_527
2478 | 221_354
2479 | 273_464
2480 | 267_432
2481 | 181_237
2482 | 242_441
2483 | 242_467
2484 | 184_407
2485 | 236_289
2486 | 167_406
2487 | 251_260
2488 | 109_416
2489 | 218_451
2490 | 158_393
2491 | 169_407
2492 | 183_365
2493 | 110_336
2494 | 093_355
2495 | 109_471
2496 | 235_435
2497 | 049_103
2498 | 173_503
2499 | 324_312
2500 | 093_213
2501 | 194_502
2502 | 218_431
2503 | 110_343
2504 | 118_467
2505 | 238_428
2506 | 115_504
2507 | 284_263
2508 | 257_276
2509 | 100_307
2510 | 273_459
2511 | 145_372
2512 | 137_275
2513 | 236_188
2514 | 242_469
2515 | 210_334
2516 | 130_506
2517 | 141_484
2518 | 094_355
2519 | 309_517
2520 | 227_605
2521 | 165_412
2522 | 160_400
2523 | 299_453
2524 | 181_404
2525 | 077_616
2526 | 122_445
2527 | 194_296
2528 | 191_459
2529 | 255_446
2530 | 054_264
2531 | 179_579
2532 | 165_593
2533 | 130_487
2534 | 209_387
2535 | 117_493
2536 | 212_289
2537 | 042_100
2538 | 331_510
2539 | 101_318
2540 | 134_496
2541 | 060_118
2542 | 236_284
2543 | 133_303
2544 | 044_169
2545 | 194_406
2546 | 123_391
2547 | 037_101
2548 | 264_583
2549 | 168_408
2550 | 134_508
2551 | 093_339
2552 | 139_428
2553 | 360_323
2554 | 266_435
2555 | 108_330
2556 | 143_479
2557 | 343_339
2558 | 079_235
2559 | 165_517
2560 | 169_384
2561 | 186_488
2562 | 162_533
2563 | 227_276
2564 | 247_597
2565 | 326_521
2566 | 220_428
2567 | 090_337
2568 | 141_434
2569 | 127_499
2570 | 106_336
2571 | 112_404
2572 | 318_532
2573 | 244_335
2574 | 241_331
2575 | 219_430
2576 | 142_497
2577 | 254_593
2578 | 066_122
2579 | 241_441
2580 | 135_324
2581 | 048_171
2582 | 252_285
2583 | 149_491
2584 | 133_482
2585 | 280_308
2586 | 075_171
2587 | 165_599
2588 | 322_534
2589 | 161_468
2590 | 183_604
2591 | 063_356
2592 | 252_265
2593 | 191_412
2594 | 194_256
2595 | 348_330
2596 | 167_343
2597 | 325_315
2598 | 136_276
2599 | 228_324
2600 | 250_589
2601 | 176_580
2602 | 217_426
2603 | 168_589
2604 | 036_118
2605 | 271_265
2606 | 139_430
2607 | 233_355
2608 | 102_305
2609 | 272_460
2610 | 104_329
2611 | 299_349
2612 | 370_360
2613 | 202_016
2614 | 170_264
2615 | 210_482
2616 | 179_405
2617 | 185_302
2618 | 226_328
2619 | 223_437
2620 | 074_267
2621 | 171_518
2622 | 098_103
2623 | 192_366
2624 | 188_420
2625 | 368_359
2626 | 102_344
2627 | 145_332
2628 | 247_428
2629 | 244_591
2630 | 213_290
2631 | 105_103
2632 | 235_289
2633 | 312_433
2634 | 274_474
2635 | 169_586
2636 | 243_586
2637 | 249_589
2638 | 177_239
2639 | 180_284
2640 | 113_449
2641 | 154_472
2642 | 219_333
2643 | 341_533
2644 | 243_452
2645 | 246_428
2646 | 197_591
2647 | 240_277
2648 | 229_330
2649 | 243_334
2650 | 229_276
2651 | 211_336
2652 | 144_485
2653 | 176_426
2654 | 161_459
2655 | 279_564
2656 | 298_431
2657 | 196_288
2658 | 173_488
2659 | 109_335
2660 | 054_199
2661 | 167_482
2662 | 107_104
2663 | 166_482
2664 | 189_494
2665 | 251_418
2666 | 193_364
2667 | 195_484
2668 | 243_593
2669 | 172_579
2670 | 166_412
2671 | 248_270
2672 | 049_070
2673 | 231_331
2674 | 211_284
2675 | 189_515
2676 | 267_453
2677 | 087_216
2678 | 236_450
2679 | 149_479
2680 | 128_501
2681 | 118_503
2682 | 215_452
2683 | 104_334
2684 | 133_340
2685 | 093_356
2686 | 180_584
2687 | 322_524
2688 | 151_441
2689 | 151_592
2690 | 268_446
2691 | 194_337
2692 | 256_356
2693 | 353_423
2694 | 163_371
2695 | 133_492
2696 | 049_231
2697 | 049_071
2698 | 116_447
2699 | 156_466
2700 | 345_470
2701 | 129_435
2702 | 156_475
2703 | 331_508
2704 | 270_477
2705 | 231_280
2706 | 207_392
2707 | 215_306
2708 | 124_433
2709 | 162_406
2710 | 245_332
2711 | 223_294
2712 | 116_346
2713 | 183_606
2714 | 156_480
2715 | 327_516
2716 | 126_480
2717 | 178_505
2718 | 297_352
2719 | 254_260
2720 | 234_591
2721 | 126_304
2722 | 125_481
2723 | 166_523
2724 | 066_289
2725 | 220_444
2726 | 283_262
2727 | 058_076
2728 | 101_343
2729 | 270_264
2730 | 217_425
2731 | 220_487
2732 | 166_435
2733 | 126_445
2734 | 102_330
2735 | 295_540
2736 | 252_263
2737 | 315_541
2738 | 253_257
2739 | 173_413
2740 | 053_176
2741 | 266_362
2742 | 108_204
2743 | 137_498
2744 | 179_605
2745 | 154_489
2746 | 061_191
2747 | 288_311
2748 | 128_481
2749 | 228_023
2750 | 216_426
2751 | 262_434
2752 | 217_336
2753 | 181_427
2754 | 238_331
2755 | 222_185
2756 | 174_017
2757 | 220_305
2758 | 097_323
2759 | 114_441
2760 | 118_492
2761 | 130_439
2762 | 121_507
2763 | 232_023
2764 | 210_444
2765 | 153_466
2766 | 205_389
2767 | 273_460
2768 | 154_399
2769 | 115_487
2770 | 319_530
2771 | 242_450
2772 | 351_328
2773 | 182_401
2774 | 122_482
2775 | 318_525
2776 | 041_089
2777 | 224_375
2778 | 221_335
2779 | 227_293
2780 | 247_335
2781 | 152_491
2782 | 139_409
2783 | 323_387
2784 | 133_498
2785 | 218_436
2786 | 209_333
2787 | 199_367
2788 | 270_478
2789 | 054_200
2790 | 063_349
2791 | 220_436
2792 | 113_493
2793 | 344_401
2794 | 246_444
2795 | 171_579
2796 | 164_478
2797 | 334_505
2798 | 171_589
2799 | 142_443
2800 | 198_356
2801 | 221_313
2802 | 097_328
2803 | 148_493
2804 | 167_456
2805 | 273_470
2806 | 244_446
2807 | 174_347
2808 | 198_594
2809 | 195_350
2810 | 254_265
2811 | 238_336
2812 | 111_449
2813 | 421_327
2814 | 420_322
2815 | 370_348
2816 | 210_393
2817 | 311_470
2818 | 163_522
2819 | 306_541
2820 | 416_319
2821 | 182_237
2822 | 093_322
2823 | 113_434
2824 | 370_342
2825 | 073_105
2826 | 340_533
2827 | 265_567
2828 | 201_319
2829 | 239_336
2830 | 179_518
2831 | 234_444
2832 | 094_617
2833 | 326_499
2834 | 184_458
2835 | 150_267
2836 | 139_434
2837 | 314_539
2838 | 163_455
2839 | 237_290
2840 | 252_589
2841 | 120_383
2842 | 219_256
2843 | 220_257
2844 | 153_480
2845 | 298_454
2846 | 154_473
2847 | 313_521
2848 | 207_477
2849 | 095_358
2850 | 248_590
2851 | 223_491
2852 | 161_480
2853 | 250_270
2854 | 283_558
2855 | 150_266
2856 | 267_306
2857 | 145_440
2858 | 206_297
2859 | 341_534
2860 | 117_503
2861 | 242_451
2862 | 147_488
2863 | 234_354
2864 | 214_401
2865 | 056_197
2866 | 226_370
2867 | 116_438
2868 | 177_581
2869 | 223_332
2870 | 210_329
2871 | 230_309
2872 | 185_503
2873 | 281_559
2874 | 070_352
2875 | 128_442
2876 | 320_518
2877 | 122_123
2878 | 253_416
2879 | 254_446
2880 | 217_435
2881 | 254_272
2882 | 174_360
2883 | 123_448
2884 | 199_595
2885 | 287_307
2886 | 249_271
2887 | 100_318
2888 | 283_559
2889 | 366_355
2890 | 182_022
2891 | 225_429
2892 | 316_543
2893 | 199_282
2894 | 123_560
2895 | 181_606
2896 | 191_286
2897 | 222_467
2898 | 114_447
2899 | 106_405
2900 | 256_266
2901 | 166_580
2902 | 176_581
2903 | 204_315
2904 | 182_499
2905 | 123_485
2906 | 222_281
2907 | 329_496
2908 | 217_314
2909 | 324_314
2910 | 208_416
2911 | 252_438
2912 | 100_101
2913 | 120_556
2914 | 198_318
2915 | 144_472
2916 | 106_288
2917 | 183_484
2918 | 141_491
2919 | 225_327
2920 | 151_375
2921 | 125_433
2922 | 257_265
2923 | 250_588
2924 | 159_618
2925 | 329_493
2926 | 327_527
2927 | 293_135
2928 | 258_587
2929 | 133_443
2930 | 219_270
2931 | 112_587
2932 | 336_500
2933 | 321_314
2934 | 308_484
2935 | 220_329
2936 | 177_603
2937 | 277_252
2938 | 057_118
2939 | 421_326
2940 | 314_516
2941 | 218_318
2942 | 098_327
2943 | 060_083
2944 | 334_496
2945 | 216_312
2946 | 156_390
2947 | 167_446
2948 | 253_274
2949 | 218_335
2950 | 044_226
2951 | 213_310
2952 | 207_420
2953 | 221_432
2954 | 216_436
2955 | 052_238
2956 | 130_505
2957 | 203_017
2958 | 248_445
2959 | 165_419
2960 | 249_284
2961 | 200_281
2962 | 072_257
2963 | 068_351
2964 | 324_315
2965 | 216_287
2966 | 179_424
2967 | 055_202
2968 | 170_410
2969 | 226_484
2970 | 421_329
2971 | 215_327
2972 | 227_269
2973 | 064_350
2974 | 209_285
2975 | 063_256
2976 | 302_430
2977 | 096_358
2978 | 131_320
2979 | 350_560
2980 | 164_392
2981 | 117_114
2982 | 264_466
2983 | 333_507
2984 | 152_468
2985 | 153_488
2986 | 089_360
2987 | 113_606
2988 | 148_478
2989 | 079_333
2990 | 184_392
2991 | 203_397
2992 | 224_331
2993 | 179_308
2994 | 069_294
2995 | 054_083
2996 | 217_313
2997 | 162_370
2998 | 363_321
2999 | 114_505
3000 | 231_295
3001 | 194_331
3002 | 127_317
3003 | 209_425
3004 | 196_291
3005 | 142_489
3006 | 243_445
3007 | 219_451
3008 | 182_502
3009 | 174_520
3010 | 184_237
3011 | 119_351
3012 | 353_328
3013 | 267_371
3014 | 238_330
3015 | 213_438
3016 | 205_285
3017 | 170_585
3018 | 081_345
3019 | 169_026
3020 | 129_442
3021 | 156_008
3022 | 272_468
3023 | 253_263
3024 | 120_448
3025 | 179_466
3026 | 216_284
3027 | 214_283
3028 | 372_349
3029 | 129_445
3030 | 154_402
3031 | 324_528
3032 | 061_195
3033 | 219_315
3034 | 208_390
3035 | 320_510
3036 | 030_093
3037 | 164_393
3038 | 114_402
3039 | 137_344
3040 | 051_173
3041 | 113_479
3042 | 310_469
3043 | 154_421
3044 | 126_479
3045 | 256_424
3046 | 318_531
3047 | 122_559
3048 | 167_599
3049 | 189_260
3050 | 213_456
3051 | 284_311
3052 | 328_509
3053 | 034_191
3054 | 163_250
3055 | 135_401
3056 | 183_497
3057 | 221_331
3058 | 048_091
3059 | 107_335
3060 | 149_487
3061 | 225_333
3062 | 205_342
3063 | 190_368
3064 | 200_324
3065 | 165_516
3066 | 162_524
3067 | 055_091
3068 | 213_336
3069 | 135_489
3070 | 229_291
3071 | 177_608
3072 | 236_278
3073 | 312_476
3074 | 260_413
3075 | 166_264
3076 | 114_429
3077 | 306_525
3078 | 099_310
3079 | 228_605
3080 | 167_383
3081 | 054_265
3082 | 083_174
3083 | 325_524
3084 | 263_288
--------------------------------------------------------------------------------
/training_dst.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ClarkCGA/multi-temporal-crop-classification-training-data/82cca1accdf761e4eb44bad7ba8f1e5160d9bb93/training_dst.png
--------------------------------------------------------------------------------
/val_ids.csv:
--------------------------------------------------------------------------------
1 | ids
2 | 142_475
3 | 091_197
4 | 278_432
5 | 229_280
6 | 250_448
7 | 039_109
8 | 121_436
9 | 316_522
10 | 121_491
11 | 104_318
12 | 184_503
13 | 117_426
14 | 230_315
15 | 143_480
16 | 304_299
17 | 100_330
18 | 089_201
19 | 116_449
20 | 092_338
21 | 102_327
22 | 119_439
23 | 251_589
24 | 183_614
25 | 172_611
26 | 144_489
27 | 265_566
28 | 140_487
29 | 096_323
30 | 152_477
31 | 150_497
32 | 121_492
33 | 202_419
34 | 102_345
35 | 143_411
36 | 118_558
37 | 207_324
38 | 166_434
39 | 125_446
40 | 130_471
41 | 250_594
42 | 289_444
43 | 218_379
44 | 173_578
45 | 154_486
46 | 154_476
47 | 387_325
48 | 288_426
49 | 142_434
50 | 281_264
51 | 190_428
52 | 146_481
53 | 219_378
54 | 033_196
55 | 149_477
56 | 235_584
57 | 218_327
58 | 210_307
59 | 274_466
60 | 252_448
61 | 117_323
62 | 324_517
63 | 185_239
64 | 126_305
65 | 118_484
66 | 256_580
67 | 151_487
68 | 139_486
69 | 127_304
70 | 113_422
71 | 125_482
72 | 255_276
73 | 114_448
74 | 128_443
75 | 239_476
76 | 174_400
77 | 232_294
78 | 193_419
79 | 232_484
80 | 373_344
81 | 088_360
82 | 118_493
83 | 142_470
84 | 234_261
85 | 114_438
86 | 320_314
87 | 201_402
88 | 132_490
89 | 159_617
90 | 137_283
91 | 126_396
92 | 263_449
93 | 123_343
94 | 113_423
95 | 063_037
96 | 143_485
97 | 208_361
98 | 219_288
99 | 053_113
100 | 145_497
101 | 120_555
102 | 089_589
103 | 239_597
104 | 238_591
105 | 171_521
106 | 238_590
107 | 113_038
108 | 244_593
109 | 150_492
110 | 204_417
111 | 338_531
112 | 164_444
113 | 261_412
114 | 164_262
115 | 292_540
116 | 089_217
117 | 184_400
118 | 115_446
119 | 200_274
120 | 193_315
121 | 237_597
122 | 159_407
123 | 180_407
124 | 249_449
125 | 253_272
126 | 342_533
127 | 104_104
128 | 134_437
129 | 154_253
130 | 164_522
131 | 187_406
132 | 226_335
133 | 100_306
134 | 120_505
135 | 182_398
136 | 153_476
137 | 156_402
138 | 106_407
139 | 232_329
140 | 095_218
141 | 131_433
142 | 144_434
143 | 284_564
144 | 098_338
145 | 219_313
146 | 155_468
147 | 052_043
148 | 258_444
149 | 175_515
150 | 205_286
151 | 231_595
152 | 181_238
153 | 125_441
154 | 100_355
155 | 182_497
156 | 273_471
157 | 257_271
158 | 320_519
159 | 233_293
160 | 081_337
161 | 239_430
162 | 256_276
163 | 091_364
164 | 228_489
165 | 234_333
166 | 221_439
167 | 254_444
168 | 138_477
169 | 165_404
170 | 118_447
171 | 002_060
172 | 165_469
173 | 171_585
174 | 154_601
175 | 191_473
176 | 262_414
177 | 190_375
178 | 189_429
179 | 244_267
180 | 139_477
181 | 210_331
182 | 146_343
183 | 293_546
184 | 215_305
185 | 200_016
186 | 207_451
187 | 420_326
188 | 141_473
189 | 239_443
190 | 075_105
191 | 385_323
192 | 302_542
193 | 229_272
194 | 169_433
195 | 129_480
196 | 316_524
197 | 158_522
198 | 269_299
199 | 187_259
200 | 094_321
201 | 210_277
202 | 120_502
203 | 034_147
204 | 239_269
205 | 156_476
206 | 330_414
207 | 117_501
208 | 282_404
209 | 146_470
210 | 009_061
211 | 202_336
212 | 125_502
213 | 216_460
214 | 084_175
215 | 118_555
216 | 134_441
217 | 391_323
218 | 258_270
219 | 217_330
220 | 324_525
221 | 347_343
222 | 169_399
223 | 186_348
224 | 146_469
225 | 208_391
226 | 149_263
227 | 181_579
228 | 215_454
229 | 156_500
230 | 228_297
231 | 113_421
232 | 148_491
233 | 220_308
234 | 045_106
235 | 165_464
236 | 182_406
237 | 131_496
238 | 153_467
239 | 072_280
240 | 115_505
241 | 186_422
242 | 054_201
243 | 248_276
244 | 151_477
245 | 124_303
246 | 164_477
247 | 112_330
248 | 180_339
249 | 057_087
250 | 275_408
251 | 250_271
252 | 332_332
253 | 296_352
254 | 315_514
255 | 126_439
256 | 232_269
257 | 202_392
258 | 250_272
259 | 392_325
260 | 024_087
261 | 221_442
262 | 126_446
263 | 245_580
264 | 228_330
265 | 330_498
266 | 244_332
267 | 313_475
268 | 098_358
269 | 369_349
270 | 216_467
271 | 060_116
272 | 164_528
273 | 216_280
274 | 213_466
275 | 132_471
276 | 227_461
277 | 291_553
278 | 081_218
279 | 319_526
280 | 365_345
281 | 253_443
282 | 097_358
283 | 165_601
284 | 161_423
285 | 220_431
286 | 159_594
287 | 270_267
288 | 178_022
289 | 191_364
290 | 106_415
291 | 223_438
292 | 250_587
293 | 174_348
294 | 128_445
295 | 193_312
296 | 298_453
297 | 169_514
298 | 110_041
299 | 181_503
300 | 327_499
301 | 179_238
302 | 281_560
303 | 306_527
304 | 139_476
305 | 130_430
306 | 061_115
307 | 169_032
308 | 170_593
309 | 215_429
310 | 196_369
311 | 256_269
312 | 334_525
313 | 359_558
314 | 162_479
315 | 102_332
316 | 239_459
317 | 130_411
318 | 236_445
319 | 327_510
320 | 163_518
321 | 181_023
322 | 101_332
323 | 149_493
324 | 123_447
325 | 127_427
326 | 120_508
327 | 238_285
328 | 271_264
329 | 132_474
330 | 180_609
331 | 112_433
332 | 149_478
333 | 244_595
334 | 092_323
335 | 219_323
336 | 133_474
337 | 141_482
338 | 220_334
339 | 270_263
340 | 259_331
341 | 335_506
342 | 163_601
343 | 230_279
344 | 111_611
345 | 115_502
346 | 113_043
347 | 242_332
348 | 146_473
349 | 164_481
350 | 284_307
351 | 279_252
352 | 149_486
353 | 104_498
354 | 124_124
355 | 081_333
356 | 129_505
357 | 120_443
358 | 233_283
359 | 238_469
360 | 245_595
361 | 314_518
362 | 140_482
363 | 166_581
364 | 283_264
365 | 251_583
366 | 300_412
367 | 220_309
368 | 112_422
369 | 203_308
370 | 061_092
371 | 138_475
372 | 234_443
373 | 183_514
374 | 256_267
375 | 126_481
376 | 130_473
377 | 316_383
378 | 150_488
379 | 144_487
380 | 144_473
381 | 228_323
382 | 113_424
383 | 061_190
384 | 200_318
385 | 134_320
386 | 187_482
387 | 228_280
388 | 331_527
389 | 089_267
390 | 191_368
391 | 221_307
392 | 221_438
393 | 334_470
394 | 155_476
395 | 319_523
396 | 199_321
397 | 124_439
398 | 101_103
399 | 138_430
400 | 195_410
401 | 126_491
402 | 080_345
403 | 131_316
404 | 232_595
405 | 370_344
406 | 290_557
407 | 199_283
408 | 151_470
409 | 235_482
410 | 127_491
411 | 110_421
412 | 128_482
413 | 153_405
414 | 197_368
415 | 222_315
416 | 106_442
417 | 215_467
418 | 068_347
419 | 149_262
420 | 059_259
421 | 281_252
422 | 176_506
423 | 134_489
424 | 179_609
425 | 257_269
426 | 237_434
427 | 106_441
428 | 088_362
429 | 164_519
430 | 063_084
431 | 237_441
432 | 183_481
433 | 062_081
434 | 292_455
435 | 154_351
436 | 351_399
437 | 153_485
438 | 207_423
439 | 201_445
440 | 206_301
441 | 059_082
442 | 159_420
443 | 255_273
444 | 147_496
445 | 187_401
446 | 128_312
447 | 065_059
448 | 213_315
449 | 160_399
450 | 177_339
451 | 195_308
452 | 122_439
453 | 208_431
454 | 229_293
455 | 194_485
456 | 138_476
457 | 335_498
458 | 237_364
459 | 253_256
460 | 126_506
461 | 216_183
462 | 136_486
463 | 157_421
464 | 065_117
465 | 100_356
466 | 304_528
467 | 379_348
468 | 179_610
469 | 177_517
470 | 241_288
471 | 186_511
472 | 120_430
473 | 243_466
474 | 084_363
475 | 197_488
476 | 304_343
477 | 124_419
478 | 224_334
479 | 164_527
480 | 245_592
481 | 141_500
482 | 330_501
483 | 330_513
484 | 125_303
485 | 230_311
486 | 145_489
487 | 164_523
488 | 329_510
489 | 376_348
490 | 114_433
491 | 157_332
492 | 172_577
493 | 141_481
494 | 207_438
495 | 168_268
496 | 251_448
497 | 330_510
498 | 224_489
499 | 233_484
500 | 232_322
501 | 133_440
502 | 174_407
503 | 083_210
504 | 191_492
505 | 174_015
506 | 209_275
507 | 132_482
508 | 248_444
509 | 046_088
510 | 034_112
511 | 300_453
512 | 067_077
513 | 237_592
514 | 183_489
515 | 234_292
516 | 141_401
517 | 292_542
518 | 200_365
519 | 061_194
520 | 351_330
521 | 065_352
522 | 314_514
523 | 211_410
524 | 195_406
525 | 112_493
526 | 142_433
527 | 172_586
528 | 181_504
529 | 108_333
530 | 213_300
531 | 104_417
532 | 151_480
533 | 201_364
534 | 235_446
535 | 163_479
536 | 271_486
537 | 129_479
538 | 182_612
539 | 231_274
540 | 166_597
541 | 159_417
542 | 140_442
543 | 241_587
544 | 183_502
545 | 130_486
546 | 160_479
547 | 034_146
548 | 139_487
549 | 203_337
550 | 129_473
551 | 262_441
552 | 164_446
553 | 232_279
554 | 126_303
555 | 168_386
556 | 242_280
557 | 317_513
558 | 227_283
559 | 133_360
560 | 202_425
561 | 121_446
562 | 275_409
563 | 250_258
564 | 214_467
565 | 063_198
566 | 147_481
567 | 220_286
568 | 156_332
569 | 113_431
570 | 252_256
571 | 151_486
572 | 213_416
573 | 183_378
574 | 276_403
575 | 183_239
576 | 271_429
577 | 052_076
578 | 321_340
579 | 122_446
580 | 115_406
581 | 149_474
582 | 213_293
583 | 329_511
584 | 221_328
585 | 219_478
586 | 144_335
587 | 153_490
588 | 165_447
589 | 143_474
590 | 080_346
591 | 237_449
592 | 140_494
593 | 143_496
594 | 194_468
595 | 138_363
596 | 086_360
597 | 236_273
598 | 194_264
599 | 127_396
600 | 252_597
601 | 050_168
602 | 236_335
603 | 252_583
604 | 232_333
605 | 253_447
606 | 252_592
607 | 236_375
608 | 156_387
609 | 225_335
610 | 066_060
611 | 262_413
612 | 234_334
613 | 186_408
614 | 227_277
615 | 189_360
616 | 201_384
617 | 234_289
618 | 129_499
619 | 328_498
620 | 199_425
621 | 217_429
622 | 057_195
623 | 173_350
624 | 194_345
625 | 301_342
626 | 130_479
627 | 110_417
628 | 217_335
629 | 184_356
630 | 163_526
631 | 218_292
632 | 065_120
633 | 243_450
634 | 191_495
635 | 094_337
636 | 131_428
637 | 326_316
638 | 167_586
639 | 218_314
640 | 124_499
641 | 185_410
642 | 149_497
643 | 183_407
644 | 077_300
645 | 330_516
646 | 269_263
647 | 256_271
648 | 159_461
649 | 171_352
650 | 204_480
651 | 166_406
652 | 150_486
653 | 110_450
654 | 182_240
655 | 094_358
656 | 243_263
657 | 233_297
658 | 236_593
659 | 246_334
660 | 271_440
661 | 170_402
662 | 141_487
663 | 191_510
664 | 143_488
665 | 240_447
666 | 131_422
667 | 141_409
668 | 192_490
669 | 369_348
670 | 135_484
671 | 247_333
672 | 122_504
673 | 088_617
674 | 207_388
675 | 049_232
676 | 228_466
677 | 242_597
678 | 165_523
679 | 227_432
680 | 218_427
681 | 183_605
682 | 051_176
683 | 255_275
684 | 100_357
685 | 236_336
686 | 257_277
687 | 213_332
688 | 196_360
689 | 151_469
690 | 077_612
691 | 127_502
692 | 369_341
693 | 159_405
694 | 252_595
695 | 168_274
696 | 096_341
697 | 215_425
698 | 080_334
699 | 121_502
700 | 319_513
701 | 325_500
702 | 184_236
703 | 253_285
704 | 185_458
705 | 364_321
706 | 201_433
707 | 124_301
708 | 194_400
709 | 143_497
710 | 241_469
711 | 317_344
712 | 222_289
713 | 231_279
714 | 060_086
715 | 209_433
716 | 119_466
717 | 224_298
718 | 109_421
719 | 115_440
720 | 232_285
721 | 275_475
722 | 281_251
723 | 100_326
724 | 243_270
725 | 261_443
726 | 116_432
727 | 123_509
728 | 376_349
729 | 266_439
730 | 191_331
731 | 052_247
732 | 239_363
733 | 220_266
734 | 313_341
735 | 181_607
736 | 193_366
737 | 358_323
738 | 121_499
739 | 131_438
740 | 200_456
741 | 184_604
742 | 236_332
743 | 197_450
744 | 239_334
745 | 109_158
746 | 171_413
747 | 239_356
748 | 330_503
749 | 205_288
750 | 198_593
751 | 224_429
752 | 319_340
753 | 119_332
754 | 214_319
755 | 304_542
756 | 227_292
757 | 318_513
758 | 254_285
759 | 227_290
760 | 180_241
761 | 176_507
762 | 080_613
763 | 368_344
764 | 133_484
765 | 295_409
766 | 173_504
767 | 121_559
768 | 125_484
769 | 268_324
770 | 168_587
771 | 225_326
772 | 231_296
--------------------------------------------------------------------------------
/validation_dst.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ClarkCGA/multi-temporal-crop-classification-training-data/82cca1accdf761e4eb44bad7ba8f1e5160d9bb93/validation_dst.png
--------------------------------------------------------------------------------
/workflow.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "cells": [
3 | {
4 | "cell_type": "markdown",
5 | "id": "88bf173d-f24b-401e-bdc5-68356cd84d26",
6 | "metadata": {},
7 | "source": [
8 | "# Pipeline to query, download and chip HLS and CDL layers"
9 | ]
10 | },
11 | {
12 | "cell_type": "code",
13 | "execution_count": null,
14 | "id": "5c8cde70-7cf0-49c8-a69e-3f2a45757876",
15 | "metadata": {},
16 | "outputs": [],
17 | "source": [
18 | "import geopandas\n",
19 | "import json\n",
20 | "import xarray\n",
21 | "import rasterio\n",
22 | "import rioxarray\n",
23 | "import os\n",
24 | "import fiona\n",
25 | "import urllib.request as urlreq\n",
26 | "import pandas as pd\n",
27 | "import numpy as np\n",
28 | "import requests\n",
29 | "import xmltodict\n",
30 | "import shutil\n",
31 | "import datetime\n",
32 | "import boto3\n",
33 | "import pyproj\n",
34 | "import multiprocessing as mp\n",
35 | "\n",
36 | "from pystac_client import Client \n",
37 | "from collections import defaultdict\n",
38 | "from glob import glob\n",
39 | "from rasterio.enums import Resampling\n",
40 | "import matplotlib.pyplot as plt\n",
41 | "from subprocess import Popen, PIPE\n",
42 | "from tqdm import tqdm\n",
43 | "from netrc import netrc\n",
44 | "from platform import system\n",
45 | "from getpass import getpass\n",
46 | "from rasterio.session import AWSSession\n",
47 | "from pathlib import Path\n",
48 | "\n",
49 | "pd.set_option('display.max_columns', None)\n",
50 | "pd.set_option('display.max_rows', None)"
51 | ]
52 | },
53 | {
54 | "cell_type": "markdown",
55 | "id": "a3a6db73-e1af-470e-8a13-e923fcd74fc0",
56 | "metadata": {},
57 | "source": [
58 | "### Setting folder paths and file paths"
59 | ]
60 | },
61 | {
62 | "cell_type": "code",
63 | "execution_count": null,
64 | "id": "d9f65500-9fc0-4b7f-8e4f-46c40a079acb",
65 | "metadata": {},
66 | "outputs": [],
67 | "source": [
68 | "cloud_thres = 5 # percent cloud cover for tile level query\n",
69 | "\n",
70 | "root_path = \"/data/\"\n",
71 | "req_path = \"/home/cdl_training_data/data/\"\n",
72 | "extra_files = \"/data/requirements/\"\n",
73 | "\n",
74 | "## file paths\n",
75 | "chip_file = req_path + \"chip_bbox.geojson\"\n",
76 | "chipping_json = req_path + \"chip_bbox_5070.geojson\"\n",
77 | "chip_csv = req_path + \"chip_tracker.csv\"\n",
78 | "kml_file = extra_files + 'sentinel_tile_grid.kml'\n",
79 | "# tile_tracker_csv = req_path + \"tile_tracker.csv\"\n",
80 | "cdl_file = extra_files + \"2022_30m_cdls.tif\"\n",
81 | "cdl_reclass_csv = req_path + \"cdl_total_dst.csv\"\n",
82 | "\n",
83 | "## folder paths\n",
84 | "chip_dir = root_path + 'chips/'\n",
85 | "tile_dir = root_path + 'tiles/'\n",
86 | "chip_dir_filt = root_path + 'chips_filtered/'\n",
87 | "chip_fmask_dir = root_path + 'chips_fmask/'\n",
88 | "\n",
89 | "## Create necessary folders\n",
90 | "if not os.path.exists(chip_dir):\n",
91 | " os.makedirs(chip_dir)\n",
92 | "if not os.path.exists(tile_dir):\n",
93 | " os.makedirs(tile_dir)\n",
94 | "if not os.path.exists(chip_dir_filt):\n",
95 | " os.makedirs(chip_dir_filt)\n",
96 | "if not os.path.exists(chip_fmask_dir):\n",
97 | " os.makedirs(chip_fmask_dir)\n"
98 | ]
99 | },
100 | {
101 | "cell_type": "markdown",
102 | "id": "da2291ef-8954-45fe-9d77-41048a58ecaa",
103 | "metadata": {},
104 | "source": [
105 | "### Data Processing"
106 | ]
107 | },
108 | {
109 | "cell_type": "code",
110 | "execution_count": null,
111 | "id": "572aad79-35fe-4640-85a0-a489fa18f09e",
112 | "metadata": {},
113 | "outputs": [],
114 | "source": [
115 | "# Loading chips bounding boxes from geojson\n",
116 | "with open(chip_file, \"r\") as file:\n",
117 | " chips = json.load(file)\n",
118 | " # print(chips)\n",
119 | "\n",
120 | "# Create lists about chip information to find tiles corresponding to it later\n",
121 | "chip_ids = []\n",
122 | "chip_x = []\n",
123 | "chip_y = []\n",
124 | "for item in chips['features']:\n",
125 | " chip_ids.append(item['properties']['id'])\n",
126 | " chip_x.append(item['properties']['center'][0])\n",
127 | " chip_y.append(item['properties']['center'][1])"
128 | ]
129 | },
130 | {
131 | "cell_type": "code",
132 | "execution_count": null,
133 | "id": "5d739262-045a-4ed9-8930-9dd5ea203e1e",
134 | "metadata": {},
135 | "outputs": [],
136 | "source": [
137 | "with open(\"/data/chip_ids.json\", \"w\") as f:\n",
138 | " json.dump(chip_ids, f, indent=2)"
139 | ]
140 | },
141 | {
142 | "cell_type": "code",
143 | "execution_count": null,
144 | "id": "a56e07ac-9b0a-4d7c-bad1-51586770a1f7",
145 | "metadata": {},
146 | "outputs": [],
147 | "source": [
148 | "# Read in sentinel kml file\n",
149 | "fiona.drvsupport.supported_drivers['KML'] = 'rw'\n",
150 | "tile_src = geopandas.read_file(kml_file, driver='KML')\n",
151 | "\n",
152 | "# Create table containing information about sentinel tiles\n",
153 | "tile_name = []\n",
154 | "tile_x = []\n",
155 | "tile_y = []\n",
156 | "for tile_ind in range(tile_src.shape[0]):\n",
157 | " tile_name.append(tile_src.iloc[tile_ind].Name)\n",
158 | " tile_x.append(tile_src.iloc[tile_ind].geometry.centroid.x)\n",
159 | " tile_y.append(tile_src.iloc[tile_ind].geometry.centroid.y)\n",
160 | "tile_name = np.array(tile_name)\n",
161 | "tile_x = np.array(tile_x)\n",
162 | "tile_y = np.array(tile_y)\n",
163 | "tile_src = pd.concat([tile_src, tile_src.bounds], axis = 1)"
164 | ]
165 | },
166 | {
167 | "cell_type": "code",
168 | "execution_count": null,
169 | "id": "618cc2a5-678c-47fd-a324-0d89e8f74e41",
170 | "metadata": {},
171 | "outputs": [],
172 | "source": [
173 | "def find_tile(x,y):\n",
174 | " \"\"\"\n",
175 | " Identify closest tile\n",
176 | " \"\"\"\n",
177 | " \n",
178 | " s = (tile_x - x)**2+(tile_y - y)**2\n",
179 | " tname = tile_name[np.argmin(s)]\n",
180 | " return(tname)"
181 | ]
182 | },
183 | {
184 | "cell_type": "code",
185 | "execution_count": null,
186 | "id": "923bcea3-efa4-41fc-90ef-0877dd270270",
187 | "metadata": {},
188 | "outputs": [],
189 | "source": [
190 | "chip_df = pd.DataFrame({\"chip_id\" : chip_ids, \"chip_x\" : chip_x, \"chip_y\" : chip_y})\n",
191 | "chip_df['tile'] = chip_df.apply(lambda row : find_tile(row['chip_x'], row['chip_y']), axis = 1)\n",
192 | "chip_df.tail(5)"
193 | ]
194 | },
195 | {
196 | "cell_type": "code",
197 | "execution_count": null,
198 | "id": "be986eb5-6c6e-4834-8fa5-dc36134e7839",
199 | "metadata": {},
200 | "outputs": [],
201 | "source": [
202 | "# Save dataframe to csv for later uses\n",
203 | "chip_df.to_csv(req_path + \"chip_df.csv\", index=False)"
204 | ]
205 | },
206 | {
207 | "cell_type": "code",
208 | "execution_count": null,
209 | "id": "67b451da-cf5f-4c2b-9531-15fa20121ea4",
210 | "metadata": {},
211 | "outputs": [],
212 | "source": [
213 | "chip_df = pd.read_csv(req_path + \"chip_df.csv\")"
214 | ]
215 | },
216 | {
217 | "cell_type": "code",
218 | "execution_count": null,
219 | "id": "9ca55cff-659f-4d68-a894-278af32fb9a9",
220 | "metadata": {},
221 | "outputs": [],
222 | "source": [
223 | "tiles = chip_df.tile.unique().tolist()\n",
224 | "tiles[0:5]"
225 | ]
226 | },
227 | {
228 | "cell_type": "markdown",
229 | "id": "e8575471-42cb-47e2-86c3-15068d840868",
230 | "metadata": {},
231 | "source": [
232 | "### Querying tile links based on geometry of chips"
233 | ]
234 | },
235 | {
236 | "cell_type": "code",
237 | "execution_count": null,
238 | "id": "8679f41b-87b1-4b3d-a2f2-559b9df3fdfe",
239 | "metadata": {},
240 | "outputs": [],
241 | "source": [
242 | "STAC_URL = 'https://cmr.earthdata.nasa.gov/stac'\n",
243 | "catalog = Client.open(f'{STAC_URL}/LPCLOUD/')"
244 | ]
245 | },
246 | {
247 | "cell_type": "code",
248 | "execution_count": null,
249 | "id": "c6d6080c-87aa-4ae8-bcee-71e9757b3dd6",
250 | "metadata": {},
251 | "outputs": [],
252 | "source": [
253 | "# Tile-level 5% cloud cover threshold query\n",
254 | "# tile_list = []\n",
255 | "print(f\"There are a total of {len(tiles)} tiles\")\n",
256 | "tile_iter = 0\n",
257 | "for current_tile in tiles:\n",
258 | "\n",
259 | " chip_df_filt = chip_df.loc[chip_df.tile == current_tile]#.reset_index()\n",
260 | " first_chip_id = chip_df_filt.chip_id.iloc[0]\n",
261 | " first_chip_index_in_json = chip_ids.index(first_chip_id)\n",
262 | " roi = chips['features'][first_chip_index_in_json]['geometry']\n",
263 | "\n",
264 | " search = catalog.search(\n",
265 | " collections = ['HLSS30.v2.0'],\n",
266 | " intersects = roi,\n",
267 | " datetime = '2022-03-01/2022-09-30',\n",
268 | " ) \n",
269 | " \n",
270 | " num_results = search.matched()\n",
271 | " item_collection = search.get_all_items()\n",
272 | " \n",
273 | " tile_name = \"T\" + current_tile\n",
274 | " iter_items = 0\n",
275 | " for i in tqdm(item_collection ,desc=f\"({tile_iter}/{len(tiles)})\"):\n",
276 | " if i.id.split('.')[2] == tile_name:\n",
277 | " if i.properties['eo:cloud_cover'] <= cloud_thres:\n",
278 | " response = requests.get(i.assets['metadata'].href)\n",
279 | " if response.status_code == 200:\n",
280 | " temp_xml = response.text\n",
281 | " temp_xml = xmltodict.parse(temp_xml)\n",
282 | " temp_dict = {\"tile_id\": tile_name, \"cloud_cover\": i.properties['eo:cloud_cover'],\n",
283 | " \"date\": datetime.datetime.strptime(i.properties['datetime'].split('T')[0], \"%Y-%m-%d\"), \n",
284 | " \"spatial_cover\": int(temp_xml['Granule']['AdditionalAttributes']['AdditionalAttribute'][3]['Values']['Value']),\n",
285 | " \"http_links\": {\"B02\": i.assets['B02'].href, \"B03\": i.assets['B03'].href, \"B04\": i.assets['B04'].href, \"B8A\": i.assets['B8A'].href,\n",
286 | " \"B11\": i.assets['B11'].href, \"B12\": i.assets['B12'].href, \"Fmask\": i.assets['Fmask']},\n",
287 | " \"s3_links\": {\"B02\": i.assets['B02'].href.replace('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/', 's3:/'), \n",
288 | " \"B03\": i.assets['B03'].href.replace('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/', 's3:/'), \n",
289 | " \"B04\": i.assets['B04'].href.replace('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/', 's3:/'), \n",
290 | " \"B8A\": i.assets['B8A'].href.replace('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/', 's3:/'),\n",
291 | " \"B11\": i.assets['B11'].href.replace('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/', 's3:/'),\n",
292 | " \"B12\": i.assets['B12'].href.replace('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/', 's3:/'),\n",
293 | " \"Fmask\": i.assets['Fmask'].href.replace('https://data.lpdaac.earthdatacloud.nasa.gov/lp-prod-protected/', 's3:/')}}\n",
294 | " tile_list.append(temp_dict)\n",
295 | " iter_items += 1\n",
296 | " else: \n",
297 | " assert False, f\"Failed to fetch XML from {i.assets['metadata'].href}. Error code: {response.status_code}\"\n",
298 | " \n",
299 | " tile_iter += 1\n",
300 | " \n",
301 | "tile_df = pd.DataFrame(tile_list)"
302 | ]
303 | },
304 | {
305 | "cell_type": "code",
306 | "execution_count": null,
307 | "id": "7431fc8b-0f64-408b-8500-24a1bae5aa01",
308 | "metadata": {},
309 | "outputs": [],
310 | "source": [
311 | "# Save to csv for later uses\n",
312 | "tile_df.to_csv(req_path + \"tile_df.csv\", index=False)"
313 | ]
314 | },
315 | {
316 | "cell_type": "code",
317 | "execution_count": null,
318 | "id": "5a1fbb28-c2ec-41f1-bc60-c0c18b358726",
319 | "metadata": {},
320 | "outputs": [],
321 | "source": [
322 | "tile_df = pd.read_csv(req_path + \"tile_df.csv\")"
323 | ]
324 | },
325 | {
326 | "cell_type": "code",
327 | "execution_count": null,
328 | "id": "bf8f2617-3d4a-4651-92ba-6d82f4e39f35",
329 | "metadata": {},
330 | "outputs": [],
331 | "source": [
332 | "tile_df.head()"
333 | ]
334 | },
335 | {
336 | "cell_type": "markdown",
337 | "id": "6c8a157d-9e4a-4a79-b98e-d94e8f1916bf",
338 | "metadata": {},
339 | "source": [
340 | "### Filtering based on spatial coverage of the tiles we gathered earlier"
341 | ]
342 | },
343 | {
344 | "cell_type": "code",
345 | "execution_count": null,
346 | "id": "59767242-2158-4cd7-bdc7-1719207f0552",
347 | "metadata": {},
348 | "outputs": [],
349 | "source": [
350 | "def spatial_filtering (dataframe):\n",
351 | " \"\"\"\n",
352 | " Using spatial coverage percentage to filter chips\n",
353 | "\n",
354 | " Args:\n",
355 | " dataframe: A pandas dataframe that generated previously\n",
356 | " \"\"\"\n",
357 | " cover_list = [100, 90, 80, 70, 60, 50]\n",
358 | " tile_list_ft = []\n",
359 | " tile_list = dataframe.tile_id.unique().tolist()\n",
360 | " \n",
361 | " for tile in tqdm(tile_list):\n",
362 | " temp_df = dataframe[dataframe.tile_id == tile]\n",
363 | " for cover_pct in cover_list:\n",
364 | " \n",
365 | " temp_df_filtered = temp_df[temp_df.spatial_cover >= cover_pct]\n",
366 | " if len(temp_df_filtered) >= 3:\n",
367 | " for i in range(len(temp_df_filtered)):\n",
368 | " tile_list_ft.append(temp_df_filtered.iloc[i])\n",
369 | " break\n",
370 | " \n",
371 | " tile_df_filtered = pd.DataFrame(tile_list_ft)\n",
372 | " return tile_df_filtered"
373 | ]
374 | },
375 | {
376 | "cell_type": "code",
377 | "execution_count": null,
378 | "id": "ca6941b5-9241-4626-bbec-56118c91ba95",
379 | "metadata": {},
380 | "outputs": [],
381 | "source": [
382 | "cover_df = spatial_filtering(tile_df)"
383 | ]
384 | },
385 | {
386 | "cell_type": "code",
387 | "execution_count": null,
388 | "id": "2b02b75c-55cc-43a1-a6d9-913771eb3043",
389 | "metadata": {},
390 | "outputs": [],
391 | "source": [
392 | "def select_scenes(dataframe):\n",
393 | " \"\"\"\n",
394 | " Selecting best spatial covered scenes based on timesteps\n",
395 | "\n",
396 | " Args:\n",
397 | " dataframe: A pandas dataframe that generated previously\n",
398 | " \"\"\"\n",
399 | " select_tiles = []\n",
400 | " tile_list = dataframe.tile_id.unique().tolist()\n",
401 | "\n",
402 | " for tile in tqdm(tile_list):\n",
403 | " temp_df = dataframe[dataframe.tile_id == tile].sort_values('date').reset_index(drop=True)\n",
404 | " select_tiles.extend([temp_df.iloc[0], temp_df.iloc[len(temp_df) // 2], temp_df.iloc[-1]])\n",
405 | "\n",
406 | " return pd.DataFrame(select_tiles).reset_index(drop=True)"
407 | ]
408 | },
409 | {
410 | "cell_type": "code",
411 | "execution_count": null,
412 | "id": "b23da90c-e163-45fc-9ea6-ef8e3130bc95",
413 | "metadata": {},
414 | "outputs": [],
415 | "source": [
416 | "selected_tiles = select_scenes(cover_df)"
417 | ]
418 | },
419 | {
420 | "cell_type": "code",
421 | "execution_count": null,
422 | "id": "f7caff71-b5be-4d54-be44-a0938bd55159",
423 | "metadata": {},
424 | "outputs": [],
425 | "source": [
426 | "selected_tiles.head()"
427 | ]
428 | },
429 | {
430 | "cell_type": "code",
431 | "execution_count": null,
432 | "id": "264fae95-6630-46e0-bd05-83f866cd7a41",
433 | "metadata": {},
434 | "outputs": [],
435 | "source": [
436 | "# Save to csv for later uses\n",
437 | "selected_tiles.to_csv(req_path + \"selected_tiles.csv\", index=False)"
438 | ]
439 | },
440 | {
441 | "cell_type": "code",
442 | "execution_count": null,
443 | "id": "3fc0ddb0-da8d-4fa7-8de3-241e70cacbe1",
444 | "metadata": {},
445 | "outputs": [],
446 | "source": [
447 | "selected_tiles = pd.read_csv(req_path + \"selected_tiles.csv\")"
448 | ]
449 | },
450 | {
451 | "cell_type": "code",
452 | "execution_count": null,
453 | "id": "3221c2e7-ed10-412e-8b12-fd398875aafc",
454 | "metadata": {},
455 | "outputs": [],
456 | "source": [
457 | "selected_tiles.head()"
458 | ]
459 | },
460 | {
461 | "cell_type": "markdown",
462 | "id": "5e34d499-1a6e-483f-9f19-8daa6b1dd4c3",
463 | "metadata": {},
464 | "source": [
465 | "### Data downloading\n",
466 | "\n",
467 | "Creating netrc file on root for credentials (Run Once each session)"
468 | ]
469 | },
470 | {
471 | "cell_type": "code",
472 | "execution_count": null,
473 | "id": "38664ffa-cd57-4f1b-a7f3-cdb63554ad94",
474 | "metadata": {},
475 | "outputs": [],
476 | "source": [
477 | "urs = 'urs.earthdata.nasa.gov' # Earthdata URL endpoint for authentication\n",
478 | "prompts = ['Enter NASA Earthdata Login Username: ',\n",
479 | " 'Enter NASA Earthdata Login Password: ']\n",
480 | "\n",
481 | "# Determine the OS (Windows machines usually use an '_netrc' file)\n",
482 | "netrc_name = \"_netrc\" if system()==\"Windows\" else \".netrc\"\n",
483 | "\n",
484 | "# Determine if netrc file exists, and if so, if it includes NASA Earthdata Login Credentials\n",
485 | "try:\n",
486 | " netrcDir = os.path.expanduser(f\"~/{netrc_name}\")\n",
487 | " netrc(netrcDir).authenticators(urs)[0]\n",
488 | "\n",
489 | "# Below, create a netrc file and prompt user for NASA Earthdata Login Username and Password\n",
490 | "except FileNotFoundError:\n",
491 | " homeDir = os.path.expanduser(\"~\")\n",
492 | " Popen('touch {0}{2} | echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)\n",
493 | " Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)\n",
494 | " Popen('echo \\'password {} \\'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)\n",
495 | " # Set restrictive permissions\n",
496 | " Popen('chmod 0600 {0}{1}'.format(homeDir + os.sep, netrc_name), shell=True)\n",
497 | "\n",
498 | " # Determine OS and edit netrc file if it exists but is not set up for NASA Earthdata Login\n",
499 | "except TypeError:\n",
500 | " homeDir = os.path.expanduser(\"~\")\n",
501 | " Popen('echo machine {1} >> {0}{2}'.format(homeDir + os.sep, urs, netrc_name), shell=True)\n",
502 | " Popen('echo login {} >> {}{}'.format(getpass(prompt=prompts[0]), homeDir + os.sep, netrc_name), shell=True)\n",
503 | " Popen('echo \\'password {} \\'>> {}{}'.format(getpass(prompt=prompts[1]), homeDir + os.sep, netrc_name), shell=True)"
504 | ]
505 | },
506 | {
507 | "cell_type": "markdown",
508 | "id": "9aace234-4fba-4cb2-adb6-8e17601b067c",
509 | "metadata": {},
510 | "source": [
511 | "### Getting temporary credentials for NASA's S3 Bucket(Run once every 1 hrs)"
512 | ]
513 | },
514 | {
515 | "cell_type": "code",
516 | "execution_count": null,
517 | "id": "fadfd603-27d5-4354-912a-f5f7ebdb799f",
518 | "metadata": {},
519 | "outputs": [],
520 | "source": [
521 | "s3_cred_endpoint = 'https://data.lpdaac.earthdatacloud.nasa.gov/s3credentials'"
522 | ]
523 | },
524 | {
525 | "cell_type": "code",
526 | "execution_count": null,
527 | "id": "23b5232e-2040-40de-8915-875c13925ee8",
528 | "metadata": {},
529 | "outputs": [],
530 | "source": [
531 | "def get_temp_creds():\n",
532 | " temp_creds_url = s3_cred_endpoint\n",
533 | " return requests.get(temp_creds_url).json()"
534 | ]
535 | },
536 | {
537 | "cell_type": "code",
538 | "execution_count": null,
539 | "id": "6eadb6ff-5b4a-4579-bbef-a3a3e2b8212a",
540 | "metadata": {},
541 | "outputs": [],
542 | "source": [
543 | "temp_creds_req = get_temp_creds()\n",
544 | "\n",
545 | "boto3_session = boto3.Session(aws_access_key_id=temp_creds_req['accessKeyId'], \n",
546 | " aws_secret_access_key=temp_creds_req['secretAccessKey'],\n",
547 | " aws_session_token=temp_creds_req['sessionToken'],\n",
548 | " region_name='us-west-2')\n",
549 | "rio_env = rasterio.Env(AWSSession(boto3_session),\n",
550 | " GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',\n",
551 | " GDAL_HTTP_COOKIEFILE=os.path.expanduser('~/cookies.txt'),\n",
552 | " GDAL_HTTP_COOKIEJAR=os.path.expanduser('~/cookies.txt'))\n",
553 | "rio_env.__enter__()"
554 | ]
555 | },
556 | {
557 | "cell_type": "markdown",
558 | "id": "bcda5481-8823-4e02-8535-67f490da9782",
559 | "metadata": {},
560 | "source": [
561 | "### Tile downloading (Run the crendentials chunk if connecting error)"
562 | ]
563 | },
564 | {
565 | "cell_type": "code",
566 | "execution_count": null,
567 | "id": "3ee37044-857c-4d4a-a41d-971561f92ece",
568 | "metadata": {},
569 | "outputs": [],
570 | "source": [
571 | "selected_tiles.head()"
572 | ]
573 | },
574 | {
575 | "cell_type": "code",
576 | "execution_count": null,
577 | "id": "3577d9aa-cf45-4cce-b0f6-df61dd51215d",
578 | "metadata": {},
579 | "outputs": [],
580 | "source": [
581 | "def tile_download(table, from_csv = True):\n",
582 | " \"\"\"\n",
583 | " Downloading tiles by reading from the metadata information gathered earlier\n",
584 | "\n",
585 | " Args:\n",
586 | " table: A pandas dataframe that generated previously\n",
587 | " boto3_session: The session that set earlier when getting credentials\n",
588 | " from_csv: If the tile information is from a csv, then True\n",
589 | " \"\"\"\n",
590 | " \n",
591 | " \n",
592 | " info_list = []\n",
593 | " bands = [\"B02\",\"B03\",\"B04\",\"B8A\",\"B11\",\"B12\",\"Fmask\"]\n",
594 | " accept_tiles = np.unique(table.tile_id)\n",
595 | " for tile in tqdm(accept_tiles):\n",
596 | " temp_creds_req = get_temp_creds()\n",
597 | "\n",
598 | " boto3_session = boto3.Session(aws_access_key_id=temp_creds_req['accessKeyId'], \n",
599 | " aws_secret_access_key=temp_creds_req['secretAccessKey'],\n",
600 | " aws_session_token=temp_creds_req['sessionToken'],\n",
601 | " region_name='us-west-2')\n",
602 | " temp_tb = table[table.tile_id == tile]\n",
603 | " for i in range(3):\n",
604 | " if from_csv:\n",
605 | " bands_dict = json.loads(temp_tb.iloc[i].s3_links.replace(\"'\", '\"'))\n",
606 | " else:\n",
607 | " bands_dict = temp_tb.iloc[i].s3_links\n",
608 | " for band in bands:\n",
609 | " temp_key = bands_dict[band].replace(\"s3:/\", \"\")\n",
610 | " temp_sav_path = f\"/data/tiles/{bands_dict[band].split('/')[2]}/{bands_dict[band].split('/')[3]}\"\n",
611 | " os.makedirs(f\"/data/tiles/{bands_dict[band].split('/')[2]}\", exist_ok=True)\n",
612 | " if not Path(temp_sav_path).is_file():\n",
613 | " boto3_session.resource('s3').Bucket('lp-prod-protected').download_file(Key = temp_key, Filename = temp_sav_path)\n",
614 | " temp_dict = {\"tile\":tile, \"timestep\":i, \"date\":temp_tb.iloc[i].date, \"save_path\":f\"/data/tiles/{bands_dict[band].split('/')[2]}/\", \"filename\":bands_dict[\"B02\"].split('/')[3].replace(\".B02.tif\",\"\")}\n",
615 | " info_list.append(temp_dict)\n",
616 | " return pd.DataFrame(info_list)"
617 | ]
618 | },
619 | {
620 | "cell_type": "code",
621 | "execution_count": null,
622 | "id": "90e5a786-2fc1-48de-9931-b2cbb7ccef67",
623 | "metadata": {},
624 | "outputs": [],
625 | "source": [
626 | "track_df = tile_download(selected_tiles, from_csv=True)"
627 | ]
628 | },
629 | {
630 | "cell_type": "code",
631 | "execution_count": null,
632 | "id": "57d8a34d-61e6-42ca-a71e-7e85a2fd67d4",
633 | "metadata": {},
634 | "outputs": [],
635 | "source": [
636 | "track_df.to_csv(req_path + \"track_df.csv\", index=False)"
637 | ]
638 | },
639 | {
640 | "cell_type": "markdown",
641 | "id": "468880b3-37e1-41c4-98fd-70944d92e2bd",
642 | "metadata": {},
643 | "source": [
644 | "### Using the CDL tif to reproject each HLS scene to CDL projection"
645 | ]
646 | },
647 | {
648 | "cell_type": "code",
649 | "execution_count": null,
650 | "id": "7f8fa877-023b-47c0-85cd-eddb65e222d8",
651 | "metadata": {},
652 | "outputs": [],
653 | "source": [
654 | "track_df = pd.read_csv(req_path + \"track_df.csv\")"
655 | ]
656 | },
657 | {
658 | "cell_type": "code",
659 | "execution_count": null,
660 | "id": "34938d1a-1c6e-4aec-aae7-7835583c478c",
661 | "metadata": {},
662 | "outputs": [],
663 | "source": [
664 | "def point_transform(coor, src_crs, target_crs=5070):\n",
665 | " proj = pyproj.Transformer.from_crs(src_crs, target_crs, always_xy=True)\n",
666 | " projected_coor = proj.transform(coor[0], coor[1])\n",
667 | " return [projected_coor[0], projected_coor[1]]\n",
668 | "\n",
669 | "def find_nearest(array, value):\n",
670 | " idx = (np.abs(array - value)).argmin()\n",
671 | " return array[idx]"
672 | ]
673 | },
674 | {
675 | "cell_type": "code",
676 | "execution_count": null,
677 | "id": "6a3f20ce-bffe-4fb3-bb29-4a1c90c99820",
678 | "metadata": {},
679 | "outputs": [],
680 | "source": [
681 | "def reproject_hls(tile_path,\n",
682 | " cdl_ds,\n",
683 | " target_crs =\"EPSG:5070\", \n",
684 | " remove_original = True, \n",
685 | " resampling_method = Resampling.bilinear):\n",
686 | " \n",
687 | " \"\"\"\n",
688 | " This function receives the path to a specific HLS tile and reproject it to the targeting crs_ds.\n",
689 | " The option of removing the raw HLS tile is provided\n",
690 | " \n",
691 | " Assumptions:\n",
692 | " - tile_path is a full path that end with .tif\n",
693 | " - cdl_ds is a rioxarray dataset that is opened with `cache=False` setting.\n",
694 | " \n",
695 | " \n",
696 | " Inputs:\n",
697 | " - tile_path: The full path to a specific HLS tile\n",
698 | " - target_crs: The crs that you wish to reproject the tile to, default is EPSG 4326\n",
699 | " - remove_original: The option to remove raw HLS tile after reprojecting, default is True\n",
700 | " - resampling_method: The method that rioxarray use to reproject, default is bilinear\n",
701 | " \"\"\"\n",
702 | "\n",
703 | " xds = rioxarray.open_rasterio(tile_path)\n",
704 | " half_scene_len = np.abs(np.round((xds.x.max().data - xds.x.min().data) / 2))\n",
705 | " coor_min = point_transform([xds.x.min().data - half_scene_len, xds.y.min().data - half_scene_len], xds.rio.crs)\n",
706 | " coor_max = point_transform([xds.x.max().data + half_scene_len, xds.y.max().data + half_scene_len], xds.rio.crs)\n",
707 | " \n",
708 | " x0 = find_nearest(cdl_ds.x.data, coor_min[0])\n",
709 | " y0 = find_nearest(cdl_ds.y.data, coor_min[1])\n",
710 | " x1 = find_nearest(cdl_ds.x.data, coor_max[0])\n",
711 | " y1 = find_nearest(cdl_ds.y.data, coor_max[1])\n",
712 | " \n",
713 | " cdl_for_reprojection = cdl_ds.rio.slice_xy(x0, y0, x1, y1)\n",
714 | " \n",
715 | " xds_new = xds.rio.reproject_match(cdl_for_reprojection, resampling = resampling_method)\n",
716 | "\n",
717 | " if remove_original:\n",
718 | " if Path(tile_path).is_file():\n",
719 | " os.remove(tile_path)\n",
720 | " xds_new.rio.to_raster(raster_path = tile_path.replace(\".tif\", \".reproject.tif\"))\n",
721 | " else:\n",
722 | " xds_new.rio.to_raster(raster_path = tile_path.replace(\".tif\", \".reproject.tif\"))"
723 | ]
724 | },
725 | {
726 | "cell_type": "code",
727 | "execution_count": null,
728 | "id": "2b9d0026-b5a8-4766-aa5c-b58a9194b3c5",
729 | "metadata": {},
730 | "outputs": [],
731 | "source": [
732 | "# Add a quality control to ensure there are three scenes for each tile.\n",
733 | "failed_tiles = []\n",
734 | "for tile in list(track_df.tile.unique()):\n",
735 | " if len(track_df[track_df.tile == tile]) != 3:\n",
736 | " failed_tiles.append(tile)\n",
737 | "if len(failed_tiles) == 0:\n",
738 | " print(\"All tiles passed the quality test!\")\n",
739 | "else:\n",
740 | " print(f\"Tile {failed_tiles} does not pass the quality test.\")"
741 | ]
742 | },
743 | {
744 | "cell_type": "code",
745 | "execution_count": null,
746 | "id": "ee2ae184-b9dc-4b90-bf50-32e291799aec",
747 | "metadata": {},
748 | "outputs": [],
749 | "source": [
750 | "track_df[\"cdl_file\"] = cdl_file\n",
751 | "track_df.loc[:, \"bands\"] = '[\"B02\",\"B03\",\"B04\",\"B8A\",\"B11\",\"B12\",\"Fmask\"]'"
752 | ]
753 | },
754 | {
755 | "cell_type": "code",
756 | "execution_count": null,
757 | "id": "524e4a00-1773-4615-b844-f60d4ba52eff",
758 | "metadata": {},
759 | "outputs": [],
760 | "source": [
761 | "track_df.head()"
762 | ]
763 | },
764 | {
765 | "cell_type": "code",
766 | "execution_count": null,
767 | "id": "4530d1d2-a04d-431e-8b89-51651163537d",
768 | "metadata": {},
769 | "outputs": [],
770 | "source": [
771 | "def hls_process(kwargs):\n",
772 | "\n",
773 | " remove_original = True\n",
774 | " \n",
775 | " save_path = kwargs[\"save_path\"]\n",
776 | " filename= kwargs[\"filename\"]\n",
777 | " bands = json.loads(kwargs[\"bands\"])\n",
778 | " cdl_file = kwargs[\"cdl_file\"]\n",
779 | " \n",
780 | " cdl_ds = rioxarray.open_rasterio(cdl_file, cache=False)\n",
781 | " \n",
782 | " for band in bands:\n",
783 | " tile_path = f\"{save_path}{filename}.{band}.tif\"\n",
784 | " if Path(tile_path).is_file():\n",
785 | " if band == \"Fmask\":\n",
786 | " reproject_hls(tile_path, cdl_ds, remove_original, resampling_method = Resampling.nearest)\n",
787 | " else :\n",
788 | " reproject_hls(tile_path, cdl_ds, remove_original) "
789 | ]
790 | },
791 | {
792 | "cell_type": "code",
793 | "execution_count": null,
794 | "id": "283baa4e-5df7-47ad-9026-6efc7bed6145",
795 | "metadata": {},
796 | "outputs": [],
797 | "source": [
798 | "with mp.Pool(processes=5) as pool:\n",
799 | " pool.map(hls_process, track_df.to_dict('records'))"
800 | ]
801 | },
802 | {
803 | "cell_type": "markdown",
804 | "id": "6fba840c-33c6-4fb3-917f-538411ba38da",
805 | "metadata": {},
806 | "source": [
807 | "### Chipping"
808 | ]
809 | },
810 | {
811 | "cell_type": "code",
812 | "execution_count": null,
813 | "id": "28119f07-521e-4356-a803-6c5e894ebe16",
814 | "metadata": {},
815 | "outputs": [],
816 | "source": [
817 | "# Getting all saved dataframes and json\n",
818 | "chip_df = pd.read_csv(req_path + \"chip_df.csv\")\n",
819 | "with open(\"/data/chip_ids.json\", 'r') as f:\n",
820 | " chip_ids = json.load(f)\n",
821 | "track_df = pd.read_csv(req_path + \"track_df.csv\")\n",
822 | "with open(chip_file, \"r\") as file:\n",
823 | " chips = json.load(file)"
824 | ]
825 | },
826 | {
827 | "cell_type": "code",
828 | "execution_count": null,
829 | "id": "38d86289-e212-41b3-9381-a30ae0ebb171",
830 | "metadata": {},
831 | "outputs": [],
832 | "source": [
833 | "tiles_to_chip = track_df.tile.unique().tolist()\n",
834 | "with open(chipping_json, \"r\") as file_chip:\n",
835 | " chipping_js = json.load(file_chip)"
836 | ]
837 | },
838 | {
839 | "cell_type": "code",
840 | "execution_count": null,
841 | "id": "11e4d2c0-4f6c-4873-a611-49ae4c2eba52",
842 | "metadata": {},
843 | "outputs": [],
844 | "source": [
845 | "## set up CDL reclass\n",
846 | "cdl_class_df = pd.read_csv(cdl_reclass_csv)\n",
847 | "crop_dict = dict(zip(cdl_class_df.old_class_value, cdl_class_df.new_class_value))\n",
848 | "\n",
849 | "def crop_multi(x):\n",
850 | " return(crop_dict[x])\n",
851 | "\n",
852 | "c_multi = np.vectorize(crop_multi)"
853 | ]
854 | },
855 | {
856 | "cell_type": "code",
857 | "execution_count": null,
858 | "id": "a2c4dbdf-ece0-45f8-803a-cfe36abb41c0",
859 | "metadata": {},
860 | "outputs": [],
861 | "source": [
862 | "def check_qa(qa_path, shape, valid_qa = [0, 4, 32, 36, 64, 68, 96, 100, 128, 132, 160, 164, 192, 196, 224, 228]):\n",
863 | " \n",
864 | " \"\"\"\n",
865 | " This function receives a path to a qa file, and a geometry. It clips the QA file to the geometry. \n",
866 | " It returns the number of valid QA pixels in the geometry, and the clipped values.\n",
867 | " \n",
868 | " Assumptions: The valid_qa values are taken from Ben Mack's post:\n",
869 | " https://benmack.github.io/nasa_hls/build/html/tutorials/Working_with_HLS_datasets_and_nasa_hls.html\n",
870 | " \n",
871 | " Inputs:\n",
872 | " - qa_path: full path to reprojected QA tif file\n",
873 | " - shape: 'geometry' property of single polygon feature read by fiona\n",
874 | " - valid_qa: list of integer values that are 'valid' for QA band.\n",
875 | " \n",
876 | "\n",
877 | " \n",
878 | " \"\"\"\n",
879 | " with rasterio.open(qa_path) as src:\n",
880 | " out_image, out_transform = rasterio.mask.mask(src, shape, crop=True)\n",
881 | " vals = out_image.flatten()\n",
882 | " unique, counts = np.unique(vals, return_counts=True)\n",
883 | " qa_df = pd.DataFrame({\"qa_val\" : unique, \"counts\" : counts})\n",
884 | " qa_df\n",
885 | " qa_df[~ qa_df.qa_val.isin(valid_qa)].sort_values(['counts'], ascending = False)\n",
886 | " qa_df['pct'] = (100 *qa_df['counts'])/(224.0 * 224.0)\n",
887 | " \n",
888 | " bad_qa = qa_df[~ qa_df.qa_val.isin(valid_qa)].sort_values(['counts'], ascending = False)\n",
889 | " if len(bad_qa) > 0:\n",
890 | " highest_invalid_percent = bad_qa.pct.tolist()[0]\n",
891 | " else: \n",
892 | " highest_invalid_percent = 0\n",
893 | " # ncell = len(vals)\n",
894 | " valid_count = sum(x in valid_qa for x in vals)\n",
895 | " return(valid_count, highest_invalid_percent, out_image[0])"
896 | ]
897 | },
898 | {
899 | "cell_type": "code",
900 | "execution_count": null,
901 | "id": "354ebe46-35c7-473a-8827-7348fa2a0249",
902 | "metadata": {},
903 | "outputs": [],
904 | "source": [
905 | "def process_chip(chip_id, \n",
906 | " chip_tile,\n",
907 | " shape,\n",
908 | " track_csv,\n",
909 | " cdl_file,\n",
910 | " bands = [\"B02\", \"B03\", \"B04\", \"B8A\", \"B11\", \"B12\"]):\n",
911 | " \n",
912 | " \"\"\"\n",
913 | " This function receives a chip id, HLS tile, chip geometry, and a list of bands to process. \n",
914 | " \n",
915 | " Assumptions:\n",
916 | " \n",
917 | " Inputs:\n",
918 | " - chip_id: string of chip id, e.g. '000_001'\n",
919 | " - chip_tile: string of HLS tile , e.g. '15ABC'\n",
920 | " - shape: 'geometry' property of single polygon feature read by fiona\n",
921 | " \n",
922 | " The function writes out a multi-date TIF containing the bands for each of the three image dates for an HLS tile. \n",
923 | " The function writes out a multi-date TIF containing the QA bands of each date.\n",
924 | " The function writes out a chipped version of CDL. \n",
925 | " The function calls check_qa(), which makes assumptions about what QA pixels are valid.\n",
926 | " The function returns the number of valid QA pixels at each date, as a tuple.\n",
927 | " \n",
928 | " \"\"\"\n",
929 | " ## get reprojected image paths\n",
930 | " tile_info_df = track_csv[track_csv.tile == chip_tile]\n",
931 | " \n",
932 | " selected_image_folders = tile_info_df.save_path.to_list()\n",
933 | "\n",
934 | " \n",
935 | " assert len(selected_image_folders) == 3\n",
936 | " \n",
937 | " \n",
938 | " first_image_date = tile_info_df.iloc[0].date\n",
939 | " second_image_date = tile_info_df.iloc[1].date\n",
940 | " third_image_date = tile_info_df.iloc[2].date\n",
941 | " \n",
942 | "\n",
943 | " all_date_images = []\n",
944 | " all_date_qa = []\n",
945 | " \n",
946 | " for i in range(3):\n",
947 | " for band in bands:\n",
948 | " all_date_images.append(tile_info_df.iloc[i].save_path + f\"{tile_info_df.iloc[i].filename}.{band}.reproject.tif\")\n",
949 | " all_date_qa.append(tile_info_df.iloc[i].save_path + f\"{tile_info_df.iloc[i].filename}.Fmask.reproject.tif\")\n",
950 | " \n",
951 | "\n",
952 | " valid_first, bad_pct_first, qa_first = check_qa(all_date_qa[0], shape)\n",
953 | " valid_second, bad_pct_second, qa_second = check_qa(all_date_qa[1], shape)\n",
954 | " valid_third, bad_pct_third, qa_third = check_qa(all_date_qa[2], shape)\n",
955 | " \n",
956 | " qa_bands = []\n",
957 | " qa_bands.append(qa_first)\n",
958 | " qa_bands.append(qa_second)\n",
959 | " qa_bands.append(qa_third)\n",
960 | " qa_bands = np.array(qa_bands).astype(np.uint8)\n",
961 | " \n",
962 | "\n",
963 | " assert len(all_date_images) == 3 * len(bands)\n",
964 | " \n",
965 | " out_bands = []\n",
966 | " print('out_bands_loop')\n",
967 | " for img in all_date_images:\n",
968 | " with rasterio.open(img) as src:\n",
969 | " print(img)\n",
970 | " out_image, out_transform = rasterio.mask.mask(src, shape, crop=True)\n",
971 | " out_meta = src.meta\n",
972 | " out_bands.append(out_image[0])\n",
973 | " \n",
974 | " out_bands = np.array(out_bands)\n",
975 | " # print(out_bands.shape)\n",
976 | "\n",
977 | "\n",
978 | " out_meta.update({\"driver\": \"GTiff\",\n",
979 | " \"height\": out_bands.shape[1],\n",
980 | " \"width\": out_bands.shape[2],\n",
981 | " \"count\": out_bands.shape[0],\n",
982 | " \"transform\": out_transform})\n",
983 | " \n",
984 | " # get NA count for HLS\n",
985 | " na_count = sum(out_bands.flatten() == -1000)\n",
986 | " \n",
987 | " # reclass negative HLS values to 0\n",
988 | " out_bands = np.clip(out_bands, 0, None)\n",
989 | " \n",
990 | " # write HLS chips\n",
991 | " with rasterio.open(chip_dir + str(chip_id) + \"_merged.tif\", \"w\", **out_meta) as dest:\n",
992 | " dest.write(out_bands)\n",
993 | " \n",
994 | " \n",
995 | " \n",
996 | " ## write QA bands\n",
997 | " out_meta.update({\"driver\": \"GTiff\",\n",
998 | " \"height\": qa_bands.shape[1],\n",
999 | " \"width\": qa_bands.shape[2],\n",
1000 | " \"count\": qa_bands.shape[0],\n",
1001 | " \"transform\": out_transform})\n",
1002 | " \n",
1003 | " with rasterio.open(chip_fmask_dir + str(chip_id) + \"_Fmask.tif\", \"w\", **out_meta) as dest:\n",
1004 | " dest.write(qa_bands) \n",
1005 | "\n",
1006 | " \n",
1007 | " ## clip cdl to chip\n",
1008 | " with rasterio.open(cdl_file) as src:\n",
1009 | " out_image, out_transform = rasterio.mask.mask(src, shape, crop=True)\n",
1010 | " out_meta = src.meta\n",
1011 | " colormap = src.colormap(1)\n",
1012 | "\n",
1013 | " out_meta.update({\"driver\": \"GTiff\",\n",
1014 | " \"height\": out_image.shape[1],\n",
1015 | " \"width\": out_image.shape[2],\n",
1016 | " \"transform\": out_transform})\n",
1017 | " \n",
1018 | " # write multiclass reclassed CDL chips\n",
1019 | " out_image_multi = c_multi(out_image).astype(np.uint8)\n",
1020 | " with rasterio.open(chip_dir + str(chip_id) + \".mask.tif\", \"w\", **out_meta) as dest:\n",
1021 | " dest.write(out_image_multi)\n",
1022 | " dest.write_colormap(1, colormap)\n",
1023 | "\n",
1024 | " \n",
1025 | " return(valid_first,\n",
1026 | " valid_second,\n",
1027 | " valid_third, \n",
1028 | " bad_pct_first,\n",
1029 | " bad_pct_second,\n",
1030 | " bad_pct_third,\n",
1031 | " na_count,\n",
1032 | " first_image_date,\n",
1033 | " second_image_date,\n",
1034 | " third_image_date)\n",
1035 | " "
1036 | ]
1037 | },
1038 | {
1039 | "cell_type": "code",
1040 | "execution_count": null,
1041 | "id": "0088c95c-885d-4846-b357-b298f8ba4641",
1042 | "metadata": {
1043 | "scrolled": true
1044 | },
1045 | "outputs": [],
1046 | "source": [
1047 | "## process chips\n",
1048 | "failed_tiles = []\n",
1049 | "\n",
1050 | "for tile in tiles_to_chip:\n",
1051 | " print(tile)\n",
1052 | " chips_to_process = chip_df[chip_df.tile == tile[1:]].reset_index(drop = True)\n",
1053 | " for k in range(len(chips_to_process)):\n",
1054 | " current_id = chips_to_process.chip_id[k]\n",
1055 | " chip_tile = chips_to_process.tile[k]\n",
1056 | " chip_index = chip_ids.index(current_id)\n",
1057 | "\n",
1058 | " chip_feature = chipping_js['features'][chip_index]\n",
1059 | "\n",
1060 | " shape = [chip_feature['geometry']]\n",
1061 | " full_tile_name = \"T\" + chip_tile\n",
1062 | " try:\n",
1063 | " valid_first, valid_second, valid_third, bad_pct_first, bad_pct_second, bad_pct_third, na_count, first_image_date, second_image_date, third_image_date = process_chip(current_id, full_tile_name, shape, track_df, cdl_file)\n",
1064 | " except:\n",
1065 | " failed_tiles.append(tile)\n",
1066 | " break\n",
1067 | "\n",
1068 | " chip_df_index = chip_df.index[chip_df['chip_id'] == current_id].tolist()[0]\n",
1069 | " chip_df.at[chip_df_index, 'valid_first'] = valid_first\n",
1070 | " chip_df.at[chip_df_index, 'valid_second'] = valid_second\n",
1071 | " chip_df.at[chip_df_index, 'valid_third'] = valid_third\n",
1072 | " chip_df.at[chip_df_index, 'bad_pct_first'] = bad_pct_first\n",
1073 | " chip_df.at[chip_df_index, 'bad_pct_second'] = bad_pct_second\n",
1074 | " chip_df.at[chip_df_index, 'bad_pct_third'] = bad_pct_third\n",
1075 | " chip_df.at[chip_df_index, 'first_image_date'] = first_image_date\n",
1076 | " chip_df.at[chip_df_index, 'second_image_date'] = second_image_date\n",
1077 | " chip_df.at[chip_df_index, 'third_image_date'] = third_image_date\n",
1078 | " chip_df['bad_pct_max'] = chip_df[['bad_pct_first', 'bad_pct_second', 'bad_pct_third']].max(axis=1)\n",
1079 | " chip_df.at[chip_df_index, 'na_count'] = na_count\n",
1080 | "chip_df.to_csv(chip_csv, index=False)"
1081 | ]
1082 | },
1083 | {
1084 | "cell_type": "code",
1085 | "execution_count": null,
1086 | "id": "92c193dd-4761-4c26-b440-c214fbf79a3a",
1087 | "metadata": {},
1088 | "outputs": [],
1089 | "source": [
1090 | "chip_df.head()"
1091 | ]
1092 | },
1093 | {
1094 | "cell_type": "markdown",
1095 | "id": "fdfb667c-d9e4-446a-9149-3b7990e60753",
1096 | "metadata": {},
1097 | "source": [
1098 | "### Filtering Chips"
1099 | ]
1100 | },
1101 | {
1102 | "cell_type": "code",
1103 | "execution_count": null,
1104 | "id": "8960db6e-d89f-4da7-a878-ff05824f4bab",
1105 | "metadata": {},
1106 | "outputs": [],
1107 | "source": [
1108 | "selected_tiles = pd.read_csv(req_path + \"selected_tiles.csv\")\n",
1109 | "tiles_to_filter = selected_tiles.tile_id.unique()"
1110 | ]
1111 | },
1112 | {
1113 | "cell_type": "code",
1114 | "execution_count": null,
1115 | "id": "0029c420-5b71-4600-9838-c0b878835fe4",
1116 | "metadata": {
1117 | "scrolled": true
1118 | },
1119 | "outputs": [],
1120 | "source": [
1121 | "chip_df = pd.read_csv(chip_csv)\n",
1122 | "\n",
1123 | "for tile in tiles_to_filter:\n",
1124 | " print(tile)\n",
1125 | " filtered_chips = chip_df[(chip_df.tile == tile[1:]) & (chip_df.bad_pct_max < 5) & (chip_df.na_count == 0)].chip_id.tolist()\n",
1126 | " for chip_id in filtered_chips:\n",
1127 | " chip_files = glob('/data/chips/*' + chip_id + '*')\n",
1128 | " for file in chip_files:\n",
1129 | " name = file.split('/')[-1]\n",
1130 | " shutil.copyfile(file, chip_dir_filt + name)"
1131 | ]
1132 | },
1133 | {
1134 | "cell_type": "code",
1135 | "execution_count": null,
1136 | "id": "220ddeb4-abb5-4c2f-a7c4-af082898950f",
1137 | "metadata": {},
1138 | "outputs": [],
1139 | "source": []
1140 | }
1141 | ],
1142 | "metadata": {
1143 | "kernelspec": {
1144 | "display_name": "Python 3 (ipykernel)",
1145 | "language": "python",
1146 | "name": "python3"
1147 | },
1148 | "language_info": {
1149 | "codemirror_mode": {
1150 | "name": "ipython",
1151 | "version": 3
1152 | },
1153 | "file_extension": ".py",
1154 | "mimetype": "text/x-python",
1155 | "name": "python",
1156 | "nbconvert_exporter": "python",
1157 | "pygments_lexer": "ipython3",
1158 | "version": "3.10.6"
1159 | }
1160 | },
1161 | "nbformat": 4,
1162 | "nbformat_minor": 5
1163 | }
1164 |
--------------------------------------------------------------------------------