├── .editorconfig
├── .github
└── workflows
│ ├── release.yml
│ └── test.yml
├── .gitignore
├── Cargo.lock
├── Cargo.toml
├── Dockerfile
├── LICENSE
├── NOTES.md
├── README.md
├── data
└── schema.sql
├── docker-compose.yml
├── docker-entrypoint.sh
├── src
├── cli.rs
├── context.rs
├── downloader.rs
├── loader
│ ├── admini_boundary.rs
│ ├── gdal.rs
│ ├── load_queue.rs
│ ├── mapping.rs
│ ├── mod.rs
│ ├── xslx_helpers.rs
│ └── zip_traversal.rs
├── main.rs
├── metadata.rs
└── scraper
│ ├── data_page.rs
│ ├── download_queue.rs
│ ├── initial.rs
│ ├── mod.rs
│ └── table_read.rs
├── test_data
├── shp
│ ├── cp932.dbf
│ ├── cp932.shp
│ ├── cp932.shx
│ ├── src_blank.dbf
│ ├── src_blank.prj
│ ├── src_blank.shp
│ └── src_blank.shx
└── zip
│ ├── A30a5-11_4939-jgd_GML.zip
│ └── P23-12_38_GML.zip
└── tmp
└── .gitkeep
/.editorconfig:
--------------------------------------------------------------------------------
1 | root = true
2 |
3 | [*]
4 | indent_style = space
5 | indent_size = 4
6 | end_of_line = lf
7 | charset = utf-8
8 | trim_trailing_whitespace = true
9 | insert_final_newline = true
10 |
11 | [.github/**/*.yml]
12 | indent_size = 2
13 |
--------------------------------------------------------------------------------
/.github/workflows/release.yml:
--------------------------------------------------------------------------------
1 | name: Build and Release
2 |
3 | permissions:
4 | contents: write
5 |
6 | on:
7 | push:
8 | tags:
9 | - v*
10 |
11 | jobs:
12 | build:
13 | strategy:
14 | matrix:
15 | job:
16 | - os: ubuntu-latest
17 | arch: x86_64-unknown-linux-gnu
18 | - os: macos-latest
19 | arch: aarch64-apple-darwin
20 | - os: windows-latest
21 | arch: x86_64-pc-windows-msvc
22 |
23 | runs-on: ${{ matrix.job.os }}
24 |
25 | steps:
26 | - name: Checkout code
27 | uses: actions/checkout@v4
28 | - name: Set up Rust
29 | uses: actions-rust-lang/setup-rust-toolchain@v1
30 | with:
31 | toolchain: stable
32 | target: ${{ matrix.job.arch }}
33 | - name: Build
34 | run: cargo build --release --target ${{ matrix.job.arch }}
35 |
36 | - name: Create archive
37 | if: matrix.job.os == 'ubuntu-latest' || matrix.job.os == 'macos-latest'
38 | run: |
39 | mkdir -p target/artifacts
40 | cp target/${{ matrix.job.arch }}/release/jpksj-to-sql target/artifacts/
41 | cd target/artifacts
42 | zip jpksj-to-sql-${{ matrix.job.arch }}.zip jpksj-to-sql
43 |
44 | - name: Create archive (Windows)
45 | if: matrix.job.os == 'windows-latest'
46 | run: |
47 | New-Item -ItemType Directory -Force -Path target/artifacts
48 | cp target/${{ matrix.job.arch }}/release/jpksj-to-sql.exe target/artifacts/
49 | cd target/artifacts
50 | 7z a jpksj-to-sql-${{ matrix.job.arch }}.zip jpksj-to-sql.exe
51 |
52 | - name: Upload artifact
53 | uses: actions/upload-artifact@v4
54 | with:
55 | name: ${{ matrix.job.arch }}-release
56 | path: target/artifacts/jpksj-to-sql-${{ matrix.job.arch }}.zip
57 |
58 | release:
59 | needs: build
60 | runs-on: ubuntu-latest
61 | steps:
62 | - name: Download artifact for x86_64-unknown-linux-gnu
63 | uses: actions/download-artifact@v4
64 | with:
65 | name: x86_64-unknown-linux-gnu-release
66 | path: artifacts/x86_64
67 |
68 | - name: Download artifact for x86_64-pc-windows-msvc
69 | uses: actions/download-artifact@v4
70 | with:
71 | name: x86_64-pc-windows-msvc-release
72 | path: artifacts/x86_64
73 |
74 | - name: Download artifact for aarch64-apple-darwin
75 | uses: actions/download-artifact@v4
76 | with:
77 | name: aarch64-apple-darwin-release
78 | path: artifacts/aarch64
79 |
80 | - name: Create Release
81 | uses: softprops/action-gh-release@v2
82 | with:
83 | # Use the tag name without the refs/ prefix
84 | tag_name: ${{ github.ref_name }}
85 | files: |
86 | artifacts/x86_64/jpksj-to-sql-x86_64-unknown-linux-gnu.zip
87 | artifacts/x86_64/jpksj-to-sql-x86_64-pc-windows-msvc.zip
88 | artifacts/aarch64/jpksj-to-sql-aarch64-apple-darwin.zip
89 |
--------------------------------------------------------------------------------
/.github/workflows/test.yml:
--------------------------------------------------------------------------------
1 | name: Test
2 |
3 | permissions:
4 | contents: write
5 |
6 | on:
7 | pull_request:
8 | branches:
9 | - main
10 | push:
11 | branches:
12 | - main
13 |
14 | jobs:
15 | build:
16 | strategy:
17 | matrix:
18 | job:
19 | - os: ubuntu-latest
20 | arch: x86_64-unknown-linux-gnu
21 |
22 | # TODO: Tests on Windows requires gdal, but I don't know how to install it yet...
23 | # - os: windows-latest
24 | # arch: x86_64-pc-windows-msvc
25 |
26 | runs-on: ${{ matrix.job.os }}
27 |
28 | steps:
29 | - name: Checkout code
30 | uses: actions/checkout@v4
31 | - name: Set up Rust
32 | uses: actions-rust-lang/setup-rust-toolchain@v1
33 | with:
34 | toolchain: stable
35 | target: ${{ matrix.job.arch }}
36 |
37 | - name: Install dependencies
38 | if: matrix.job.os == 'ubuntu-latest'
39 | run: |
40 | sudo add-apt-repository ppa:ubuntugis/ubuntugis-unstable -y
41 | sudo apt-get install -y gdal-bin
42 | ogrinfo --version
43 |
44 | - name: Cache temporary files
45 | uses: actions/cache@v4
46 | with:
47 | key: v1-temporary-files-${{ hashFiles('src/**/*.rs') }}
48 | restore-keys: |
49 | v1-temporary-files-
50 | path: tmp
51 |
52 | - name: Build
53 | run: cargo build --release --target ${{ matrix.job.arch }}
54 |
55 | - name: Run test
56 | run: cargo test
57 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | /target
2 | /tmp/*
3 | !/tmp/.gitkeep
4 |
--------------------------------------------------------------------------------
/Cargo.lock:
--------------------------------------------------------------------------------
1 | # This file is automatically @generated by Cargo.
2 | # It is not intended for manual editing.
3 | version = 4
4 |
5 | [[package]]
6 | name = "addr2line"
7 | version = "0.24.2"
8 | source = "registry+https://github.com/rust-lang/crates.io-index"
9 | checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1"
10 | dependencies = [
11 | "gimli",
12 | ]
13 |
14 | [[package]]
15 | name = "adler2"
16 | version = "2.0.0"
17 | source = "registry+https://github.com/rust-lang/crates.io-index"
18 | checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627"
19 |
20 | [[package]]
21 | name = "aes"
22 | version = "0.8.4"
23 | source = "registry+https://github.com/rust-lang/crates.io-index"
24 | checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0"
25 | dependencies = [
26 | "cfg-if",
27 | "cipher",
28 | "cpufeatures",
29 | ]
30 |
31 | [[package]]
32 | name = "aho-corasick"
33 | version = "1.1.3"
34 | source = "registry+https://github.com/rust-lang/crates.io-index"
35 | checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
36 | dependencies = [
37 | "memchr",
38 | ]
39 |
40 | [[package]]
41 | name = "anstream"
42 | version = "0.6.18"
43 | source = "registry+https://github.com/rust-lang/crates.io-index"
44 | checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b"
45 | dependencies = [
46 | "anstyle",
47 | "anstyle-parse",
48 | "anstyle-query",
49 | "anstyle-wincon",
50 | "colorchoice",
51 | "is_terminal_polyfill",
52 | "utf8parse",
53 | ]
54 |
55 | [[package]]
56 | name = "anstyle"
57 | version = "1.0.10"
58 | source = "registry+https://github.com/rust-lang/crates.io-index"
59 | checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
60 |
61 | [[package]]
62 | name = "anstyle-parse"
63 | version = "0.2.6"
64 | source = "registry+https://github.com/rust-lang/crates.io-index"
65 | checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9"
66 | dependencies = [
67 | "utf8parse",
68 | ]
69 |
70 | [[package]]
71 | name = "anstyle-query"
72 | version = "1.1.2"
73 | source = "registry+https://github.com/rust-lang/crates.io-index"
74 | checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c"
75 | dependencies = [
76 | "windows-sys 0.59.0",
77 | ]
78 |
79 | [[package]]
80 | name = "anstyle-wincon"
81 | version = "3.0.7"
82 | source = "registry+https://github.com/rust-lang/crates.io-index"
83 | checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e"
84 | dependencies = [
85 | "anstyle",
86 | "once_cell",
87 | "windows-sys 0.59.0",
88 | ]
89 |
90 | [[package]]
91 | name = "anyhow"
92 | version = "1.0.97"
93 | source = "registry+https://github.com/rust-lang/crates.io-index"
94 | checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
95 |
96 | [[package]]
97 | name = "approx"
98 | version = "0.5.1"
99 | source = "registry+https://github.com/rust-lang/crates.io-index"
100 | checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6"
101 | dependencies = [
102 | "num-traits",
103 | ]
104 |
105 | [[package]]
106 | name = "arbitrary"
107 | version = "1.4.1"
108 | source = "registry+https://github.com/rust-lang/crates.io-index"
109 | checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223"
110 | dependencies = [
111 | "derive_arbitrary",
112 | ]
113 |
114 | [[package]]
115 | name = "async-channel"
116 | version = "2.3.1"
117 | source = "registry+https://github.com/rust-lang/crates.io-index"
118 | checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a"
119 | dependencies = [
120 | "concurrent-queue",
121 | "event-listener-strategy",
122 | "futures-core",
123 | "pin-project-lite",
124 | ]
125 |
126 | [[package]]
127 | name = "async-trait"
128 | version = "0.1.88"
129 | source = "registry+https://github.com/rust-lang/crates.io-index"
130 | checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"
131 | dependencies = [
132 | "proc-macro2",
133 | "quote",
134 | "syn",
135 | ]
136 |
137 | [[package]]
138 | name = "atomic-waker"
139 | version = "1.1.2"
140 | source = "registry+https://github.com/rust-lang/crates.io-index"
141 | checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
142 |
143 | [[package]]
144 | name = "autocfg"
145 | version = "1.4.0"
146 | source = "registry+https://github.com/rust-lang/crates.io-index"
147 | checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
148 |
149 | [[package]]
150 | name = "backtrace"
151 | version = "0.3.74"
152 | source = "registry+https://github.com/rust-lang/crates.io-index"
153 | checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a"
154 | dependencies = [
155 | "addr2line",
156 | "cfg-if",
157 | "libc",
158 | "miniz_oxide",
159 | "object",
160 | "rustc-demangle",
161 | "windows-targets 0.52.6",
162 | ]
163 |
164 | [[package]]
165 | name = "base64"
166 | version = "0.22.1"
167 | source = "registry+https://github.com/rust-lang/crates.io-index"
168 | checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
169 |
170 | [[package]]
171 | name = "bitflags"
172 | version = "2.9.0"
173 | source = "registry+https://github.com/rust-lang/crates.io-index"
174 | checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
175 |
176 | [[package]]
177 | name = "block-buffer"
178 | version = "0.10.4"
179 | source = "registry+https://github.com/rust-lang/crates.io-index"
180 | checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
181 | dependencies = [
182 | "generic-array",
183 | ]
184 |
185 | [[package]]
186 | name = "bumpalo"
187 | version = "3.17.0"
188 | source = "registry+https://github.com/rust-lang/crates.io-index"
189 | checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
190 |
191 | [[package]]
192 | name = "byteorder"
193 | version = "1.5.0"
194 | source = "registry+https://github.com/rust-lang/crates.io-index"
195 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
196 |
197 | [[package]]
198 | name = "bytes"
199 | version = "1.10.1"
200 | source = "registry+https://github.com/rust-lang/crates.io-index"
201 | checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
202 |
203 | [[package]]
204 | name = "bytesize"
205 | version = "1.3.2"
206 | source = "registry+https://github.com/rust-lang/crates.io-index"
207 | checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e"
208 |
209 | [[package]]
210 | name = "calamine"
211 | version = "0.26.1"
212 | source = "registry+https://github.com/rust-lang/crates.io-index"
213 | checksum = "138646b9af2c5d7f1804ea4bf93afc597737d2bd4f7341d67c48b03316976eb1"
214 | dependencies = [
215 | "byteorder",
216 | "codepage",
217 | "encoding_rs",
218 | "log",
219 | "quick-xml",
220 | "serde",
221 | "zip",
222 | ]
223 |
224 | [[package]]
225 | name = "cc"
226 | version = "1.2.16"
227 | source = "registry+https://github.com/rust-lang/crates.io-index"
228 | checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
229 | dependencies = [
230 | "shlex",
231 | ]
232 |
233 | [[package]]
234 | name = "cfg-if"
235 | version = "1.0.0"
236 | source = "registry+https://github.com/rust-lang/crates.io-index"
237 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
238 |
239 | [[package]]
240 | name = "cipher"
241 | version = "0.4.4"
242 | source = "registry+https://github.com/rust-lang/crates.io-index"
243 | checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
244 | dependencies = [
245 | "crypto-common",
246 | "inout",
247 | ]
248 |
249 | [[package]]
250 | name = "clap"
251 | version = "4.5.32"
252 | source = "registry+https://github.com/rust-lang/crates.io-index"
253 | checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83"
254 | dependencies = [
255 | "clap_builder",
256 | "clap_derive",
257 | ]
258 |
259 | [[package]]
260 | name = "clap_builder"
261 | version = "4.5.32"
262 | source = "registry+https://github.com/rust-lang/crates.io-index"
263 | checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8"
264 | dependencies = [
265 | "anstream",
266 | "anstyle",
267 | "clap_lex",
268 | "strsim",
269 | ]
270 |
271 | [[package]]
272 | name = "clap_derive"
273 | version = "4.5.32"
274 | source = "registry+https://github.com/rust-lang/crates.io-index"
275 | checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7"
276 | dependencies = [
277 | "heck",
278 | "proc-macro2",
279 | "quote",
280 | "syn",
281 | ]
282 |
283 | [[package]]
284 | name = "clap_lex"
285 | version = "0.7.4"
286 | source = "registry+https://github.com/rust-lang/crates.io-index"
287 | checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
288 |
289 | [[package]]
290 | name = "codepage"
291 | version = "0.1.2"
292 | source = "registry+https://github.com/rust-lang/crates.io-index"
293 | checksum = "48f68d061bc2828ae826206326e61251aca94c1e4a5305cf52d9138639c918b4"
294 | dependencies = [
295 | "encoding_rs",
296 | ]
297 |
298 | [[package]]
299 | name = "colorchoice"
300 | version = "1.0.3"
301 | source = "registry+https://github.com/rust-lang/crates.io-index"
302 | checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990"
303 |
304 | [[package]]
305 | name = "concurrent-queue"
306 | version = "2.5.0"
307 | source = "registry+https://github.com/rust-lang/crates.io-index"
308 | checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"
309 | dependencies = [
310 | "crossbeam-utils",
311 | ]
312 |
313 | [[package]]
314 | name = "console"
315 | version = "0.15.11"
316 | source = "registry+https://github.com/rust-lang/crates.io-index"
317 | checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
318 | dependencies = [
319 | "encode_unicode",
320 | "libc",
321 | "once_cell",
322 | "unicode-width 0.2.0",
323 | "windows-sys 0.59.0",
324 | ]
325 |
326 | [[package]]
327 | name = "constant_time_eq"
328 | version = "0.3.1"
329 | source = "registry+https://github.com/rust-lang/crates.io-index"
330 | checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6"
331 |
332 | [[package]]
333 | name = "core-foundation"
334 | version = "0.9.4"
335 | source = "registry+https://github.com/rust-lang/crates.io-index"
336 | checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
337 | dependencies = [
338 | "core-foundation-sys",
339 | "libc",
340 | ]
341 |
342 | [[package]]
343 | name = "core-foundation-sys"
344 | version = "0.8.7"
345 | source = "registry+https://github.com/rust-lang/crates.io-index"
346 | checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
347 |
348 | [[package]]
349 | name = "cpufeatures"
350 | version = "0.2.17"
351 | source = "registry+https://github.com/rust-lang/crates.io-index"
352 | checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
353 | dependencies = [
354 | "libc",
355 | ]
356 |
357 | [[package]]
358 | name = "crc"
359 | version = "3.2.1"
360 | source = "registry+https://github.com/rust-lang/crates.io-index"
361 | checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636"
362 | dependencies = [
363 | "crc-catalog",
364 | ]
365 |
366 | [[package]]
367 | name = "crc-catalog"
368 | version = "2.4.0"
369 | source = "registry+https://github.com/rust-lang/crates.io-index"
370 | checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
371 |
372 | [[package]]
373 | name = "crc32fast"
374 | version = "1.4.2"
375 | source = "registry+https://github.com/rust-lang/crates.io-index"
376 | checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
377 | dependencies = [
378 | "cfg-if",
379 | ]
380 |
381 | [[package]]
382 | name = "crossbeam-utils"
383 | version = "0.8.21"
384 | source = "registry+https://github.com/rust-lang/crates.io-index"
385 | checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
386 |
387 | [[package]]
388 | name = "crypto-common"
389 | version = "0.1.6"
390 | source = "registry+https://github.com/rust-lang/crates.io-index"
391 | checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
392 | dependencies = [
393 | "generic-array",
394 | "typenum",
395 | ]
396 |
397 | [[package]]
398 | name = "cssparser"
399 | version = "0.34.0"
400 | source = "registry+https://github.com/rust-lang/crates.io-index"
401 | checksum = "b7c66d1cd8ed61bf80b38432613a7a2f09401ab8d0501110655f8b341484a3e3"
402 | dependencies = [
403 | "cssparser-macros",
404 | "dtoa-short",
405 | "itoa",
406 | "phf",
407 | "smallvec",
408 | ]
409 |
410 | [[package]]
411 | name = "cssparser-macros"
412 | version = "0.6.1"
413 | source = "registry+https://github.com/rust-lang/crates.io-index"
414 | checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
415 | dependencies = [
416 | "quote",
417 | "syn",
418 | ]
419 |
420 | [[package]]
421 | name = "darling"
422 | version = "0.20.10"
423 | source = "registry+https://github.com/rust-lang/crates.io-index"
424 | checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989"
425 | dependencies = [
426 | "darling_core",
427 | "darling_macro",
428 | ]
429 |
430 | [[package]]
431 | name = "darling_core"
432 | version = "0.20.10"
433 | source = "registry+https://github.com/rust-lang/crates.io-index"
434 | checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5"
435 | dependencies = [
436 | "fnv",
437 | "ident_case",
438 | "proc-macro2",
439 | "quote",
440 | "strsim",
441 | "syn",
442 | ]
443 |
444 | [[package]]
445 | name = "darling_macro"
446 | version = "0.20.10"
447 | source = "registry+https://github.com/rust-lang/crates.io-index"
448 | checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806"
449 | dependencies = [
450 | "darling_core",
451 | "quote",
452 | "syn",
453 | ]
454 |
455 | [[package]]
456 | name = "deflate64"
457 | version = "0.1.9"
458 | source = "registry+https://github.com/rust-lang/crates.io-index"
459 | checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b"
460 |
461 | [[package]]
462 | name = "deranged"
463 | version = "0.3.11"
464 | source = "registry+https://github.com/rust-lang/crates.io-index"
465 | checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
466 | dependencies = [
467 | "powerfmt",
468 | ]
469 |
470 | [[package]]
471 | name = "derive_arbitrary"
472 | version = "1.4.1"
473 | source = "registry+https://github.com/rust-lang/crates.io-index"
474 | checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800"
475 | dependencies = [
476 | "proc-macro2",
477 | "quote",
478 | "syn",
479 | ]
480 |
481 | [[package]]
482 | name = "derive_builder"
483 | version = "0.20.2"
484 | source = "registry+https://github.com/rust-lang/crates.io-index"
485 | checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
486 | dependencies = [
487 | "derive_builder_macro",
488 | ]
489 |
490 | [[package]]
491 | name = "derive_builder_core"
492 | version = "0.20.2"
493 | source = "registry+https://github.com/rust-lang/crates.io-index"
494 | checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
495 | dependencies = [
496 | "darling",
497 | "proc-macro2",
498 | "quote",
499 | "syn",
500 | ]
501 |
502 | [[package]]
503 | name = "derive_builder_macro"
504 | version = "0.20.2"
505 | source = "registry+https://github.com/rust-lang/crates.io-index"
506 | checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
507 | dependencies = [
508 | "derive_builder_core",
509 | "syn",
510 | ]
511 |
512 | [[package]]
513 | name = "derive_more"
514 | version = "0.99.19"
515 | source = "registry+https://github.com/rust-lang/crates.io-index"
516 | checksum = "3da29a38df43d6f156149c9b43ded5e018ddff2a855cf2cfd62e8cd7d079c69f"
517 | dependencies = [
518 | "proc-macro2",
519 | "quote",
520 | "syn",
521 | ]
522 |
523 | [[package]]
524 | name = "digest"
525 | version = "0.10.7"
526 | source = "registry+https://github.com/rust-lang/crates.io-index"
527 | checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
528 | dependencies = [
529 | "block-buffer",
530 | "crypto-common",
531 | "subtle",
532 | ]
533 |
534 | [[package]]
535 | name = "displaydoc"
536 | version = "0.2.5"
537 | source = "registry+https://github.com/rust-lang/crates.io-index"
538 | checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
539 | dependencies = [
540 | "proc-macro2",
541 | "quote",
542 | "syn",
543 | ]
544 |
545 | [[package]]
546 | name = "dtoa"
547 | version = "1.0.10"
548 | source = "registry+https://github.com/rust-lang/crates.io-index"
549 | checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04"
550 |
551 | [[package]]
552 | name = "dtoa-short"
553 | version = "0.3.5"
554 | source = "registry+https://github.com/rust-lang/crates.io-index"
555 | checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
556 | dependencies = [
557 | "dtoa",
558 | ]
559 |
560 | [[package]]
561 | name = "ego-tree"
562 | version = "0.10.0"
563 | source = "registry+https://github.com/rust-lang/crates.io-index"
564 | checksum = "b2972feb8dffe7bc8c5463b1dacda1b0dfbed3710e50f977d965429692d74cd8"
565 |
566 | [[package]]
567 | name = "encode_unicode"
568 | version = "1.0.0"
569 | source = "registry+https://github.com/rust-lang/crates.io-index"
570 | checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
571 |
572 | [[package]]
573 | name = "encoding_rs"
574 | version = "0.8.35"
575 | source = "registry+https://github.com/rust-lang/crates.io-index"
576 | checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
577 | dependencies = [
578 | "cfg-if",
579 | ]
580 |
581 | [[package]]
582 | name = "equivalent"
583 | version = "1.0.2"
584 | source = "registry+https://github.com/rust-lang/crates.io-index"
585 | checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
586 |
587 | [[package]]
588 | name = "errno"
589 | version = "0.3.10"
590 | source = "registry+https://github.com/rust-lang/crates.io-index"
591 | checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d"
592 | dependencies = [
593 | "libc",
594 | "windows-sys 0.59.0",
595 | ]
596 |
597 | [[package]]
598 | name = "event-listener"
599 | version = "5.4.0"
600 | source = "registry+https://github.com/rust-lang/crates.io-index"
601 | checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae"
602 | dependencies = [
603 | "concurrent-queue",
604 | "parking",
605 | "pin-project-lite",
606 | ]
607 |
608 | [[package]]
609 | name = "event-listener-strategy"
610 | version = "0.5.3"
611 | source = "registry+https://github.com/rust-lang/crates.io-index"
612 | checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2"
613 | dependencies = [
614 | "event-listener",
615 | "pin-project-lite",
616 | ]
617 |
618 | [[package]]
619 | name = "fallible-iterator"
620 | version = "0.2.0"
621 | source = "registry+https://github.com/rust-lang/crates.io-index"
622 | checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
623 |
624 | [[package]]
625 | name = "fastrand"
626 | version = "2.3.0"
627 | source = "registry+https://github.com/rust-lang/crates.io-index"
628 | checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
629 |
630 | [[package]]
631 | name = "flate2"
632 | version = "1.1.0"
633 | source = "registry+https://github.com/rust-lang/crates.io-index"
634 | checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
635 | dependencies = [
636 | "crc32fast",
637 | "miniz_oxide",
638 | ]
639 |
640 | [[package]]
641 | name = "fnv"
642 | version = "1.0.7"
643 | source = "registry+https://github.com/rust-lang/crates.io-index"
644 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
645 |
646 | [[package]]
647 | name = "foreign-types"
648 | version = "0.3.2"
649 | source = "registry+https://github.com/rust-lang/crates.io-index"
650 | checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
651 | dependencies = [
652 | "foreign-types-shared",
653 | ]
654 |
655 | [[package]]
656 | name = "foreign-types-shared"
657 | version = "0.1.1"
658 | source = "registry+https://github.com/rust-lang/crates.io-index"
659 | checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
660 |
661 | [[package]]
662 | name = "form_urlencoded"
663 | version = "1.2.1"
664 | source = "registry+https://github.com/rust-lang/crates.io-index"
665 | checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
666 | dependencies = [
667 | "percent-encoding",
668 | ]
669 |
670 | [[package]]
671 | name = "futf"
672 | version = "0.1.5"
673 | source = "registry+https://github.com/rust-lang/crates.io-index"
674 | checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
675 | dependencies = [
676 | "mac",
677 | "new_debug_unreachable",
678 | ]
679 |
680 | [[package]]
681 | name = "futures-channel"
682 | version = "0.3.31"
683 | source = "registry+https://github.com/rust-lang/crates.io-index"
684 | checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
685 | dependencies = [
686 | "futures-core",
687 | "futures-sink",
688 | ]
689 |
690 | [[package]]
691 | name = "futures-core"
692 | version = "0.3.31"
693 | source = "registry+https://github.com/rust-lang/crates.io-index"
694 | checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
695 |
696 | [[package]]
697 | name = "futures-io"
698 | version = "0.3.31"
699 | source = "registry+https://github.com/rust-lang/crates.io-index"
700 | checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
701 |
702 | [[package]]
703 | name = "futures-macro"
704 | version = "0.3.31"
705 | source = "registry+https://github.com/rust-lang/crates.io-index"
706 | checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
707 | dependencies = [
708 | "proc-macro2",
709 | "quote",
710 | "syn",
711 | ]
712 |
713 | [[package]]
714 | name = "futures-sink"
715 | version = "0.3.31"
716 | source = "registry+https://github.com/rust-lang/crates.io-index"
717 | checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
718 |
719 | [[package]]
720 | name = "futures-task"
721 | version = "0.3.31"
722 | source = "registry+https://github.com/rust-lang/crates.io-index"
723 | checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
724 |
725 | [[package]]
726 | name = "futures-util"
727 | version = "0.3.31"
728 | source = "registry+https://github.com/rust-lang/crates.io-index"
729 | checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
730 | dependencies = [
731 | "futures-core",
732 | "futures-io",
733 | "futures-macro",
734 | "futures-sink",
735 | "futures-task",
736 | "memchr",
737 | "pin-project-lite",
738 | "pin-utils",
739 | "slab",
740 | ]
741 |
742 | [[package]]
743 | name = "fxhash"
744 | version = "0.2.1"
745 | source = "registry+https://github.com/rust-lang/crates.io-index"
746 | checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
747 | dependencies = [
748 | "byteorder",
749 | ]
750 |
751 | [[package]]
752 | name = "generic-array"
753 | version = "0.14.7"
754 | source = "registry+https://github.com/rust-lang/crates.io-index"
755 | checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
756 | dependencies = [
757 | "typenum",
758 | "version_check",
759 | ]
760 |
761 | [[package]]
762 | name = "geo-types"
763 | version = "0.7.15"
764 | source = "registry+https://github.com/rust-lang/crates.io-index"
765 | checksum = "3bd1157f0f936bf0cd68dec91e8f7c311afe60295574d62b70d4861a1bfdf2d9"
766 | dependencies = [
767 | "approx",
768 | "num-traits",
769 | "serde",
770 | ]
771 |
772 | [[package]]
773 | name = "getopts"
774 | version = "0.2.21"
775 | source = "registry+https://github.com/rust-lang/crates.io-index"
776 | checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
777 | dependencies = [
778 | "unicode-width 0.1.14",
779 | ]
780 |
781 | [[package]]
782 | name = "getrandom"
783 | version = "0.2.15"
784 | source = "registry+https://github.com/rust-lang/crates.io-index"
785 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
786 | dependencies = [
787 | "cfg-if",
788 | "libc",
789 | "wasi 0.11.0+wasi-snapshot-preview1",
790 | ]
791 |
792 | [[package]]
793 | name = "getrandom"
794 | version = "0.3.2"
795 | source = "registry+https://github.com/rust-lang/crates.io-index"
796 | checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
797 | dependencies = [
798 | "cfg-if",
799 | "js-sys",
800 | "libc",
801 | "r-efi",
802 | "wasi 0.14.2+wasi-0.2.4",
803 | "wasm-bindgen",
804 | ]
805 |
806 | [[package]]
807 | name = "gimli"
808 | version = "0.31.1"
809 | source = "registry+https://github.com/rust-lang/crates.io-index"
810 | checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
811 |
812 | [[package]]
813 | name = "h2"
814 | version = "0.4.8"
815 | source = "registry+https://github.com/rust-lang/crates.io-index"
816 | checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2"
817 | dependencies = [
818 | "atomic-waker",
819 | "bytes",
820 | "fnv",
821 | "futures-core",
822 | "futures-sink",
823 | "http",
824 | "indexmap",
825 | "slab",
826 | "tokio",
827 | "tokio-util",
828 | "tracing",
829 | ]
830 |
831 | [[package]]
832 | name = "hashbrown"
833 | version = "0.15.2"
834 | source = "registry+https://github.com/rust-lang/crates.io-index"
835 | checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
836 |
837 | [[package]]
838 | name = "heck"
839 | version = "0.5.0"
840 | source = "registry+https://github.com/rust-lang/crates.io-index"
841 | checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
842 |
843 | [[package]]
844 | name = "hermit-abi"
845 | version = "0.3.9"
846 | source = "registry+https://github.com/rust-lang/crates.io-index"
847 | checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
848 |
849 | [[package]]
850 | name = "hmac"
851 | version = "0.12.1"
852 | source = "registry+https://github.com/rust-lang/crates.io-index"
853 | checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
854 | dependencies = [
855 | "digest",
856 | ]
857 |
858 | [[package]]
859 | name = "html5ever"
860 | version = "0.29.1"
861 | source = "registry+https://github.com/rust-lang/crates.io-index"
862 | checksum = "3b7410cae13cbc75623c98ac4cbfd1f0bedddf3227afc24f370cf0f50a44a11c"
863 | dependencies = [
864 | "log",
865 | "mac",
866 | "markup5ever",
867 | "match_token",
868 | ]
869 |
870 | [[package]]
871 | name = "http"
872 | version = "1.3.1"
873 | source = "registry+https://github.com/rust-lang/crates.io-index"
874 | checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565"
875 | dependencies = [
876 | "bytes",
877 | "fnv",
878 | "itoa",
879 | ]
880 |
881 | [[package]]
882 | name = "http-body"
883 | version = "1.0.1"
884 | source = "registry+https://github.com/rust-lang/crates.io-index"
885 | checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
886 | dependencies = [
887 | "bytes",
888 | "http",
889 | ]
890 |
891 | [[package]]
892 | name = "http-body-util"
893 | version = "0.1.3"
894 | source = "registry+https://github.com/rust-lang/crates.io-index"
895 | checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
896 | dependencies = [
897 | "bytes",
898 | "futures-core",
899 | "http",
900 | "http-body",
901 | "pin-project-lite",
902 | ]
903 |
904 | [[package]]
905 | name = "httparse"
906 | version = "1.10.1"
907 | source = "registry+https://github.com/rust-lang/crates.io-index"
908 | checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
909 |
910 | [[package]]
911 | name = "hyper"
912 | version = "1.6.0"
913 | source = "registry+https://github.com/rust-lang/crates.io-index"
914 | checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
915 | dependencies = [
916 | "bytes",
917 | "futures-channel",
918 | "futures-util",
919 | "h2",
920 | "http",
921 | "http-body",
922 | "httparse",
923 | "itoa",
924 | "pin-project-lite",
925 | "smallvec",
926 | "tokio",
927 | "want",
928 | ]
929 |
930 | [[package]]
931 | name = "hyper-rustls"
932 | version = "0.27.5"
933 | source = "registry+https://github.com/rust-lang/crates.io-index"
934 | checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
935 | dependencies = [
936 | "futures-util",
937 | "http",
938 | "hyper",
939 | "hyper-util",
940 | "rustls",
941 | "rustls-pki-types",
942 | "tokio",
943 | "tokio-rustls",
944 | "tower-service",
945 | ]
946 |
947 | [[package]]
948 | name = "hyper-tls"
949 | version = "0.6.0"
950 | source = "registry+https://github.com/rust-lang/crates.io-index"
951 | checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
952 | dependencies = [
953 | "bytes",
954 | "http-body-util",
955 | "hyper",
956 | "hyper-util",
957 | "native-tls",
958 | "tokio",
959 | "tokio-native-tls",
960 | "tower-service",
961 | ]
962 |
963 | [[package]]
964 | name = "hyper-util"
965 | version = "0.1.10"
966 | source = "registry+https://github.com/rust-lang/crates.io-index"
967 | checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
968 | dependencies = [
969 | "bytes",
970 | "futures-channel",
971 | "futures-util",
972 | "http",
973 | "http-body",
974 | "hyper",
975 | "pin-project-lite",
976 | "socket2",
977 | "tokio",
978 | "tower-service",
979 | "tracing",
980 | ]
981 |
982 | [[package]]
983 | name = "icu_collections"
984 | version = "1.5.0"
985 | source = "registry+https://github.com/rust-lang/crates.io-index"
986 | checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
987 | dependencies = [
988 | "displaydoc",
989 | "yoke",
990 | "zerofrom",
991 | "zerovec",
992 | ]
993 |
994 | [[package]]
995 | name = "icu_locid"
996 | version = "1.5.0"
997 | source = "registry+https://github.com/rust-lang/crates.io-index"
998 | checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
999 | dependencies = [
1000 | "displaydoc",
1001 | "litemap",
1002 | "tinystr",
1003 | "writeable",
1004 | "zerovec",
1005 | ]
1006 |
1007 | [[package]]
1008 | name = "icu_locid_transform"
1009 | version = "1.5.0"
1010 | source = "registry+https://github.com/rust-lang/crates.io-index"
1011 | checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
1012 | dependencies = [
1013 | "displaydoc",
1014 | "icu_locid",
1015 | "icu_locid_transform_data",
1016 | "icu_provider",
1017 | "tinystr",
1018 | "zerovec",
1019 | ]
1020 |
1021 | [[package]]
1022 | name = "icu_locid_transform_data"
1023 | version = "1.5.0"
1024 | source = "registry+https://github.com/rust-lang/crates.io-index"
1025 | checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
1026 |
1027 | [[package]]
1028 | name = "icu_normalizer"
1029 | version = "1.5.0"
1030 | source = "registry+https://github.com/rust-lang/crates.io-index"
1031 | checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
1032 | dependencies = [
1033 | "displaydoc",
1034 | "icu_collections",
1035 | "icu_normalizer_data",
1036 | "icu_properties",
1037 | "icu_provider",
1038 | "smallvec",
1039 | "utf16_iter",
1040 | "utf8_iter",
1041 | "write16",
1042 | "zerovec",
1043 | ]
1044 |
1045 | [[package]]
1046 | name = "icu_normalizer_data"
1047 | version = "1.5.0"
1048 | source = "registry+https://github.com/rust-lang/crates.io-index"
1049 | checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
1050 |
1051 | [[package]]
1052 | name = "icu_properties"
1053 | version = "1.5.1"
1054 | source = "registry+https://github.com/rust-lang/crates.io-index"
1055 | checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5"
1056 | dependencies = [
1057 | "displaydoc",
1058 | "icu_collections",
1059 | "icu_locid_transform",
1060 | "icu_properties_data",
1061 | "icu_provider",
1062 | "tinystr",
1063 | "zerovec",
1064 | ]
1065 |
1066 | [[package]]
1067 | name = "icu_properties_data"
1068 | version = "1.5.0"
1069 | source = "registry+https://github.com/rust-lang/crates.io-index"
1070 | checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
1071 |
1072 | [[package]]
1073 | name = "icu_provider"
1074 | version = "1.5.0"
1075 | source = "registry+https://github.com/rust-lang/crates.io-index"
1076 | checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
1077 | dependencies = [
1078 | "displaydoc",
1079 | "icu_locid",
1080 | "icu_provider_macros",
1081 | "stable_deref_trait",
1082 | "tinystr",
1083 | "writeable",
1084 | "yoke",
1085 | "zerofrom",
1086 | "zerovec",
1087 | ]
1088 |
1089 | [[package]]
1090 | name = "icu_provider_macros"
1091 | version = "1.5.0"
1092 | source = "registry+https://github.com/rust-lang/crates.io-index"
1093 | checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
1094 | dependencies = [
1095 | "proc-macro2",
1096 | "quote",
1097 | "syn",
1098 | ]
1099 |
1100 | [[package]]
1101 | name = "ident_case"
1102 | version = "1.0.1"
1103 | source = "registry+https://github.com/rust-lang/crates.io-index"
1104 | checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
1105 |
1106 | [[package]]
1107 | name = "idna"
1108 | version = "1.0.3"
1109 | source = "registry+https://github.com/rust-lang/crates.io-index"
1110 | checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e"
1111 | dependencies = [
1112 | "idna_adapter",
1113 | "smallvec",
1114 | "utf8_iter",
1115 | ]
1116 |
1117 | [[package]]
1118 | name = "idna_adapter"
1119 | version = "1.2.0"
1120 | source = "registry+https://github.com/rust-lang/crates.io-index"
1121 | checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71"
1122 | dependencies = [
1123 | "icu_normalizer",
1124 | "icu_properties",
1125 | ]
1126 |
1127 | [[package]]
1128 | name = "indexmap"
1129 | version = "2.8.0"
1130 | source = "registry+https://github.com/rust-lang/crates.io-index"
1131 | checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058"
1132 | dependencies = [
1133 | "equivalent",
1134 | "hashbrown",
1135 | ]
1136 |
1137 | [[package]]
1138 | name = "indicatif"
1139 | version = "0.17.11"
1140 | source = "registry+https://github.com/rust-lang/crates.io-index"
1141 | checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
1142 | dependencies = [
1143 | "console",
1144 | "number_prefix",
1145 | "portable-atomic",
1146 | "tokio",
1147 | "unicode-width 0.2.0",
1148 | "web-time",
1149 | ]
1150 |
1151 | [[package]]
1152 | name = "inout"
1153 | version = "0.1.4"
1154 | source = "registry+https://github.com/rust-lang/crates.io-index"
1155 | checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
1156 | dependencies = [
1157 | "generic-array",
1158 | ]
1159 |
1160 | [[package]]
1161 | name = "ipnet"
1162 | version = "2.11.0"
1163 | source = "registry+https://github.com/rust-lang/crates.io-index"
1164 | checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
1165 |
1166 | [[package]]
1167 | name = "is_terminal_polyfill"
1168 | version = "1.70.1"
1169 | source = "registry+https://github.com/rust-lang/crates.io-index"
1170 | checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
1171 |
1172 | [[package]]
1173 | name = "itoa"
1174 | version = "1.0.15"
1175 | source = "registry+https://github.com/rust-lang/crates.io-index"
1176 | checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
1177 |
1178 | [[package]]
1179 | name = "jpksj-to-sql"
1180 | version = "0.2.0"
1181 | dependencies = [
1182 | "anyhow",
1183 | "async-channel",
1184 | "bytesize",
1185 | "calamine",
1186 | "clap",
1187 | "derive_builder",
1188 | "encoding_rs",
1189 | "futures-util",
1190 | "geo-types",
1191 | "indicatif",
1192 | "km-to-sql",
1193 | "ndarray",
1194 | "num_cpus",
1195 | "once_cell",
1196 | "regex",
1197 | "reqwest",
1198 | "scraper",
1199 | "serde",
1200 | "serde_json",
1201 | "tokio",
1202 | "tokio-postgres",
1203 | "unicode-normalization",
1204 | "url",
1205 | "zip",
1206 | ]
1207 |
1208 | [[package]]
1209 | name = "js-sys"
1210 | version = "0.3.77"
1211 | source = "registry+https://github.com/rust-lang/crates.io-index"
1212 | checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
1213 | dependencies = [
1214 | "once_cell",
1215 | "wasm-bindgen",
1216 | ]
1217 |
1218 | [[package]]
1219 | name = "km-to-sql"
1220 | version = "0.1.1"
1221 | source = "registry+https://github.com/rust-lang/crates.io-index"
1222 | checksum = "bc79fcfe445d24690aae31f84d9666cec80f929d7359a4cb0a5a7433330bca8a"
1223 | dependencies = [
1224 | "serde",
1225 | "serde_json",
1226 | "thiserror",
1227 | "tokio-postgres",
1228 | "url",
1229 | ]
1230 |
1231 | [[package]]
1232 | name = "libc"
1233 | version = "0.2.171"
1234 | source = "registry+https://github.com/rust-lang/crates.io-index"
1235 | checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
1236 |
1237 | [[package]]
1238 | name = "libm"
1239 | version = "0.2.11"
1240 | source = "registry+https://github.com/rust-lang/crates.io-index"
1241 | checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
1242 |
1243 | [[package]]
1244 | name = "linux-raw-sys"
1245 | version = "0.9.3"
1246 | source = "registry+https://github.com/rust-lang/crates.io-index"
1247 | checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413"
1248 |
1249 | [[package]]
1250 | name = "litemap"
1251 | version = "0.7.5"
1252 | source = "registry+https://github.com/rust-lang/crates.io-index"
1253 | checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856"
1254 |
1255 | [[package]]
1256 | name = "lock_api"
1257 | version = "0.4.12"
1258 | source = "registry+https://github.com/rust-lang/crates.io-index"
1259 | checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
1260 | dependencies = [
1261 | "autocfg",
1262 | "scopeguard",
1263 | ]
1264 |
1265 | [[package]]
1266 | name = "lockfree-object-pool"
1267 | version = "0.1.6"
1268 | source = "registry+https://github.com/rust-lang/crates.io-index"
1269 | checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e"
1270 |
1271 | [[package]]
1272 | name = "log"
1273 | version = "0.4.26"
1274 | source = "registry+https://github.com/rust-lang/crates.io-index"
1275 | checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e"
1276 |
1277 | [[package]]
1278 | name = "lzma-rs"
1279 | version = "0.3.0"
1280 | source = "registry+https://github.com/rust-lang/crates.io-index"
1281 | checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e"
1282 | dependencies = [
1283 | "byteorder",
1284 | "crc",
1285 | ]
1286 |
1287 | [[package]]
1288 | name = "mac"
1289 | version = "0.1.1"
1290 | source = "registry+https://github.com/rust-lang/crates.io-index"
1291 | checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
1292 |
1293 | [[package]]
1294 | name = "markup5ever"
1295 | version = "0.14.1"
1296 | source = "registry+https://github.com/rust-lang/crates.io-index"
1297 | checksum = "c7a7213d12e1864c0f002f52c2923d4556935a43dec5e71355c2760e0f6e7a18"
1298 | dependencies = [
1299 | "log",
1300 | "phf",
1301 | "phf_codegen",
1302 | "string_cache",
1303 | "string_cache_codegen",
1304 | "tendril",
1305 | ]
1306 |
1307 | [[package]]
1308 | name = "match_token"
1309 | version = "0.1.0"
1310 | source = "registry+https://github.com/rust-lang/crates.io-index"
1311 | checksum = "88a9689d8d44bf9964484516275f5cd4c9b59457a6940c1d5d0ecbb94510a36b"
1312 | dependencies = [
1313 | "proc-macro2",
1314 | "quote",
1315 | "syn",
1316 | ]
1317 |
1318 | [[package]]
1319 | name = "matrixmultiply"
1320 | version = "0.3.9"
1321 | source = "registry+https://github.com/rust-lang/crates.io-index"
1322 | checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a"
1323 | dependencies = [
1324 | "autocfg",
1325 | "rawpointer",
1326 | ]
1327 |
1328 | [[package]]
1329 | name = "md-5"
1330 | version = "0.10.6"
1331 | source = "registry+https://github.com/rust-lang/crates.io-index"
1332 | checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
1333 | dependencies = [
1334 | "cfg-if",
1335 | "digest",
1336 | ]
1337 |
1338 | [[package]]
1339 | name = "memchr"
1340 | version = "2.7.4"
1341 | source = "registry+https://github.com/rust-lang/crates.io-index"
1342 | checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
1343 |
1344 | [[package]]
1345 | name = "mime"
1346 | version = "0.3.17"
1347 | source = "registry+https://github.com/rust-lang/crates.io-index"
1348 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
1349 |
1350 | [[package]]
1351 | name = "miniz_oxide"
1352 | version = "0.8.5"
1353 | source = "registry+https://github.com/rust-lang/crates.io-index"
1354 | checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
1355 | dependencies = [
1356 | "adler2",
1357 | ]
1358 |
1359 | [[package]]
1360 | name = "mio"
1361 | version = "1.0.3"
1362 | source = "registry+https://github.com/rust-lang/crates.io-index"
1363 | checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
1364 | dependencies = [
1365 | "libc",
1366 | "wasi 0.11.0+wasi-snapshot-preview1",
1367 | "windows-sys 0.52.0",
1368 | ]
1369 |
1370 | [[package]]
1371 | name = "native-tls"
1372 | version = "0.2.14"
1373 | source = "registry+https://github.com/rust-lang/crates.io-index"
1374 | checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e"
1375 | dependencies = [
1376 | "libc",
1377 | "log",
1378 | "openssl",
1379 | "openssl-probe",
1380 | "openssl-sys",
1381 | "schannel",
1382 | "security-framework",
1383 | "security-framework-sys",
1384 | "tempfile",
1385 | ]
1386 |
1387 | [[package]]
1388 | name = "ndarray"
1389 | version = "0.16.1"
1390 | source = "registry+https://github.com/rust-lang/crates.io-index"
1391 | checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
1392 | dependencies = [
1393 | "matrixmultiply",
1394 | "num-complex",
1395 | "num-integer",
1396 | "num-traits",
1397 | "portable-atomic",
1398 | "portable-atomic-util",
1399 | "rawpointer",
1400 | "serde",
1401 | ]
1402 |
1403 | [[package]]
1404 | name = "new_debug_unreachable"
1405 | version = "1.0.6"
1406 | source = "registry+https://github.com/rust-lang/crates.io-index"
1407 | checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
1408 |
1409 | [[package]]
1410 | name = "num-complex"
1411 | version = "0.4.6"
1412 | source = "registry+https://github.com/rust-lang/crates.io-index"
1413 | checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
1414 | dependencies = [
1415 | "num-traits",
1416 | ]
1417 |
1418 | [[package]]
1419 | name = "num-conv"
1420 | version = "0.1.0"
1421 | source = "registry+https://github.com/rust-lang/crates.io-index"
1422 | checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
1423 |
1424 | [[package]]
1425 | name = "num-integer"
1426 | version = "0.1.46"
1427 | source = "registry+https://github.com/rust-lang/crates.io-index"
1428 | checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
1429 | dependencies = [
1430 | "num-traits",
1431 | ]
1432 |
1433 | [[package]]
1434 | name = "num-traits"
1435 | version = "0.2.19"
1436 | source = "registry+https://github.com/rust-lang/crates.io-index"
1437 | checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
1438 | dependencies = [
1439 | "autocfg",
1440 | "libm",
1441 | ]
1442 |
1443 | [[package]]
1444 | name = "num_cpus"
1445 | version = "1.16.0"
1446 | source = "registry+https://github.com/rust-lang/crates.io-index"
1447 | checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
1448 | dependencies = [
1449 | "hermit-abi",
1450 | "libc",
1451 | ]
1452 |
1453 | [[package]]
1454 | name = "number_prefix"
1455 | version = "0.4.0"
1456 | source = "registry+https://github.com/rust-lang/crates.io-index"
1457 | checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
1458 |
1459 | [[package]]
1460 | name = "object"
1461 | version = "0.36.7"
1462 | source = "registry+https://github.com/rust-lang/crates.io-index"
1463 | checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87"
1464 | dependencies = [
1465 | "memchr",
1466 | ]
1467 |
1468 | [[package]]
1469 | name = "once_cell"
1470 | version = "1.21.1"
1471 | source = "registry+https://github.com/rust-lang/crates.io-index"
1472 | checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc"
1473 |
1474 | [[package]]
1475 | name = "openssl"
1476 | version = "0.10.72"
1477 | source = "registry+https://github.com/rust-lang/crates.io-index"
1478 | checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da"
1479 | dependencies = [
1480 | "bitflags",
1481 | "cfg-if",
1482 | "foreign-types",
1483 | "libc",
1484 | "once_cell",
1485 | "openssl-macros",
1486 | "openssl-sys",
1487 | ]
1488 |
1489 | [[package]]
1490 | name = "openssl-macros"
1491 | version = "0.1.1"
1492 | source = "registry+https://github.com/rust-lang/crates.io-index"
1493 | checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
1494 | dependencies = [
1495 | "proc-macro2",
1496 | "quote",
1497 | "syn",
1498 | ]
1499 |
1500 | [[package]]
1501 | name = "openssl-probe"
1502 | version = "0.1.6"
1503 | source = "registry+https://github.com/rust-lang/crates.io-index"
1504 | checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
1505 |
1506 | [[package]]
1507 | name = "openssl-sys"
1508 | version = "0.9.108"
1509 | source = "registry+https://github.com/rust-lang/crates.io-index"
1510 | checksum = "e145e1651e858e820e4860f7b9c5e169bc1d8ce1c86043be79fa7b7634821847"
1511 | dependencies = [
1512 | "cc",
1513 | "libc",
1514 | "pkg-config",
1515 | "vcpkg",
1516 | ]
1517 |
1518 | [[package]]
1519 | name = "parking"
1520 | version = "2.2.1"
1521 | source = "registry+https://github.com/rust-lang/crates.io-index"
1522 | checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
1523 |
1524 | [[package]]
1525 | name = "parking_lot"
1526 | version = "0.12.3"
1527 | source = "registry+https://github.com/rust-lang/crates.io-index"
1528 | checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
1529 | dependencies = [
1530 | "lock_api",
1531 | "parking_lot_core",
1532 | ]
1533 |
1534 | [[package]]
1535 | name = "parking_lot_core"
1536 | version = "0.9.10"
1537 | source = "registry+https://github.com/rust-lang/crates.io-index"
1538 | checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
1539 | dependencies = [
1540 | "cfg-if",
1541 | "libc",
1542 | "redox_syscall",
1543 | "smallvec",
1544 | "windows-targets 0.52.6",
1545 | ]
1546 |
1547 | [[package]]
1548 | name = "pbkdf2"
1549 | version = "0.12.2"
1550 | source = "registry+https://github.com/rust-lang/crates.io-index"
1551 | checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2"
1552 | dependencies = [
1553 | "digest",
1554 | "hmac",
1555 | ]
1556 |
1557 | [[package]]
1558 | name = "percent-encoding"
1559 | version = "2.3.1"
1560 | source = "registry+https://github.com/rust-lang/crates.io-index"
1561 | checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
1562 |
1563 | [[package]]
1564 | name = "phf"
1565 | version = "0.11.3"
1566 | source = "registry+https://github.com/rust-lang/crates.io-index"
1567 | checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
1568 | dependencies = [
1569 | "phf_macros",
1570 | "phf_shared",
1571 | ]
1572 |
1573 | [[package]]
1574 | name = "phf_codegen"
1575 | version = "0.11.3"
1576 | source = "registry+https://github.com/rust-lang/crates.io-index"
1577 | checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a"
1578 | dependencies = [
1579 | "phf_generator",
1580 | "phf_shared",
1581 | ]
1582 |
1583 | [[package]]
1584 | name = "phf_generator"
1585 | version = "0.11.3"
1586 | source = "registry+https://github.com/rust-lang/crates.io-index"
1587 | checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
1588 | dependencies = [
1589 | "phf_shared",
1590 | "rand 0.8.5",
1591 | ]
1592 |
1593 | [[package]]
1594 | name = "phf_macros"
1595 | version = "0.11.3"
1596 | source = "registry+https://github.com/rust-lang/crates.io-index"
1597 | checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
1598 | dependencies = [
1599 | "phf_generator",
1600 | "phf_shared",
1601 | "proc-macro2",
1602 | "quote",
1603 | "syn",
1604 | ]
1605 |
1606 | [[package]]
1607 | name = "phf_shared"
1608 | version = "0.11.3"
1609 | source = "registry+https://github.com/rust-lang/crates.io-index"
1610 | checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
1611 | dependencies = [
1612 | "siphasher",
1613 | ]
1614 |
1615 | [[package]]
1616 | name = "pin-project-lite"
1617 | version = "0.2.16"
1618 | source = "registry+https://github.com/rust-lang/crates.io-index"
1619 | checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
1620 |
1621 | [[package]]
1622 | name = "pin-utils"
1623 | version = "0.1.0"
1624 | source = "registry+https://github.com/rust-lang/crates.io-index"
1625 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
1626 |
1627 | [[package]]
1628 | name = "pkg-config"
1629 | version = "0.3.32"
1630 | source = "registry+https://github.com/rust-lang/crates.io-index"
1631 | checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
1632 |
1633 | [[package]]
1634 | name = "portable-atomic"
1635 | version = "1.11.0"
1636 | source = "registry+https://github.com/rust-lang/crates.io-index"
1637 | checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
1638 |
1639 | [[package]]
1640 | name = "portable-atomic-util"
1641 | version = "0.2.4"
1642 | source = "registry+https://github.com/rust-lang/crates.io-index"
1643 | checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
1644 | dependencies = [
1645 | "portable-atomic",
1646 | ]
1647 |
1648 | [[package]]
1649 | name = "postgres-protocol"
1650 | version = "0.6.8"
1651 | source = "registry+https://github.com/rust-lang/crates.io-index"
1652 | checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54"
1653 | dependencies = [
1654 | "base64",
1655 | "byteorder",
1656 | "bytes",
1657 | "fallible-iterator",
1658 | "hmac",
1659 | "md-5",
1660 | "memchr",
1661 | "rand 0.9.0",
1662 | "sha2",
1663 | "stringprep",
1664 | ]
1665 |
1666 | [[package]]
1667 | name = "postgres-types"
1668 | version = "0.2.9"
1669 | source = "registry+https://github.com/rust-lang/crates.io-index"
1670 | checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48"
1671 | dependencies = [
1672 | "bytes",
1673 | "fallible-iterator",
1674 | "geo-types",
1675 | "postgres-protocol",
1676 | "serde",
1677 | "serde_json",
1678 | ]
1679 |
1680 | [[package]]
1681 | name = "powerfmt"
1682 | version = "0.2.0"
1683 | source = "registry+https://github.com/rust-lang/crates.io-index"
1684 | checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
1685 |
1686 | [[package]]
1687 | name = "ppv-lite86"
1688 | version = "0.2.21"
1689 | source = "registry+https://github.com/rust-lang/crates.io-index"
1690 | checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
1691 | dependencies = [
1692 | "zerocopy",
1693 | ]
1694 |
1695 | [[package]]
1696 | name = "precomputed-hash"
1697 | version = "0.1.1"
1698 | source = "registry+https://github.com/rust-lang/crates.io-index"
1699 | checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
1700 |
1701 | [[package]]
1702 | name = "proc-macro2"
1703 | version = "1.0.94"
1704 | source = "registry+https://github.com/rust-lang/crates.io-index"
1705 | checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84"
1706 | dependencies = [
1707 | "unicode-ident",
1708 | ]
1709 |
1710 | [[package]]
1711 | name = "quick-xml"
1712 | version = "0.31.0"
1713 | source = "registry+https://github.com/rust-lang/crates.io-index"
1714 | checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
1715 | dependencies = [
1716 | "encoding_rs",
1717 | "memchr",
1718 | ]
1719 |
1720 | [[package]]
1721 | name = "quote"
1722 | version = "1.0.40"
1723 | source = "registry+https://github.com/rust-lang/crates.io-index"
1724 | checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
1725 | dependencies = [
1726 | "proc-macro2",
1727 | ]
1728 |
1729 | [[package]]
1730 | name = "r-efi"
1731 | version = "5.2.0"
1732 | source = "registry+https://github.com/rust-lang/crates.io-index"
1733 | checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
1734 |
1735 | [[package]]
1736 | name = "rand"
1737 | version = "0.8.5"
1738 | source = "registry+https://github.com/rust-lang/crates.io-index"
1739 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
1740 | dependencies = [
1741 | "rand_core 0.6.4",
1742 | ]
1743 |
1744 | [[package]]
1745 | name = "rand"
1746 | version = "0.9.0"
1747 | source = "registry+https://github.com/rust-lang/crates.io-index"
1748 | checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94"
1749 | dependencies = [
1750 | "rand_chacha",
1751 | "rand_core 0.9.3",
1752 | "zerocopy",
1753 | ]
1754 |
1755 | [[package]]
1756 | name = "rand_chacha"
1757 | version = "0.9.0"
1758 | source = "registry+https://github.com/rust-lang/crates.io-index"
1759 | checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
1760 | dependencies = [
1761 | "ppv-lite86",
1762 | "rand_core 0.9.3",
1763 | ]
1764 |
1765 | [[package]]
1766 | name = "rand_core"
1767 | version = "0.6.4"
1768 | source = "registry+https://github.com/rust-lang/crates.io-index"
1769 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
1770 |
1771 | [[package]]
1772 | name = "rand_core"
1773 | version = "0.9.3"
1774 | source = "registry+https://github.com/rust-lang/crates.io-index"
1775 | checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
1776 | dependencies = [
1777 | "getrandom 0.3.2",
1778 | ]
1779 |
1780 | [[package]]
1781 | name = "rawpointer"
1782 | version = "0.2.1"
1783 | source = "registry+https://github.com/rust-lang/crates.io-index"
1784 | checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
1785 |
1786 | [[package]]
1787 | name = "redox_syscall"
1788 | version = "0.5.10"
1789 | source = "registry+https://github.com/rust-lang/crates.io-index"
1790 | checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1"
1791 | dependencies = [
1792 | "bitflags",
1793 | ]
1794 |
1795 | [[package]]
1796 | name = "regex"
1797 | version = "1.11.1"
1798 | source = "registry+https://github.com/rust-lang/crates.io-index"
1799 | checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
1800 | dependencies = [
1801 | "aho-corasick",
1802 | "memchr",
1803 | "regex-automata",
1804 | "regex-syntax",
1805 | ]
1806 |
1807 | [[package]]
1808 | name = "regex-automata"
1809 | version = "0.4.9"
1810 | source = "registry+https://github.com/rust-lang/crates.io-index"
1811 | checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
1812 | dependencies = [
1813 | "aho-corasick",
1814 | "memchr",
1815 | "regex-syntax",
1816 | ]
1817 |
1818 | [[package]]
1819 | name = "regex-syntax"
1820 | version = "0.8.5"
1821 | source = "registry+https://github.com/rust-lang/crates.io-index"
1822 | checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
1823 |
1824 | [[package]]
1825 | name = "reqwest"
1826 | version = "0.12.14"
1827 | source = "registry+https://github.com/rust-lang/crates.io-index"
1828 | checksum = "989e327e510263980e231de548a33e63d34962d29ae61b467389a1a09627a254"
1829 | dependencies = [
1830 | "base64",
1831 | "bytes",
1832 | "encoding_rs",
1833 | "futures-core",
1834 | "futures-util",
1835 | "h2",
1836 | "http",
1837 | "http-body",
1838 | "http-body-util",
1839 | "hyper",
1840 | "hyper-rustls",
1841 | "hyper-tls",
1842 | "hyper-util",
1843 | "ipnet",
1844 | "js-sys",
1845 | "log",
1846 | "mime",
1847 | "native-tls",
1848 | "once_cell",
1849 | "percent-encoding",
1850 | "pin-project-lite",
1851 | "rustls-pemfile",
1852 | "serde",
1853 | "serde_json",
1854 | "serde_urlencoded",
1855 | "sync_wrapper",
1856 | "system-configuration",
1857 | "tokio",
1858 | "tokio-native-tls",
1859 | "tokio-util",
1860 | "tower",
1861 | "tower-service",
1862 | "url",
1863 | "wasm-bindgen",
1864 | "wasm-bindgen-futures",
1865 | "wasm-streams",
1866 | "web-sys",
1867 | "windows-registry",
1868 | ]
1869 |
1870 | [[package]]
1871 | name = "ring"
1872 | version = "0.17.14"
1873 | source = "registry+https://github.com/rust-lang/crates.io-index"
1874 | checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
1875 | dependencies = [
1876 | "cc",
1877 | "cfg-if",
1878 | "getrandom 0.2.15",
1879 | "libc",
1880 | "untrusted",
1881 | "windows-sys 0.52.0",
1882 | ]
1883 |
1884 | [[package]]
1885 | name = "rustc-demangle"
1886 | version = "0.1.24"
1887 | source = "registry+https://github.com/rust-lang/crates.io-index"
1888 | checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
1889 |
1890 | [[package]]
1891 | name = "rustix"
1892 | version = "1.0.2"
1893 | source = "registry+https://github.com/rust-lang/crates.io-index"
1894 | checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825"
1895 | dependencies = [
1896 | "bitflags",
1897 | "errno",
1898 | "libc",
1899 | "linux-raw-sys",
1900 | "windows-sys 0.59.0",
1901 | ]
1902 |
1903 | [[package]]
1904 | name = "rustls"
1905 | version = "0.23.25"
1906 | source = "registry+https://github.com/rust-lang/crates.io-index"
1907 | checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c"
1908 | dependencies = [
1909 | "once_cell",
1910 | "rustls-pki-types",
1911 | "rustls-webpki",
1912 | "subtle",
1913 | "zeroize",
1914 | ]
1915 |
1916 | [[package]]
1917 | name = "rustls-pemfile"
1918 | version = "2.2.0"
1919 | source = "registry+https://github.com/rust-lang/crates.io-index"
1920 | checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
1921 | dependencies = [
1922 | "rustls-pki-types",
1923 | ]
1924 |
1925 | [[package]]
1926 | name = "rustls-pki-types"
1927 | version = "1.11.0"
1928 | source = "registry+https://github.com/rust-lang/crates.io-index"
1929 | checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
1930 |
1931 | [[package]]
1932 | name = "rustls-webpki"
1933 | version = "0.103.0"
1934 | source = "registry+https://github.com/rust-lang/crates.io-index"
1935 | checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f"
1936 | dependencies = [
1937 | "ring",
1938 | "rustls-pki-types",
1939 | "untrusted",
1940 | ]
1941 |
1942 | [[package]]
1943 | name = "rustversion"
1944 | version = "1.0.20"
1945 | source = "registry+https://github.com/rust-lang/crates.io-index"
1946 | checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
1947 |
1948 | [[package]]
1949 | name = "ryu"
1950 | version = "1.0.20"
1951 | source = "registry+https://github.com/rust-lang/crates.io-index"
1952 | checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
1953 |
1954 | [[package]]
1955 | name = "schannel"
1956 | version = "0.1.27"
1957 | source = "registry+https://github.com/rust-lang/crates.io-index"
1958 | checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d"
1959 | dependencies = [
1960 | "windows-sys 0.59.0",
1961 | ]
1962 |
1963 | [[package]]
1964 | name = "scopeguard"
1965 | version = "1.2.0"
1966 | source = "registry+https://github.com/rust-lang/crates.io-index"
1967 | checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
1968 |
1969 | [[package]]
1970 | name = "scraper"
1971 | version = "0.22.0"
1972 | source = "registry+https://github.com/rust-lang/crates.io-index"
1973 | checksum = "cc3d051b884f40e309de6c149734eab57aa8cc1347992710dc80bcc1c2194c15"
1974 | dependencies = [
1975 | "cssparser",
1976 | "ego-tree",
1977 | "getopts",
1978 | "html5ever",
1979 | "precomputed-hash",
1980 | "selectors",
1981 | "tendril",
1982 | ]
1983 |
1984 | [[package]]
1985 | name = "security-framework"
1986 | version = "2.11.1"
1987 | source = "registry+https://github.com/rust-lang/crates.io-index"
1988 | checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
1989 | dependencies = [
1990 | "bitflags",
1991 | "core-foundation",
1992 | "core-foundation-sys",
1993 | "libc",
1994 | "security-framework-sys",
1995 | ]
1996 |
1997 | [[package]]
1998 | name = "security-framework-sys"
1999 | version = "2.14.0"
2000 | source = "registry+https://github.com/rust-lang/crates.io-index"
2001 | checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32"
2002 | dependencies = [
2003 | "core-foundation-sys",
2004 | "libc",
2005 | ]
2006 |
2007 | [[package]]
2008 | name = "selectors"
2009 | version = "0.26.0"
2010 | source = "registry+https://github.com/rust-lang/crates.io-index"
2011 | checksum = "fd568a4c9bb598e291a08244a5c1f5a8a6650bee243b5b0f8dbb3d9cc1d87fe8"
2012 | dependencies = [
2013 | "bitflags",
2014 | "cssparser",
2015 | "derive_more",
2016 | "fxhash",
2017 | "log",
2018 | "new_debug_unreachable",
2019 | "phf",
2020 | "phf_codegen",
2021 | "precomputed-hash",
2022 | "servo_arc",
2023 | "smallvec",
2024 | ]
2025 |
2026 | [[package]]
2027 | name = "serde"
2028 | version = "1.0.219"
2029 | source = "registry+https://github.com/rust-lang/crates.io-index"
2030 | checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
2031 | dependencies = [
2032 | "serde_derive",
2033 | ]
2034 |
2035 | [[package]]
2036 | name = "serde_derive"
2037 | version = "1.0.219"
2038 | source = "registry+https://github.com/rust-lang/crates.io-index"
2039 | checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
2040 | dependencies = [
2041 | "proc-macro2",
2042 | "quote",
2043 | "syn",
2044 | ]
2045 |
2046 | [[package]]
2047 | name = "serde_json"
2048 | version = "1.0.140"
2049 | source = "registry+https://github.com/rust-lang/crates.io-index"
2050 | checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
2051 | dependencies = [
2052 | "itoa",
2053 | "memchr",
2054 | "ryu",
2055 | "serde",
2056 | ]
2057 |
2058 | [[package]]
2059 | name = "serde_urlencoded"
2060 | version = "0.7.1"
2061 | source = "registry+https://github.com/rust-lang/crates.io-index"
2062 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
2063 | dependencies = [
2064 | "form_urlencoded",
2065 | "itoa",
2066 | "ryu",
2067 | "serde",
2068 | ]
2069 |
2070 | [[package]]
2071 | name = "servo_arc"
2072 | version = "0.4.0"
2073 | source = "registry+https://github.com/rust-lang/crates.io-index"
2074 | checksum = "ae65c4249478a2647db249fb43e23cec56a2c8974a427e7bd8cb5a1d0964921a"
2075 | dependencies = [
2076 | "stable_deref_trait",
2077 | ]
2078 |
2079 | [[package]]
2080 | name = "sha1"
2081 | version = "0.10.6"
2082 | source = "registry+https://github.com/rust-lang/crates.io-index"
2083 | checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
2084 | dependencies = [
2085 | "cfg-if",
2086 | "cpufeatures",
2087 | "digest",
2088 | ]
2089 |
2090 | [[package]]
2091 | name = "sha2"
2092 | version = "0.10.8"
2093 | source = "registry+https://github.com/rust-lang/crates.io-index"
2094 | checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
2095 | dependencies = [
2096 | "cfg-if",
2097 | "cpufeatures",
2098 | "digest",
2099 | ]
2100 |
2101 | [[package]]
2102 | name = "shlex"
2103 | version = "1.3.0"
2104 | source = "registry+https://github.com/rust-lang/crates.io-index"
2105 | checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
2106 |
2107 | [[package]]
2108 | name = "signal-hook-registry"
2109 | version = "1.4.2"
2110 | source = "registry+https://github.com/rust-lang/crates.io-index"
2111 | checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
2112 | dependencies = [
2113 | "libc",
2114 | ]
2115 |
2116 | [[package]]
2117 | name = "simd-adler32"
2118 | version = "0.3.7"
2119 | source = "registry+https://github.com/rust-lang/crates.io-index"
2120 | checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
2121 |
2122 | [[package]]
2123 | name = "siphasher"
2124 | version = "1.0.1"
2125 | source = "registry+https://github.com/rust-lang/crates.io-index"
2126 | checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
2127 |
2128 | [[package]]
2129 | name = "slab"
2130 | version = "0.4.9"
2131 | source = "registry+https://github.com/rust-lang/crates.io-index"
2132 | checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
2133 | dependencies = [
2134 | "autocfg",
2135 | ]
2136 |
2137 | [[package]]
2138 | name = "smallvec"
2139 | version = "1.14.0"
2140 | source = "registry+https://github.com/rust-lang/crates.io-index"
2141 | checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd"
2142 |
2143 | [[package]]
2144 | name = "socket2"
2145 | version = "0.5.8"
2146 | source = "registry+https://github.com/rust-lang/crates.io-index"
2147 | checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
2148 | dependencies = [
2149 | "libc",
2150 | "windows-sys 0.52.0",
2151 | ]
2152 |
2153 | [[package]]
2154 | name = "stable_deref_trait"
2155 | version = "1.2.0"
2156 | source = "registry+https://github.com/rust-lang/crates.io-index"
2157 | checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
2158 |
2159 | [[package]]
2160 | name = "string_cache"
2161 | version = "0.8.8"
2162 | source = "registry+https://github.com/rust-lang/crates.io-index"
2163 | checksum = "938d512196766101d333398efde81bc1f37b00cb42c2f8350e5df639f040bbbe"
2164 | dependencies = [
2165 | "new_debug_unreachable",
2166 | "parking_lot",
2167 | "phf_shared",
2168 | "precomputed-hash",
2169 | "serde",
2170 | ]
2171 |
2172 | [[package]]
2173 | name = "string_cache_codegen"
2174 | version = "0.5.4"
2175 | source = "registry+https://github.com/rust-lang/crates.io-index"
2176 | checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0"
2177 | dependencies = [
2178 | "phf_generator",
2179 | "phf_shared",
2180 | "proc-macro2",
2181 | "quote",
2182 | ]
2183 |
2184 | [[package]]
2185 | name = "stringprep"
2186 | version = "0.1.5"
2187 | source = "registry+https://github.com/rust-lang/crates.io-index"
2188 | checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
2189 | dependencies = [
2190 | "unicode-bidi",
2191 | "unicode-normalization",
2192 | "unicode-properties",
2193 | ]
2194 |
2195 | [[package]]
2196 | name = "strsim"
2197 | version = "0.11.1"
2198 | source = "registry+https://github.com/rust-lang/crates.io-index"
2199 | checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
2200 |
2201 | [[package]]
2202 | name = "subtle"
2203 | version = "2.6.1"
2204 | source = "registry+https://github.com/rust-lang/crates.io-index"
2205 | checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
2206 |
2207 | [[package]]
2208 | name = "syn"
2209 | version = "2.0.100"
2210 | source = "registry+https://github.com/rust-lang/crates.io-index"
2211 | checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
2212 | dependencies = [
2213 | "proc-macro2",
2214 | "quote",
2215 | "unicode-ident",
2216 | ]
2217 |
2218 | [[package]]
2219 | name = "sync_wrapper"
2220 | version = "1.0.2"
2221 | source = "registry+https://github.com/rust-lang/crates.io-index"
2222 | checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
2223 | dependencies = [
2224 | "futures-core",
2225 | ]
2226 |
2227 | [[package]]
2228 | name = "synstructure"
2229 | version = "0.13.1"
2230 | source = "registry+https://github.com/rust-lang/crates.io-index"
2231 | checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
2232 | dependencies = [
2233 | "proc-macro2",
2234 | "quote",
2235 | "syn",
2236 | ]
2237 |
2238 | [[package]]
2239 | name = "system-configuration"
2240 | version = "0.6.1"
2241 | source = "registry+https://github.com/rust-lang/crates.io-index"
2242 | checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
2243 | dependencies = [
2244 | "bitflags",
2245 | "core-foundation",
2246 | "system-configuration-sys",
2247 | ]
2248 |
2249 | [[package]]
2250 | name = "system-configuration-sys"
2251 | version = "0.6.0"
2252 | source = "registry+https://github.com/rust-lang/crates.io-index"
2253 | checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
2254 | dependencies = [
2255 | "core-foundation-sys",
2256 | "libc",
2257 | ]
2258 |
2259 | [[package]]
2260 | name = "tempfile"
2261 | version = "3.19.0"
2262 | source = "registry+https://github.com/rust-lang/crates.io-index"
2263 | checksum = "488960f40a3fd53d72c2a29a58722561dee8afdd175bd88e3db4677d7b2ba600"
2264 | dependencies = [
2265 | "fastrand",
2266 | "getrandom 0.3.2",
2267 | "once_cell",
2268 | "rustix",
2269 | "windows-sys 0.59.0",
2270 | ]
2271 |
2272 | [[package]]
2273 | name = "tendril"
2274 | version = "0.4.3"
2275 | source = "registry+https://github.com/rust-lang/crates.io-index"
2276 | checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
2277 | dependencies = [
2278 | "futf",
2279 | "mac",
2280 | "utf-8",
2281 | ]
2282 |
2283 | [[package]]
2284 | name = "thiserror"
2285 | version = "2.0.12"
2286 | source = "registry+https://github.com/rust-lang/crates.io-index"
2287 | checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708"
2288 | dependencies = [
2289 | "thiserror-impl",
2290 | ]
2291 |
2292 | [[package]]
2293 | name = "thiserror-impl"
2294 | version = "2.0.12"
2295 | source = "registry+https://github.com/rust-lang/crates.io-index"
2296 | checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d"
2297 | dependencies = [
2298 | "proc-macro2",
2299 | "quote",
2300 | "syn",
2301 | ]
2302 |
2303 | [[package]]
2304 | name = "time"
2305 | version = "0.3.39"
2306 | source = "registry+https://github.com/rust-lang/crates.io-index"
2307 | checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8"
2308 | dependencies = [
2309 | "deranged",
2310 | "num-conv",
2311 | "powerfmt",
2312 | "serde",
2313 | "time-core",
2314 | ]
2315 |
2316 | [[package]]
2317 | name = "time-core"
2318 | version = "0.1.3"
2319 | source = "registry+https://github.com/rust-lang/crates.io-index"
2320 | checksum = "765c97a5b985b7c11d7bc27fa927dc4fe6af3a6dfb021d28deb60d3bf51e76ef"
2321 |
2322 | [[package]]
2323 | name = "tinystr"
2324 | version = "0.7.6"
2325 | source = "registry+https://github.com/rust-lang/crates.io-index"
2326 | checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
2327 | dependencies = [
2328 | "displaydoc",
2329 | "zerovec",
2330 | ]
2331 |
2332 | [[package]]
2333 | name = "tinyvec"
2334 | version = "1.9.0"
2335 | source = "registry+https://github.com/rust-lang/crates.io-index"
2336 | checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71"
2337 | dependencies = [
2338 | "tinyvec_macros",
2339 | ]
2340 |
2341 | [[package]]
2342 | name = "tinyvec_macros"
2343 | version = "0.1.1"
2344 | source = "registry+https://github.com/rust-lang/crates.io-index"
2345 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
2346 |
2347 | [[package]]
2348 | name = "tokio"
2349 | version = "1.44.2"
2350 | source = "registry+https://github.com/rust-lang/crates.io-index"
2351 | checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48"
2352 | dependencies = [
2353 | "backtrace",
2354 | "bytes",
2355 | "libc",
2356 | "mio",
2357 | "parking_lot",
2358 | "pin-project-lite",
2359 | "signal-hook-registry",
2360 | "socket2",
2361 | "tokio-macros",
2362 | "windows-sys 0.52.0",
2363 | ]
2364 |
2365 | [[package]]
2366 | name = "tokio-macros"
2367 | version = "2.5.0"
2368 | source = "registry+https://github.com/rust-lang/crates.io-index"
2369 | checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
2370 | dependencies = [
2371 | "proc-macro2",
2372 | "quote",
2373 | "syn",
2374 | ]
2375 |
2376 | [[package]]
2377 | name = "tokio-native-tls"
2378 | version = "0.3.1"
2379 | source = "registry+https://github.com/rust-lang/crates.io-index"
2380 | checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
2381 | dependencies = [
2382 | "native-tls",
2383 | "tokio",
2384 | ]
2385 |
2386 | [[package]]
2387 | name = "tokio-postgres"
2388 | version = "0.7.13"
2389 | source = "registry+https://github.com/rust-lang/crates.io-index"
2390 | checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0"
2391 | dependencies = [
2392 | "async-trait",
2393 | "byteorder",
2394 | "bytes",
2395 | "fallible-iterator",
2396 | "futures-channel",
2397 | "futures-util",
2398 | "log",
2399 | "parking_lot",
2400 | "percent-encoding",
2401 | "phf",
2402 | "pin-project-lite",
2403 | "postgres-protocol",
2404 | "postgres-types",
2405 | "rand 0.9.0",
2406 | "socket2",
2407 | "tokio",
2408 | "tokio-util",
2409 | "whoami",
2410 | ]
2411 |
2412 | [[package]]
2413 | name = "tokio-rustls"
2414 | version = "0.26.2"
2415 | source = "registry+https://github.com/rust-lang/crates.io-index"
2416 | checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b"
2417 | dependencies = [
2418 | "rustls",
2419 | "tokio",
2420 | ]
2421 |
2422 | [[package]]
2423 | name = "tokio-util"
2424 | version = "0.7.14"
2425 | source = "registry+https://github.com/rust-lang/crates.io-index"
2426 | checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034"
2427 | dependencies = [
2428 | "bytes",
2429 | "futures-core",
2430 | "futures-sink",
2431 | "pin-project-lite",
2432 | "tokio",
2433 | ]
2434 |
2435 | [[package]]
2436 | name = "tower"
2437 | version = "0.5.2"
2438 | source = "registry+https://github.com/rust-lang/crates.io-index"
2439 | checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
2440 | dependencies = [
2441 | "futures-core",
2442 | "futures-util",
2443 | "pin-project-lite",
2444 | "sync_wrapper",
2445 | "tokio",
2446 | "tower-layer",
2447 | "tower-service",
2448 | ]
2449 |
2450 | [[package]]
2451 | name = "tower-layer"
2452 | version = "0.3.3"
2453 | source = "registry+https://github.com/rust-lang/crates.io-index"
2454 | checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
2455 |
2456 | [[package]]
2457 | name = "tower-service"
2458 | version = "0.3.3"
2459 | source = "registry+https://github.com/rust-lang/crates.io-index"
2460 | checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
2461 |
2462 | [[package]]
2463 | name = "tracing"
2464 | version = "0.1.41"
2465 | source = "registry+https://github.com/rust-lang/crates.io-index"
2466 | checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
2467 | dependencies = [
2468 | "pin-project-lite",
2469 | "tracing-core",
2470 | ]
2471 |
2472 | [[package]]
2473 | name = "tracing-core"
2474 | version = "0.1.33"
2475 | source = "registry+https://github.com/rust-lang/crates.io-index"
2476 | checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c"
2477 | dependencies = [
2478 | "once_cell",
2479 | ]
2480 |
2481 | [[package]]
2482 | name = "try-lock"
2483 | version = "0.2.5"
2484 | source = "registry+https://github.com/rust-lang/crates.io-index"
2485 | checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
2486 |
2487 | [[package]]
2488 | name = "typenum"
2489 | version = "1.18.0"
2490 | source = "registry+https://github.com/rust-lang/crates.io-index"
2491 | checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f"
2492 |
2493 | [[package]]
2494 | name = "unicode-bidi"
2495 | version = "0.3.18"
2496 | source = "registry+https://github.com/rust-lang/crates.io-index"
2497 | checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
2498 |
2499 | [[package]]
2500 | name = "unicode-ident"
2501 | version = "1.0.18"
2502 | source = "registry+https://github.com/rust-lang/crates.io-index"
2503 | checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
2504 |
2505 | [[package]]
2506 | name = "unicode-normalization"
2507 | version = "0.1.24"
2508 | source = "registry+https://github.com/rust-lang/crates.io-index"
2509 | checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956"
2510 | dependencies = [
2511 | "tinyvec",
2512 | ]
2513 |
2514 | [[package]]
2515 | name = "unicode-properties"
2516 | version = "0.1.3"
2517 | source = "registry+https://github.com/rust-lang/crates.io-index"
2518 | checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0"
2519 |
2520 | [[package]]
2521 | name = "unicode-width"
2522 | version = "0.1.14"
2523 | source = "registry+https://github.com/rust-lang/crates.io-index"
2524 | checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
2525 |
2526 | [[package]]
2527 | name = "unicode-width"
2528 | version = "0.2.0"
2529 | source = "registry+https://github.com/rust-lang/crates.io-index"
2530 | checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
2531 |
2532 | [[package]]
2533 | name = "untrusted"
2534 | version = "0.9.0"
2535 | source = "registry+https://github.com/rust-lang/crates.io-index"
2536 | checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
2537 |
2538 | [[package]]
2539 | name = "url"
2540 | version = "2.5.4"
2541 | source = "registry+https://github.com/rust-lang/crates.io-index"
2542 | checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60"
2543 | dependencies = [
2544 | "form_urlencoded",
2545 | "idna",
2546 | "percent-encoding",
2547 | "serde",
2548 | ]
2549 |
2550 | [[package]]
2551 | name = "utf-8"
2552 | version = "0.7.6"
2553 | source = "registry+https://github.com/rust-lang/crates.io-index"
2554 | checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
2555 |
2556 | [[package]]
2557 | name = "utf16_iter"
2558 | version = "1.0.5"
2559 | source = "registry+https://github.com/rust-lang/crates.io-index"
2560 | checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
2561 |
2562 | [[package]]
2563 | name = "utf8_iter"
2564 | version = "1.0.4"
2565 | source = "registry+https://github.com/rust-lang/crates.io-index"
2566 | checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
2567 |
2568 | [[package]]
2569 | name = "utf8parse"
2570 | version = "0.2.2"
2571 | source = "registry+https://github.com/rust-lang/crates.io-index"
2572 | checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
2573 |
2574 | [[package]]
2575 | name = "vcpkg"
2576 | version = "0.2.15"
2577 | source = "registry+https://github.com/rust-lang/crates.io-index"
2578 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
2579 |
2580 | [[package]]
2581 | name = "version_check"
2582 | version = "0.9.5"
2583 | source = "registry+https://github.com/rust-lang/crates.io-index"
2584 | checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
2585 |
2586 | [[package]]
2587 | name = "want"
2588 | version = "0.3.1"
2589 | source = "registry+https://github.com/rust-lang/crates.io-index"
2590 | checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
2591 | dependencies = [
2592 | "try-lock",
2593 | ]
2594 |
2595 | [[package]]
2596 | name = "wasi"
2597 | version = "0.11.0+wasi-snapshot-preview1"
2598 | source = "registry+https://github.com/rust-lang/crates.io-index"
2599 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
2600 |
2601 | [[package]]
2602 | name = "wasi"
2603 | version = "0.14.2+wasi-0.2.4"
2604 | source = "registry+https://github.com/rust-lang/crates.io-index"
2605 | checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
2606 | dependencies = [
2607 | "wit-bindgen-rt",
2608 | ]
2609 |
2610 | [[package]]
2611 | name = "wasite"
2612 | version = "0.1.0"
2613 | source = "registry+https://github.com/rust-lang/crates.io-index"
2614 | checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
2615 |
2616 | [[package]]
2617 | name = "wasm-bindgen"
2618 | version = "0.2.100"
2619 | source = "registry+https://github.com/rust-lang/crates.io-index"
2620 | checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
2621 | dependencies = [
2622 | "cfg-if",
2623 | "once_cell",
2624 | "rustversion",
2625 | "wasm-bindgen-macro",
2626 | ]
2627 |
2628 | [[package]]
2629 | name = "wasm-bindgen-backend"
2630 | version = "0.2.100"
2631 | source = "registry+https://github.com/rust-lang/crates.io-index"
2632 | checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
2633 | dependencies = [
2634 | "bumpalo",
2635 | "log",
2636 | "proc-macro2",
2637 | "quote",
2638 | "syn",
2639 | "wasm-bindgen-shared",
2640 | ]
2641 |
2642 | [[package]]
2643 | name = "wasm-bindgen-futures"
2644 | version = "0.4.50"
2645 | source = "registry+https://github.com/rust-lang/crates.io-index"
2646 | checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61"
2647 | dependencies = [
2648 | "cfg-if",
2649 | "js-sys",
2650 | "once_cell",
2651 | "wasm-bindgen",
2652 | "web-sys",
2653 | ]
2654 |
2655 | [[package]]
2656 | name = "wasm-bindgen-macro"
2657 | version = "0.2.100"
2658 | source = "registry+https://github.com/rust-lang/crates.io-index"
2659 | checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
2660 | dependencies = [
2661 | "quote",
2662 | "wasm-bindgen-macro-support",
2663 | ]
2664 |
2665 | [[package]]
2666 | name = "wasm-bindgen-macro-support"
2667 | version = "0.2.100"
2668 | source = "registry+https://github.com/rust-lang/crates.io-index"
2669 | checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
2670 | dependencies = [
2671 | "proc-macro2",
2672 | "quote",
2673 | "syn",
2674 | "wasm-bindgen-backend",
2675 | "wasm-bindgen-shared",
2676 | ]
2677 |
2678 | [[package]]
2679 | name = "wasm-bindgen-shared"
2680 | version = "0.2.100"
2681 | source = "registry+https://github.com/rust-lang/crates.io-index"
2682 | checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
2683 | dependencies = [
2684 | "unicode-ident",
2685 | ]
2686 |
2687 | [[package]]
2688 | name = "wasm-streams"
2689 | version = "0.4.2"
2690 | source = "registry+https://github.com/rust-lang/crates.io-index"
2691 | checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
2692 | dependencies = [
2693 | "futures-util",
2694 | "js-sys",
2695 | "wasm-bindgen",
2696 | "wasm-bindgen-futures",
2697 | "web-sys",
2698 | ]
2699 |
2700 | [[package]]
2701 | name = "web-sys"
2702 | version = "0.3.77"
2703 | source = "registry+https://github.com/rust-lang/crates.io-index"
2704 | checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
2705 | dependencies = [
2706 | "js-sys",
2707 | "wasm-bindgen",
2708 | ]
2709 |
2710 | [[package]]
2711 | name = "web-time"
2712 | version = "1.1.0"
2713 | source = "registry+https://github.com/rust-lang/crates.io-index"
2714 | checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
2715 | dependencies = [
2716 | "js-sys",
2717 | "wasm-bindgen",
2718 | ]
2719 |
2720 | [[package]]
2721 | name = "whoami"
2722 | version = "1.5.2"
2723 | source = "registry+https://github.com/rust-lang/crates.io-index"
2724 | checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d"
2725 | dependencies = [
2726 | "redox_syscall",
2727 | "wasite",
2728 | "web-sys",
2729 | ]
2730 |
2731 | [[package]]
2732 | name = "windows-link"
2733 | version = "0.1.0"
2734 | source = "registry+https://github.com/rust-lang/crates.io-index"
2735 | checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3"
2736 |
2737 | [[package]]
2738 | name = "windows-registry"
2739 | version = "0.4.0"
2740 | source = "registry+https://github.com/rust-lang/crates.io-index"
2741 | checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3"
2742 | dependencies = [
2743 | "windows-result",
2744 | "windows-strings",
2745 | "windows-targets 0.53.0",
2746 | ]
2747 |
2748 | [[package]]
2749 | name = "windows-result"
2750 | version = "0.3.1"
2751 | source = "registry+https://github.com/rust-lang/crates.io-index"
2752 | checksum = "06374efe858fab7e4f881500e6e86ec8bc28f9462c47e5a9941a0142ad86b189"
2753 | dependencies = [
2754 | "windows-link",
2755 | ]
2756 |
2757 | [[package]]
2758 | name = "windows-strings"
2759 | version = "0.3.1"
2760 | source = "registry+https://github.com/rust-lang/crates.io-index"
2761 | checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319"
2762 | dependencies = [
2763 | "windows-link",
2764 | ]
2765 |
2766 | [[package]]
2767 | name = "windows-sys"
2768 | version = "0.52.0"
2769 | source = "registry+https://github.com/rust-lang/crates.io-index"
2770 | checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
2771 | dependencies = [
2772 | "windows-targets 0.52.6",
2773 | ]
2774 |
2775 | [[package]]
2776 | name = "windows-sys"
2777 | version = "0.59.0"
2778 | source = "registry+https://github.com/rust-lang/crates.io-index"
2779 | checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
2780 | dependencies = [
2781 | "windows-targets 0.52.6",
2782 | ]
2783 |
2784 | [[package]]
2785 | name = "windows-targets"
2786 | version = "0.52.6"
2787 | source = "registry+https://github.com/rust-lang/crates.io-index"
2788 | checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
2789 | dependencies = [
2790 | "windows_aarch64_gnullvm 0.52.6",
2791 | "windows_aarch64_msvc 0.52.6",
2792 | "windows_i686_gnu 0.52.6",
2793 | "windows_i686_gnullvm 0.52.6",
2794 | "windows_i686_msvc 0.52.6",
2795 | "windows_x86_64_gnu 0.52.6",
2796 | "windows_x86_64_gnullvm 0.52.6",
2797 | "windows_x86_64_msvc 0.52.6",
2798 | ]
2799 |
2800 | [[package]]
2801 | name = "windows-targets"
2802 | version = "0.53.0"
2803 | source = "registry+https://github.com/rust-lang/crates.io-index"
2804 | checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b"
2805 | dependencies = [
2806 | "windows_aarch64_gnullvm 0.53.0",
2807 | "windows_aarch64_msvc 0.53.0",
2808 | "windows_i686_gnu 0.53.0",
2809 | "windows_i686_gnullvm 0.53.0",
2810 | "windows_i686_msvc 0.53.0",
2811 | "windows_x86_64_gnu 0.53.0",
2812 | "windows_x86_64_gnullvm 0.53.0",
2813 | "windows_x86_64_msvc 0.53.0",
2814 | ]
2815 |
2816 | [[package]]
2817 | name = "windows_aarch64_gnullvm"
2818 | version = "0.52.6"
2819 | source = "registry+https://github.com/rust-lang/crates.io-index"
2820 | checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
2821 |
2822 | [[package]]
2823 | name = "windows_aarch64_gnullvm"
2824 | version = "0.53.0"
2825 | source = "registry+https://github.com/rust-lang/crates.io-index"
2826 | checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
2827 |
2828 | [[package]]
2829 | name = "windows_aarch64_msvc"
2830 | version = "0.52.6"
2831 | source = "registry+https://github.com/rust-lang/crates.io-index"
2832 | checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
2833 |
2834 | [[package]]
2835 | name = "windows_aarch64_msvc"
2836 | version = "0.53.0"
2837 | source = "registry+https://github.com/rust-lang/crates.io-index"
2838 | checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
2839 |
2840 | [[package]]
2841 | name = "windows_i686_gnu"
2842 | version = "0.52.6"
2843 | source = "registry+https://github.com/rust-lang/crates.io-index"
2844 | checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
2845 |
2846 | [[package]]
2847 | name = "windows_i686_gnu"
2848 | version = "0.53.0"
2849 | source = "registry+https://github.com/rust-lang/crates.io-index"
2850 | checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
2851 |
2852 | [[package]]
2853 | name = "windows_i686_gnullvm"
2854 | version = "0.52.6"
2855 | source = "registry+https://github.com/rust-lang/crates.io-index"
2856 | checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
2857 |
2858 | [[package]]
2859 | name = "windows_i686_gnullvm"
2860 | version = "0.53.0"
2861 | source = "registry+https://github.com/rust-lang/crates.io-index"
2862 | checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
2863 |
2864 | [[package]]
2865 | name = "windows_i686_msvc"
2866 | version = "0.52.6"
2867 | source = "registry+https://github.com/rust-lang/crates.io-index"
2868 | checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
2869 |
2870 | [[package]]
2871 | name = "windows_i686_msvc"
2872 | version = "0.53.0"
2873 | source = "registry+https://github.com/rust-lang/crates.io-index"
2874 | checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
2875 |
2876 | [[package]]
2877 | name = "windows_x86_64_gnu"
2878 | version = "0.52.6"
2879 | source = "registry+https://github.com/rust-lang/crates.io-index"
2880 | checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
2881 |
2882 | [[package]]
2883 | name = "windows_x86_64_gnu"
2884 | version = "0.53.0"
2885 | source = "registry+https://github.com/rust-lang/crates.io-index"
2886 | checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
2887 |
2888 | [[package]]
2889 | name = "windows_x86_64_gnullvm"
2890 | version = "0.52.6"
2891 | source = "registry+https://github.com/rust-lang/crates.io-index"
2892 | checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
2893 |
2894 | [[package]]
2895 | name = "windows_x86_64_gnullvm"
2896 | version = "0.53.0"
2897 | source = "registry+https://github.com/rust-lang/crates.io-index"
2898 | checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
2899 |
2900 | [[package]]
2901 | name = "windows_x86_64_msvc"
2902 | version = "0.52.6"
2903 | source = "registry+https://github.com/rust-lang/crates.io-index"
2904 | checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
2905 |
2906 | [[package]]
2907 | name = "windows_x86_64_msvc"
2908 | version = "0.53.0"
2909 | source = "registry+https://github.com/rust-lang/crates.io-index"
2910 | checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
2911 |
2912 | [[package]]
2913 | name = "wit-bindgen-rt"
2914 | version = "0.39.0"
2915 | source = "registry+https://github.com/rust-lang/crates.io-index"
2916 | checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
2917 | dependencies = [
2918 | "bitflags",
2919 | ]
2920 |
2921 | [[package]]
2922 | name = "write16"
2923 | version = "1.0.0"
2924 | source = "registry+https://github.com/rust-lang/crates.io-index"
2925 | checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
2926 |
2927 | [[package]]
2928 | name = "writeable"
2929 | version = "0.5.5"
2930 | source = "registry+https://github.com/rust-lang/crates.io-index"
2931 | checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
2932 |
2933 | [[package]]
2934 | name = "yoke"
2935 | version = "0.7.5"
2936 | source = "registry+https://github.com/rust-lang/crates.io-index"
2937 | checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40"
2938 | dependencies = [
2939 | "serde",
2940 | "stable_deref_trait",
2941 | "yoke-derive",
2942 | "zerofrom",
2943 | ]
2944 |
2945 | [[package]]
2946 | name = "yoke-derive"
2947 | version = "0.7.5"
2948 | source = "registry+https://github.com/rust-lang/crates.io-index"
2949 | checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154"
2950 | dependencies = [
2951 | "proc-macro2",
2952 | "quote",
2953 | "syn",
2954 | "synstructure",
2955 | ]
2956 |
2957 | [[package]]
2958 | name = "zerocopy"
2959 | version = "0.8.23"
2960 | source = "registry+https://github.com/rust-lang/crates.io-index"
2961 | checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6"
2962 | dependencies = [
2963 | "zerocopy-derive",
2964 | ]
2965 |
2966 | [[package]]
2967 | name = "zerocopy-derive"
2968 | version = "0.8.23"
2969 | source = "registry+https://github.com/rust-lang/crates.io-index"
2970 | checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154"
2971 | dependencies = [
2972 | "proc-macro2",
2973 | "quote",
2974 | "syn",
2975 | ]
2976 |
2977 | [[package]]
2978 | name = "zerofrom"
2979 | version = "0.1.6"
2980 | source = "registry+https://github.com/rust-lang/crates.io-index"
2981 | checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
2982 | dependencies = [
2983 | "zerofrom-derive",
2984 | ]
2985 |
2986 | [[package]]
2987 | name = "zerofrom-derive"
2988 | version = "0.1.6"
2989 | source = "registry+https://github.com/rust-lang/crates.io-index"
2990 | checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
2991 | dependencies = [
2992 | "proc-macro2",
2993 | "quote",
2994 | "syn",
2995 | "synstructure",
2996 | ]
2997 |
2998 | [[package]]
2999 | name = "zeroize"
3000 | version = "1.8.1"
3001 | source = "registry+https://github.com/rust-lang/crates.io-index"
3002 | checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde"
3003 | dependencies = [
3004 | "zeroize_derive",
3005 | ]
3006 |
3007 | [[package]]
3008 | name = "zeroize_derive"
3009 | version = "1.4.2"
3010 | source = "registry+https://github.com/rust-lang/crates.io-index"
3011 | checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69"
3012 | dependencies = [
3013 | "proc-macro2",
3014 | "quote",
3015 | "syn",
3016 | ]
3017 |
3018 | [[package]]
3019 | name = "zerovec"
3020 | version = "0.10.4"
3021 | source = "registry+https://github.com/rust-lang/crates.io-index"
3022 | checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079"
3023 | dependencies = [
3024 | "yoke",
3025 | "zerofrom",
3026 | "zerovec-derive",
3027 | ]
3028 |
3029 | [[package]]
3030 | name = "zerovec-derive"
3031 | version = "0.10.3"
3032 | source = "registry+https://github.com/rust-lang/crates.io-index"
3033 | checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6"
3034 | dependencies = [
3035 | "proc-macro2",
3036 | "quote",
3037 | "syn",
3038 | ]
3039 |
3040 | [[package]]
3041 | name = "zip"
3042 | version = "2.5.0"
3043 | source = "registry+https://github.com/rust-lang/crates.io-index"
3044 | checksum = "27c03817464f64e23f6f37574b4fdc8cf65925b5bfd2b0f2aedf959791941f88"
3045 | dependencies = [
3046 | "aes",
3047 | "arbitrary",
3048 | "constant_time_eq",
3049 | "crc32fast",
3050 | "crossbeam-utils",
3051 | "deflate64",
3052 | "flate2",
3053 | "getrandom 0.3.2",
3054 | "hmac",
3055 | "indexmap",
3056 | "lzma-rs",
3057 | "memchr",
3058 | "pbkdf2",
3059 | "sha1",
3060 | "time",
3061 | "zeroize",
3062 | "zopfli",
3063 | ]
3064 |
3065 | [[package]]
3066 | name = "zopfli"
3067 | version = "0.8.1"
3068 | source = "registry+https://github.com/rust-lang/crates.io-index"
3069 | checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946"
3070 | dependencies = [
3071 | "bumpalo",
3072 | "crc32fast",
3073 | "lockfree-object-pool",
3074 | "log",
3075 | "once_cell",
3076 | "simd-adler32",
3077 | ]
3078 |
--------------------------------------------------------------------------------
/Cargo.toml:
--------------------------------------------------------------------------------
1 | [package]
2 | name = "jpksj-to-sql"
3 | version = "0.2.0"
4 | edition = "2021"
5 | license = "MIT"
6 |
7 | [dependencies]
8 | anyhow = "1.0"
9 | async-channel = "2.3.1"
10 | bytesize = "1.3.2"
11 | calamine = "0.26"
12 | clap = { version = "4.5", features = ["derive"] }
13 | derive_builder = "0.20.2"
14 | encoding_rs = "0.8"
15 | futures-util = "0.3.31"
16 | geo-types = "0.7"
17 | indicatif = { version = "0.17.11", features = ["tokio"] }
18 | km-to-sql = "0.1.1"
19 | ndarray = { version = "0.16", features = ["serde"] }
20 | num_cpus = "1"
21 | once_cell = "1.20.3"
22 | regex = "1"
23 | reqwest = { version = "0.12", features = ["stream"] }
24 | scraper = "0.22"
25 | serde = { version = "1", features = ["derive"] }
26 | serde_json = "1"
27 | tokio = { version = "1", features = ["full"] }
28 | tokio-postgres = { version = "0.7", features = ["with-geo-types-0_7", "with-serde_json-1"] }
29 | unicode-normalization = "0.1.24"
30 | url = { version = "2", features = ["serde"] }
31 |
32 | [dependencies.zip]
33 | version = "2.2"
34 | default-features = false
35 | features = [
36 | "aes-crypto",
37 | # "bzip2",
38 | "deflate64",
39 | "deflate",
40 | "lzma",
41 | "time",
42 | # "zstd",
43 | # "xz",
44 | ]
45 |
--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | # ビルドステージ
2 | FROM rust:latest as builder
3 |
4 | # GDAL開発ライブラリと依存関係をインストール
5 | RUN apt-get update && apt-get install -y \
6 | software-properties-common \
7 | gnupg \
8 | wget \
9 | && apt-get install -y \
10 | build-essential \
11 | cmake \
12 | pkg-config \
13 | libgdal-dev \
14 | gdal-bin \
15 | libpq-dev \
16 | zip \
17 | unzip \
18 | && apt-get clean \
19 | && rm -rf /var/lib/apt/lists/*
20 |
21 | # GDAL バージョンを確認
22 | RUN gdalinfo --version
23 |
24 | # アプリケーションのビルド
25 | WORKDIR /app
26 | COPY . .
27 | RUN cargo build --release
28 |
29 | # 実行ステージ
30 | FROM debian:bookworm-slim
31 |
32 | # ランタイム依存関係をインストール
33 | RUN apt-get update && apt-get install -y \
34 | libgdal32 \
35 | gdal-bin \
36 | libpq5 \
37 | postgresql-client \
38 | curl \
39 | ca-certificates \
40 | libssl3 \
41 | && apt-get clean \
42 | && rm -rf /var/lib/apt/lists/*
43 |
44 | # GDAL バージョンを確認
45 | RUN gdalinfo --version
46 |
47 | # GDAL_DATAパスを設定
48 | ENV GDAL_DATA=/usr/share/gdal
49 |
50 | # 一時ファイル用のディレクトリを作成
51 | WORKDIR /app
52 | RUN mkdir -p /app/tmp
53 | VOLUME ["/app/tmp"]
54 |
55 | # ビルドステージからバイナリをコピー
56 | COPY --from=builder /app/target/release/jpksj-to-sql /usr/local/bin/jpksj-to-sql
57 |
58 | # 実行時の待機スクリプト
59 | COPY ./docker-entrypoint.sh /usr/local/bin/
60 | RUN chmod +x /usr/local/bin/docker-entrypoint.sh
61 |
62 | ENTRYPOINT ["docker-entrypoint.sh"]
63 | CMD ["jpksj-to-sql"]
64 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2025 Keitaroh Kobayashi
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/NOTES.md:
--------------------------------------------------------------------------------
1 | ```
2 | $PG_CONN_STR="host=..."
3 | ```
4 |
5 | # `A40` - 津波浸水深
6 |
7 | 「津波浸水深の区分」のフォーマットが統一化されていないためこちらで正規化しています。
8 | そのうち組み込むかも。
9 |
10 | ```sql
11 | CREATE OR REPLACE FUNCTION a40_normalize_range(range_text text)
12 | RETURNS text AS $$
13 | DECLARE
14 | norm text;
15 | lower_range text;
16 | upper_range text;
17 | BEGIN
18 | -- Step 1: Trim whitespace.
19 | norm := trim(range_text);
20 |
21 | -- Normalize A - B patterns
22 | norm := regexp_replace(norm, '^([0-9\.]+)m?(?:以上)?(?:[ ～]+)([0-9\.]+)m?(?:未満)?$', '\1m-\2m');
23 | -- A-
24 | norm := regexp_replace(norm, '^([0-9\.]+)m?(?:以上)?(?:[ ～]*)$', '\1m-');
25 | -- -B
26 | norm := regexp_replace(norm, '^(?:[ ～]*)([0-9\.]+)m?(?:未満)?$', '-\1m');
27 |
28 | -- Replace .0
29 | norm := regexp_replace(norm, '(\d+)\.0m', '\1m', 'g');
30 |
31 | RETURN norm;
32 | END;
33 | $$ LANGUAGE plpgsql IMMUTABLE;
34 |
35 | CREATE OR REPLACE FUNCTION a40_get_upper_bound(range_text text)
36 | RETURNS numeric AS $$
37 | DECLARE
38 | norm text;
39 | match_result text[];
40 | BEGIN
41 | -- Step 1: Trim whitespace.
42 | norm := trim(range_text);
43 |
44 | -- Case 1: A - B pattern (e.g., "0.5m - 1.0m未満")
45 | match_result := regexp_match(norm, '^([0-9\.]+)m?(?:以上)?(?:[ ～]+)([0-9\.]+)m?(?:未満)?$');
46 | IF match_result IS NOT NULL THEN
47 | RETURN match_result[2]::numeric;
48 | END IF;
49 |
50 | -- Case 2: A- pattern (e.g., "5m以上")
51 | match_result := regexp_match(norm, '^([0-9\.]+)m?(?:以上)?(?:[ ～]*)$');
52 | IF match_result IS NOT NULL THEN
53 | RETURN 99; -- Special value for unspecified upper bound
54 | END IF;
55 |
56 | -- Case 3: -B pattern (e.g., "0.3m未満")
57 | match_result := regexp_match(norm, '^(?:[ ～]*)([0-9\.]+)m?(?:未満)?$');
58 | IF match_result IS NOT NULL THEN
59 | RETURN match_result[1]::numeric;
60 | END IF;
61 |
62 | -- Fallback for unparseable input
63 | RETURN NULL;
64 | END;
65 | $$ LANGUAGE plpgsql IMMUTABLE;
66 |
67 | CREATE OR REPLACE FUNCTION a40_get_lower_bound(range_text text)
68 | RETURNS numeric AS $$
69 | DECLARE
70 | norm text;
71 | match_result text[];
72 | BEGIN
73 | -- Step 1: Trim whitespace.
74 | norm := trim(range_text);
75 |
76 | -- Case 1: A - B pattern (e.g., "0.5m - 1.0m未満")
77 | match_result := regexp_match(norm, '^([0-9\.]+)m?(?:以上)?(?:[ ～]+)([0-9\.]+)m?(?:未満)?$');
78 | IF match_result IS NOT NULL THEN
79 | RETURN match_result[1]::numeric;
80 | END IF;
81 |
82 | -- Case 2: A- pattern (e.g., "5m以上")
83 | match_result := regexp_match(norm, '^([0-9\.]+)m?(?:以上)?(?:[ ～]*)$');
84 | IF match_result IS NOT NULL THEN
85 | RETURN match_result[1]::numeric;
86 | END IF;
87 |
88 | -- Case 3: -B pattern (e.g., "0.3m未満")
89 | match_result := regexp_match(norm, '^(?:[ ～]*)([0-9\.]+)m?(?:未満)?$');
90 | IF match_result IS NOT NULL THEN
91 | RETURN -99; -- Special value for unspecified lower bound
92 | END IF;
93 |
94 | -- Fallback for unparseable input
95 | RETURN NULL;
96 | END;
97 | $$ LANGUAGE plpgsql IMMUTABLE;
98 |
99 | CREATE OR REPLACE VIEW a40_normalized AS
100 | SELECT
101 | t.ogc_fid,
102 | t.都道府県コード,
103 | t.都道府県名,
104 | a40_normalize_range(t."津波浸水深の区分") AS "津波浸水深の区分",
105 | a40_get_lower_bound(t."津波浸水深の区分") AS "min",
106 | a40_get_upper_bound(t."津波浸水深の区分") AS "max",
107 | t.geom
108 | FROM a40 t;
109 | ```
110 |
111 | FlatGeobuf へのエクスポート
112 |
113 | ```
114 | ogr2ogr -f FlatGeobuf a40_normalized.fgb PG:"$PG_CONN_STR" a40_normalized
115 | ```
116 |
117 | # `A38` - 医療圏
118 |
119 | ```
120 | # 3次医療圏 (都府県＋北海道6圏)
121 | ogr2ogr -f FlatGeobuf a38_3.fgb PG:"$PG_CONN_STR" a38c
122 | # 2次医療圏 (3次より細かい、簡易的な集計データも付与)
123 | ogr2ogr -f FlatGeobuf a38_2.fgb PG:"$PG_CONN_STR" a38b
124 | # 1次医療圏 (2次より更に細かい)
125 | ogr2ogr -f FlatGeobuf a38_1.fgb PG:"$PG_CONN_STR" a38a
126 | ```
127 |
128 | タイル化
129 |
130 | ```
131 | tippecanoe -n "医療圏" -N "3次、2次、1次医療圏のポリゴンデータ" -A "「国土数値情報(医療圏データ)」(国土交通省)をもとにKotobaMedia株式会社作成" -Z0 -z13 -o a38.pmtiles -Ltier1:./a38_1.fgb -Ltier2:./a38_2.fgb -Ltier3:./a38_3.fgb
132 | ```
133 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 国土数値情報データをSQLデータベースに取り込む
2 |
3 | このツールは、[国土数値情報](https://nlftp.mlit.go.jp/ksj/)のデータとメタデータをPostgreSQL (PostGIS) 用のデータベースに取り込み、すぐに自由な分析ができる状態に整理します。
4 |
5 | 取り込むデータは、[JPGIS2.1準拠整備データ一覧](https://nlftp.mlit.go.jp/ksj/gml/gml_datalist.html)から選ばれ、同一データセットに複数ファイルがある場合は1つのテーブルにまとめます（現状は最新年度のみ対応）。
6 |
7 | なお、各データセットには「商用」「非商用」「CC BY 4.0」など利用条件が設定されているため、使用時は十分にご注意ください。（今のところ、非商用データはフィルタされています）
8 |
9 | ## データベースの概要
10 |
11 | * データの識別子をテーブル名とし、カラム名は日本語へマッピング後となります。
12 | * 位置情報は `geom` カラムに入っています
13 | * Feature ID は `ogc_fid`（ogr2ogr により自動生成）
14 | * `datasets` テーブルにメタデータが入っています
15 | * メタデータは [to-sql シリーズと共通](https://github.com/KotobaMedia/km-to-sql/)になっています
16 |
17 | ### メタデータの形
18 |
19 | 識別子 `P05` からの引用
20 |
21 | ```jsonc
22 | {
23 | "desc": "全国の市役所、区役所、町役場、村役場、及びこれらの支所、出張所、連絡所等、及び市区町村が主体的に設置・管理・運営する公民館、集会所等の公的集会施設について、その位置と名称、所在地、施設分類コード、行政コードをGISデータとして整備したものである。",
24 | "name": "市町村役場等及び公的集会施設",
25 | "source_url": "https://nlftp.mlit.go.jp/ksj/gml/datalist/KsjTmplt-P05-2022.html",
26 | "primary_key": "ogc_fid",
27 | "columns": [
28 | {
29 | // ogr2ogr が自動で作るプライマリキー
30 | // `name` はデータベース上のカラム名
31 | "name": "ogc_fid",
32 | "data_type": "int4"
33 | },
34 | {
35 | // 外部キーの場合。「行政区域」はすべて admini_boundary_cd を利用します。
36 | "desc": "都道府県コードと市区町村コードからなる、行政区を特定するためのコード",
37 | "name": "行政区域コード",
38 | "data_type": "varchar",
39 | "foreign_key": {
40 | "foreign_table": "admini_boundary_cd",
41 | "foreign_column": "改正後のコード"
42 | }
43 | },
44 | {
45 | "desc": "市町村役場等及び公的集会施設の分類を表すコード",
46 | "name": "施設分類",
47 | "data_type": "varchar",
48 | "enum_values": [
49 | {
50 | "desc": "集会施設",
51 | "value": "5"
52 | },
53 | {
54 | "desc": "上記以外の行政サービス施設",
55 | "value": "3"
56 | },
57 | {
58 | "desc": "公立公民館",
59 | "value": "4"
60 | },
61 | {
62 | "desc": "本庁（市役所、区役所、町役場、村役場）",
63 | "value": "1"
64 | },
65 | {
66 | "desc": "支所、出張所、連絡所",
67 | "value": "2"
68 | }
69 | ]
70 | // コードリストではなくて定数の場合は `[{"value":"値1"},{"value":"値2"},...]` となります。
71 | },
72 | {
73 | "desc": "市町村役場等及び公的集会施設の名称",
74 | "name": "名称",
75 | "data_type": "varchar"
76 | },
77 | {
78 | "desc": "市町村役場等及び公的集会施設の所在地。",
79 | "name": "所在地",
80 | "data_type": "varchar"
81 | },
82 | {
83 | "name": "geom",
84 | "data_type": "geometry(MULTIPOINT, 6668)"
85 | }
86 | ],
87 | "license": "CC_BY_4.0"
88 | }
89 | ```
90 | ※ 'admini_boundary_cd' テーブルは別途インポートが必要です（例: 行政区域データ）。
91 |
92 | ## 利用方法
93 |
94 | ### 使用手順（概要）
95 | 1. バイナリをダウンロード
96 | 2. PostgreSQL + PostGIS を用意
97 | 3. コマンドを実行:
98 | jpksj-to-sql "host=127.0.0.1 dbname=jpksj"
99 |
100 | バイナリを [最新リリース](https://github.com/keichan34/jpksj-to-sql/releases/) からダウンロードするのがおすすめです。
101 |
102 | GDAL 3.9以上必要です (`ogr2ogr` または `ogrinfo` が実行できる環境。 `ogrinfo` は `-limit` 引数使うので、 3.9 が必要です)
103 |
104 | ```
105 | jpksj-to-sql "host=127.0.0.1 dbname=jpksj"
106 | ```
107 |
108 | macOS の場合、GitHub Release からダウンロードしたバイナリが Gatekeeper によりブロックされることがあります。その場合は、次のコマンドで実行を許可できます: xattr -d com.apple.quarantine ./jpksj-to-sql
109 |
110 | インターネット接続、メモリ、SSD転送速度等によって処理時間が大幅に左右します。途中からの続きを再開するために幾つかのオプションがあるので、 `jpksj-to-sql --help` で確認してください。
111 |
112 | ダウンロードした ZIP ファイルや解凍した shapefile をデフォルトで実行ディレクトリ内 `./tmp` に保存されます。
113 |
114 | ### Docker環境での利用方法
115 |
116 | Docker環境を使うことで、PostgreSQLとGDALの設定を自動化し、簡単に利用することができます。
117 |
118 | 1. リポジトリをクローンします。
119 | ```
120 | git clone https://github.com/keichan34/jpksj-to-sql.git
121 | cd jpksj-to-sql
122 | ```
123 |
124 | 2. Dockerコンテナをビルドして起動します。
125 | ```
126 | docker compose build
127 | docker compose up
128 | ```
129 |
130 | 3. アプリケーションのログを確認するには:
131 | ```
132 | docker compose logs -f jpksj-to-sql
133 | ```
134 |
135 | 4. データベースに接続するには:
136 | ```
137 | docker compose exec db psql -U postgres -d jpksj
138 | ```
139 |
140 | Docker環境では以下の設定が適用されます：
141 | - PostgreSQL + PostGIS（バージョン15-3.4）がデータベースコンテナとして実行されます
142 | - データベースのデータはDockerボリューム（postgres-data）に保存されます
143 | - ダウンロードファイルはホストマシンの`./tmp`ディレクトリに保存されます
144 | - デフォルトのデータベース接続情報:
145 | - ホスト: `db`
146 | - ユーザー: `postgres`
147 | - パスワード: `postgres`
148 | - データベース名: `jpksj`
149 |
150 | ## コンパイル
151 |
152 | Rust の開発環境が必要です。構築後、 cargo を使ってください
153 |
154 | ```
155 | cargo build
156 | ```
157 |
158 | ## ステータス
159 |
160 | こちらは実験的なツールであり、商用環境での使用は推奨していません。データの実験のために使われているので適当な実装が多いのですが、機能について下記をご覧ください。
161 | 「PR歓迎」となっているところは挑戦してみてください。
162 |
163 | ### 実装済み
164 | - 国土数値情報のウェブサイトからデータ一覧の取得
165 | - データのダウンロードおよび解凍
166 | - メタデータの取得およびパース（ `shape_property_table2.xlsx` ）
167 | - メタデータを元に属性名マッピング（shapefileでの `G04a_001` みたいなのを SQL カラム名 `3次メッシュコード` に変換）
168 | - メタデータをデータベース内に保存（ `datasets` テーブル参照してください ）
169 | - 読み込むデータセットの指定
170 | - 文字コードの認識
171 |
172 | ### 未対応・PR歓迎
173 | - [読み込み失敗しているデータセットはまだある](https://github.com/KotobaMedia/jpksj-to-sql/labels/%E8%AA%AD%E3%81%BF%E8%BE%BC%E3%81%BF%E5%A4%B1%E6%95%97)
174 | - VRTによるレイヤー統合から並列処理に変更
175 | - 同一データセット内に複数識別子が存在する時のハンドリング
176 | - 複数年のデータが存在する場合のハンドリング
177 | - PostgreSQL以外のデータベースにも保存
178 | - 部分更新（必要ないかも）
179 |
180 | ## ライセンス
181 |
182 | こちらのレポジトリのソースコードには MIT ライセンスのもとで提供されます。
183 |
184 | > [!IMPORTANT]
185 | > このツールでダウンロード・加工したデータを利用するときに、[国土数値情報の利用規約](https://nlftp.mlit.go.jp/ksj/other/agreement_01.html)を確認した上で利用ください。
186 |
--------------------------------------------------------------------------------
/data/schema.sql:
--------------------------------------------------------------------------------
1 | CREATE EXTENSION IF NOT EXISTS "postgis";
2 |
3 | CREATE TABLE IF NOT EXISTS "admini_boundary_cd" (
4 | "行政区域コード" VARCHAR(5) PRIMARY KEY NOT NULL,
5 | "都道府県名（漢字）" TEXT,
6 | "市区町村名（漢字）" TEXT,
7 | "都道府県名（カナ）" TEXT,
8 | "市区町村名（カナ）" TEXT,
9 | "コードの改定区分" TEXT,
10 | "改正年月日" TEXT,
11 | "改正後のコード" VARCHAR(5) NOT NULL,
12 | "改正後の名称" TEXT,
13 | "改正後の名称（カナ）" TEXT,
14 | "改正事由等" TEXT
15 | );
16 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | services:
2 | db:
3 | image: postgis/postgis:15-3.4
4 | environment:
5 | POSTGRES_USER: postgres
6 | POSTGRES_PASSWORD: postgres
7 | POSTGRES_DB: jpksj
8 | volumes:
9 | - postgres-data:/var/lib/postgresql/data
10 | - ./data/schema.sql:/docker-entrypoint-initdb.d/schema.sql
11 | ports:
12 | - "5432:5432"
13 | healthcheck:
14 | test: ["CMD-SHELL", "pg_isready -U postgres"]
15 | interval: 10s
16 | timeout: 5s
17 | retries: 5
18 |
19 | jpksj-to-sql:
20 | build:
21 | context: .
22 | dockerfile: Dockerfile
23 | depends_on:
24 | db:
25 | condition: service_healthy
26 | volumes:
27 | - ./tmp:/app/tmp
28 | environment:
29 | - DATABASE_URL=host=db user=postgres password=postgres dbname=jpksj
30 | command: jpksj-to-sql "host=db user=postgres password=postgres dbname=jpksj"
31 | restart: on-failure
32 |
33 | volumes:
34 | postgres-data:
35 |
--------------------------------------------------------------------------------
/docker-entrypoint.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | set -e
3 |
4 | # データベース接続が確立されるまで待機する関数
5 | wait_for_db() {
6 | echo "データベース接続を待機しています..."
7 |
8 | host=$(echo $1 | grep -oP 'host=\K[^ ]+')
9 | user=$(echo $1 | grep -oP 'user=\K[^ ]+')
10 | password=$(echo $1 | grep -oP 'password=\K[^ ]+')
11 | dbname=$(echo $1 | grep -oP 'dbname=\K[^ ]+')
12 |
13 | until PGPASSWORD=$password psql -h $host -U $user -d $dbname -c '\q' > /dev/null 2>&1; do
14 | echo "PostgreSQLサーバーが起動するのを待っています..."
15 | sleep 2
16 | done
17 |
18 | echo "データベース接続が確立されました！"
19 | }
20 |
21 | # データベース接続文字列が引数で渡された場合は待機する
22 | if [[ "$2" == host=* ]]; then
23 | wait_for_db "$2"
24 | fi
25 |
26 | # コマンドを実行
27 | exec "$@"
--------------------------------------------------------------------------------
/src/cli.rs:
--------------------------------------------------------------------------------
1 | use std::path::PathBuf;
2 |
3 | use clap::Parser;
4 |
5 | #[derive(Parser)]
6 | pub struct Cli {
7 | /// Postgresデータベースに接続する文字列。 ogr2ogr に渡されます。冒頭の `PG:` は省略してください。
8 | pub postgres_url: String,
9 |
10 | /// 中間ファイルの保存先 (Zip等)
11 | /// デフォルトは `./tmp` となります。
12 | #[arg(long)]
13 | pub tmp_dir: Option,
14 |
15 | /// データのダウンロードをスキップします
16 | /// データが存在しない場合はスキップされます
17 | #[arg(long, default_value = "false")]
18 | pub skip_download: bool,
19 |
20 | /// 既に存在するテーブルをスキップします
21 | /// プロセスが途中で中断された場合、テーブルが中途半端な状態にある可能性があります
22 | #[arg(long, default_value = "false")]
23 | pub skip_sql_if_exists: bool,
24 |
25 | /// 読み込むデータセットの識別子
26 | /// 指定しない場合は全てのデータセットが読み込まれます
27 | /// 複数指定する場合は `,` で区切ってください
28 | #[arg(long, value_delimiter = ',')]
29 | pub filter_identifiers: Option>,
30 | }
31 |
32 | pub fn main() -> Cli {
33 | Cli::parse()
34 | }
35 |
--------------------------------------------------------------------------------
/src/context.rs:
--------------------------------------------------------------------------------
1 | use std::{path::PathBuf, sync::OnceLock};
2 |
3 | fn default_tmp() -> PathBuf {
4 | PathBuf::from("./tmp")
5 | }
6 |
7 | static TMP: OnceLock = OnceLock::new();
8 | pub fn set_tmp(tmp: PathBuf) {
9 | TMP.set(tmp).unwrap();
10 | }
11 | pub fn tmp() -> &'static PathBuf {
12 | TMP.get_or_init(|| default_tmp())
13 | }
14 |
--------------------------------------------------------------------------------
/src/downloader.rs:
--------------------------------------------------------------------------------
1 | use anyhow::Result;
2 | use futures_util::StreamExt;
3 | use serde::{Deserialize, Serialize};
4 | use std::path::PathBuf;
5 | use tokio::fs::{self, File};
6 | use tokio::io::AsyncWriteExt;
7 | use url::Url;
8 |
9 | use crate::context;
10 |
11 | #[derive(Serialize, Deserialize)]
12 | struct Metadata {
13 | last_modified: Option,
14 | etag: Option,
15 | }
16 |
17 | pub struct DownloadedFile {
18 | pub path: PathBuf,
19 | }
20 |
21 | pub fn path_for_url(url: &Url) -> (PathBuf, PathBuf) {
22 | let tmp = context::tmp();
23 | let filename = url
24 | .path_segments()
25 | .and_then(|segments| segments.last())
26 | .unwrap_or("file");
27 | (
28 | tmp.join(filename),
29 | tmp.join(format!("{}.meta.json", filename)),
30 | )
31 | }
32 |
33 | pub async fn download_to_tmp(url: &Url) -> Result {
34 | let (file_path, meta_path) = path_for_url(&url);
35 |
36 | // Try to read existing metadata if it exists.
37 | let metadata: Option = if let Ok(meta_content) = fs::read_to_string(&meta_path).await
38 | {
39 | serde_json::from_str(&meta_content).ok()
40 | } else {
41 | None
42 | };
43 |
44 | let client = reqwest::Client::new();
45 | let mut request = client.get(url.clone());
46 |
47 | // Add conditional headers if metadata is available.
48 | if let Some(meta) = &metadata {
49 | if let Some(etag) = &meta.etag {
50 | request = request.header(reqwest::header::IF_NONE_MATCH, etag);
51 | }
52 | if let Some(last_modified) = &meta.last_modified {
53 | request = request.header(reqwest::header::IF_MODIFIED_SINCE, last_modified);
54 | }
55 | }
56 |
57 | let response = request.send().await?;
58 |
59 | // If the server indicates the file has not changed, return the existing file.
60 | if response.status() == reqwest::StatusCode::NOT_MODIFIED {
61 | // What if the file is missing even though we have metadata?
62 | if !file_path.exists() {
63 | return Err(anyhow::anyhow!(
64 | "Server returned 304 Not Modified, but file is missing"
65 | ));
66 | }
67 | return Ok(DownloadedFile { path: file_path });
68 | }
69 |
70 | // Ensure the response is successful (will error on 4xx or 5xx responses).
71 | let response = response.error_for_status()?;
72 |
73 | // Create (or overwrite) the target file.
74 | let mut file = File::create(&file_path).await?;
75 |
76 | // Extract metadata from response headers.
77 | let last_modified = response
78 | .headers()
79 | .get(reqwest::header::LAST_MODIFIED)
80 | .and_then(|v| v.to_str().ok())
81 | .map(|s| s.to_string());
82 | let etag = response
83 | .headers()
84 | .get(reqwest::header::ETAG)
85 | .and_then(|v| v.to_str().ok())
86 | .map(|s| s.to_string());
87 |
88 | let new_metadata = Metadata {
89 | last_modified,
90 | etag,
91 | };
92 |
93 | // Stream the response body and write it chunk by chunk.
94 | let mut stream = response.bytes_stream();
95 | while let Some(chunk) = stream.next().await {
96 | let chunk = chunk?;
97 | file.write_all(&chunk).await?;
98 | }
99 | file.flush().await?;
100 |
101 | // Serialize and write the metadata to a {filename}.meta.json file.
102 | let meta_json = serde_json::to_string_pretty(&new_metadata)?;
103 | fs::write(&meta_path, meta_json).await?;
104 | // Note that this is set after the file is completely written. That way, if the process crashed or was interrupted, we won't have a partial file.
105 |
106 | Ok(DownloadedFile { path: file_path })
107 | }
108 |
--------------------------------------------------------------------------------
/src/loader/admini_boundary.rs:
--------------------------------------------------------------------------------
1 | //! Loader for AdminiBoundary_CD.xslx
2 | //! This module is responsible for loading the AdminiBoundary_CD.xslx file into the database.
3 |
4 | use crate::{downloader, metadata::MetadataConnection};
5 | use anyhow::{Context, Result};
6 | use calamine::{Reader, Xlsx};
7 | use km_to_sql::metadata::{ColumnMetadata, TableMetadata};
8 | use std::vec;
9 | use tokio_postgres::{types::ToSql, NoTls};
10 | use unicode_normalization::UnicodeNormalization;
11 | use url::Url;
12 |
13 | use super::xslx_helpers::data_to_string;
14 |
15 | async fn download_admini_boundary_file() -> Result {
16 | let url = Url::parse("https://nlftp.mlit.go.jp/ksj/gml/codelist/AdminiBoundary_CD.xlsx")?;
17 | downloader::download_to_tmp(&url).await
18 | }
19 |
20 | #[derive(Debug)]
21 | struct ParsedFile {
22 | rows: Vec>>,
23 | }
24 |
25 | async fn parse() -> Result {
26 | let file = download_admini_boundary_file().await?;
27 | let path = file.path;
28 | let mut workbook: Xlsx<_> = calamine::open_workbook(&path)?;
29 | let sheet = workbook.worksheet_range("行政区域コード")?;
30 | let mut data_started = false;
31 |
32 | let mut out = Vec::new();
33 |
34 | for row in sheet.rows() {
35 | if !data_started {
36 | let first_cell_str = data_to_string(&row[0]);
37 | if first_cell_str.is_some_and(|s| s == "行政区域コード") {
38 | data_started = true;
39 | }
40 | continue;
41 | }
42 |
43 | let row_strings = row
44 | .iter()
45 | .map(|cell| {
46 | let str_opt = data_to_string(cell);
47 | if let Some(s) = str_opt {
48 | if s.is_empty() {
49 | None
50 | } else {
51 | Some(s.nfkc().to_string())
52 | }
53 | } else {
54 | None
55 | }
56 | })
57 | .collect::>>();
58 | if row_strings.iter().all(|s| s.is_none()) {
59 | continue;
60 | }
61 | out.push(row_strings);
62 | }
63 | Ok(ParsedFile { rows: out })
64 | }
65 |
66 | async fn load(postgres_url: &str, parsed: &ParsedFile) -> Result<()> {
67 | let (client, connection) = tokio_postgres::connect(postgres_url, NoTls)
68 | .await
69 | .with_context(|| "when connecting to PostgreSQL")?;
70 |
71 | tokio::spawn(async move {
72 | if let Err(e) = connection.await {
73 | eprintln!("Connection error: {}", e);
74 | }
75 | });
76 |
77 | client
78 | .execute(
79 | r#"
80 | DELETE FROM "admini_boundary_cd";
81 | "#,
82 | &[],
83 | )
84 | .await?;
85 |
86 | let query = r#"
87 | INSERT INTO "admini_boundary_cd" (
88 | "行政区域コード",
89 | "都道府県名（漢字）",
90 | "市区町村名（漢字）",
91 | "都道府県名（カナ）",
92 | "市区町村名（カナ）",
93 | "コードの改定区分",
94 | "改正年月日",
95 | "改正後のコード",
96 | "改正後の名称",
97 | "改正後の名称（カナ）",
98 | "改正事由等"
99 | ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
100 | ON CONFLICT ("行政区域コード") DO NOTHING
101 | "#;
102 | for row in parsed.rows.iter() {
103 | let params: Vec<&(dyn ToSql + Sync)> =
104 | row.iter().map(|v| v as &(dyn ToSql + Sync)).collect();
105 | client.execute(query, ¶ms).await?;
106 | }
107 | Ok(())
108 | }
109 |
110 | async fn create_admini_boundary_metadata(postgres_url: &str) -> Result<()> {
111 | let metadata_conn = MetadataConnection::new(postgres_url).await?;
112 |
113 | let metadata = TableMetadata {
114 | name: "行政区域コード".to_string(),
115 | desc: None,
116 | source: Some("国土数値情報".to_string()),
117 | source_url: Some(
118 | Url::parse("https://nlftp.mlit.go.jp/ksj/gml/codelist/AdminiBoundary_CD.xlsx").unwrap(),
119 | ),
120 | license: None,
121 | license_url: None,
122 | primary_key: Some("行政区域コード".to_string()),
123 | columns: vec![
124 | ColumnMetadata {
125 | name: "行政区域コード".to_string(),
126 | desc: None,
127 | data_type: "varchar".to_string(),
128 | foreign_key: None,
129 | enum_values: None,
130 | },
131 | ColumnMetadata {
132 | name: "都道府県名（漢字）".to_string(),
133 | desc: None,
134 | data_type: "varchar".to_string(),
135 | foreign_key: None,
136 | enum_values: None,
137 | },
138 | ColumnMetadata {
139 | name: "市区町村名（漢字）".to_string(),
140 | desc: None,
141 | data_type: "varchar".to_string(),
142 | foreign_key: None,
143 | enum_values: None,
144 | },
145 | ColumnMetadata {
146 | name: "都道府県名（カナ）".to_string(),
147 | desc: None,
148 | data_type: "varchar".to_string(),
149 | foreign_key: None,
150 | enum_values: None,
151 | },
152 | ColumnMetadata {
153 | name: "市区町村名（カナ）".to_string(),
154 | desc: None,
155 | data_type: "varchar".to_string(),
156 | foreign_key: None,
157 | enum_values: None,
158 | },
159 | ColumnMetadata {
160 | name: "コードの改定区分".to_string(),
161 | desc: None,
162 | data_type: "varchar".to_string(),
163 | foreign_key: None,
164 | enum_values: None,
165 | },
166 | ColumnMetadata {
167 | name: "改正年月日".to_string(),
168 | desc: None,
169 | data_type: "varchar".to_string(),
170 | foreign_key: None,
171 | enum_values: None,
172 | },
173 | ColumnMetadata {
174 | name: "改正後のコード".to_string(),
175 | desc: Some(
176 | "統廃合後の行政区域コード。全国地方公共団体コードに相当する値。".to_string(),
177 | ),
178 | data_type: "varchar".to_string(),
179 | foreign_key: None,
180 | enum_values: None,
181 | },
182 | ColumnMetadata {
183 | name: "改正後の名称".to_string(),
184 | desc: None,
185 | data_type: "varchar".to_string(),
186 | foreign_key: None,
187 | enum_values: None,
188 | },
189 | ColumnMetadata {
190 | name: "改正後の名称（カナ）".to_string(),
191 | desc: None,
192 | data_type: "varchar".to_string(),
193 | foreign_key: None,
194 | enum_values: None,
195 | },
196 | ColumnMetadata {
197 | name: "改正事由等".to_string(),
198 | desc: None,
199 | data_type: "varchar".to_string(),
200 | foreign_key: None,
201 | enum_values: None,
202 | },
203 | ],
204 | };
205 |
206 | metadata_conn
207 | .create_dataset("admini_boundary_cd", &metadata)
208 | .await?;
209 | Ok(())
210 | }
211 |
212 | pub async fn load_admini_boundary(postgres_url: &str) -> Result<()> {
213 | let parsed = parse().await?;
214 | load(postgres_url, &parsed).await?;
215 | create_admini_boundary_metadata(postgres_url).await?;
216 | Ok(())
217 | }
218 |
219 | #[cfg(test)]
220 | mod tests {
221 | use super::*;
222 |
223 | #[tokio::test]
224 | async fn test_download_admini_boundary_file() {
225 | let file = download_admini_boundary_file().await.unwrap();
226 | assert!(file.path.exists());
227 | }
228 |
229 | #[tokio::test]
230 | async fn test_parse_admini() {
231 | let parsed_file = parse().await.unwrap();
232 | assert!(!parsed_file.rows.is_empty());
233 | assert_eq!(parsed_file.rows[0].len(), 11);
234 | assert_eq!(parsed_file.rows[0][0], Some("01000".to_string()));
235 | assert_eq!(parsed_file.rows[0][1], Some("北海道".to_string()));
236 | assert_eq!(parsed_file.rows[0][2], None);
237 | assert_eq!(parsed_file.rows[0][3], Some("ホッカイドウ".to_string()));
238 | }
239 | }
240 |
--------------------------------------------------------------------------------
/src/loader/gdal.rs:
--------------------------------------------------------------------------------
1 | use super::mapping::ShapefileMetadata;
2 | use anyhow::{anyhow, Context, Result};
3 | use encoding_rs::{Encoding, SHIFT_JIS, UTF_8};
4 | use serde_json::Value;
5 | use std::path::PathBuf;
6 | use tokio::process::Command;
7 |
8 | pub async fn create_vrt(
9 | out: &PathBuf,
10 | shapes: &Vec,
11 | metadata: &ShapefileMetadata,
12 | ) -> Result<()> {
13 | if shapes.is_empty() {
14 | anyhow::bail!("No shapefiles found");
15 | }
16 |
17 | let bare_vrt = out.with_extension("");
18 | let layer_name = bare_vrt.file_name().unwrap().to_str().unwrap();
19 | // let vrt_path = shape.with_extension("vrt");
20 |
21 | let mut fields = String::new();
22 | let attributes = get_attribute_list(&shapes[0]).await?;
23 | for (field_name, shape_name) in metadata.field_mappings.iter() {
24 | // ignore attributes in the mapping that are not in the shapefile
25 | if attributes.iter().find(|&attr| attr == shape_name).is_none() {
26 | continue;
27 | }
28 | fields.push_str(&format!(
29 | r#""#,
30 | field_name, shape_name
31 | ));
32 | }
33 | if fields.is_empty() {
34 | anyhow::bail!("No fields found in shapefile");
35 | }
36 |
37 | let mut layers = String::new();
38 | for shape in shapes {
39 | let bare_shape = shape.with_extension("");
40 | let shape_filename = bare_shape.file_name().unwrap().to_str().unwrap();
41 | let encoding = detect_encoding(shape).await?;
42 | layers.push_str(&format!(
43 | r#"
44 |
45 | {}
46 | {}
47 | {}
48 |
49 | "#,
50 | shape_filename,
51 | shape.canonicalize().unwrap().to_str().unwrap(),
52 | encoding,
53 | fields
54 | ));
55 | }
56 |
57 | let vrt = format!(
58 | r#"
59 |
60 |
61 | {}
62 |
63 |
64 | "#,
65 | layer_name, layers
66 | );
67 |
68 | tokio::fs::write(&out, vrt).await?;
69 |
70 | Ok(())
71 | }
72 |
73 | pub async fn load_to_postgres(vrt: &PathBuf, postgres_url: &str) -> Result<()> {
74 | let mut cmd = Command::new("ogr2ogr");
75 | let output = cmd
76 | .arg("-f")
77 | .arg("PostgreSQL")
78 | .arg(format!("PG:{}", postgres_url))
79 | // .arg("-skipfailures")
80 | .arg("-lco")
81 | .arg("GEOM_TYPE=geometry")
82 | .arg("-lco")
83 | .arg("OVERWRITE=YES")
84 | .arg("-lco")
85 | .arg("GEOMETRY_NAME=geom")
86 | .arg("-nlt")
87 | .arg("PROMOTE_TO_MULTI")
88 | .arg("--config")
89 | .arg("PG_USE_COPY=YES")
90 | .arg(vrt)
91 | .output()
92 | .await?;
93 |
94 | if !output.status.success() {
95 | // the error message may contain malformed UTF8
96 | let stderr = String::from_utf8_lossy(&output.stderr);
97 | anyhow::bail!("ogr2ogr failed: {}", stderr);
98 | }
99 |
100 | Ok(())
101 | }
102 |
103 | pub async fn has_layer(postgres_url: &str, layer_name: &str) -> Result {
104 | let layer_name_lower = layer_name.to_lowercase();
105 | let output = Command::new("ogrinfo")
106 | .arg("-if")
107 | .arg("postgresql")
108 | .arg(format!("PG:{}", postgres_url))
109 | .arg("-sql")
110 | .arg(&format!("SELECT 1 FROM \"{}\" LIMIT 1", layer_name_lower))
111 | .output()
112 | .await?;
113 |
114 | Ok(output.status.success())
115 | }
116 |
117 | async fn get_attribute_list(shape: &PathBuf) -> Result> {
118 | let ogrinfo = Command::new("ogrinfo")
119 | .arg("-json")
120 | .arg(shape)
121 | .output()
122 | .await?;
123 |
124 | if !ogrinfo.status.success() {
125 | let stderr = String::from_utf8_lossy(&ogrinfo.stderr);
126 | anyhow::bail!("ogrinfo failed: {}", stderr);
127 | }
128 |
129 | let stdout_str = String::from_utf8_lossy(&ogrinfo.stdout);
130 | let json: Value =
131 | serde_json::from_str(&stdout_str).with_context(|| "when parsing ogrinfo JSON output")?;
132 |
133 | let fields = json
134 | .pointer("/layers/0/fields")
135 | .and_then(Value::as_array)
136 | .ok_or_else(|| anyhow!("missing fields array"))?;
137 |
138 | let attributes: Vec = fields
139 | .iter()
140 | .filter_map(|field| {
141 | field
142 | .pointer("/name")
143 | .and_then(Value::as_str)
144 | .map(String::from)
145 | })
146 | .collect();
147 |
148 | Ok(attributes)
149 | }
150 |
151 | // PC932 is almost the same as Shift-JIS, but most GIS software outputs as CP932 when using Shift-JIS
152 | static ENCODINGS: &[(&str, &Encoding)] = &[("CP932", SHIFT_JIS), ("UTF-8", UTF_8)];
153 |
154 | // We get the bytes from the ogrinfo output after "successful"
155 | // this is because before "successful" is the filename, and the filename
156 | // can contain non-UTF8 characters
157 | fn bytes_after_successful(data: &Vec) -> Option<&[u8]> {
158 | let needle = b"successful"; // equivalent to "successful".as_bytes()
159 | data.windows(needle.len())
160 | .position(|window| window == needle)
161 | .map(|pos| &data[pos + needle.len()..])
162 | }
163 |
164 | async fn detect_encoding_fallback(shape: &PathBuf) -> Result> {
165 | let ogrinfo = Command::new("ogrinfo")
166 | .arg("-al")
167 | .arg("-geom=NO")
168 | .arg("-limit")
169 | .arg("100")
170 | .arg(shape)
171 | .output()
172 | .await?;
173 |
174 | if !ogrinfo.status.success() {
175 | let stderr = String::from_utf8_lossy(&ogrinfo.stderr);
176 | anyhow::bail!("ogrinfo failed: {}", stderr);
177 | }
178 |
179 | let Some(data) = bytes_after_successful(&ogrinfo.stdout) else {
180 | anyhow::bail!("ogrinfo failed to open {}", shape.display());
181 | };
182 | for (name, encoding) in ENCODINGS {
183 | // decode() returns a tuple: (decoded string, bytes read, had_errors)
184 | let (_decoded, _, had_errors) = encoding.decode(data);
185 | if !had_errors {
186 | return Ok(Some(name.to_string()));
187 | }
188 | }
189 |
190 | Ok(None)
191 | }
192 |
193 | async fn detect_encoding_ogrinfo(shape: &PathBuf) -> Result> {
194 | let ogrinfo = Command::new("ogrinfo")
195 | .arg("-json")
196 | .arg(shape)
197 | .output()
198 | .await?;
199 |
200 | if !ogrinfo.status.success() {
201 | let stderr = String::from_utf8_lossy(&ogrinfo.stderr);
202 | anyhow::bail!("ogrinfo failed: {}", stderr);
203 | }
204 |
205 | let stdout_str = String::from_utf8_lossy(&ogrinfo.stdout);
206 | let json: Value =
207 | serde_json::from_str(&stdout_str).with_context(|| "when parsing ogrinfo JSON output")?;
208 |
209 | let source_encoding = json
210 | .pointer("/layers/0/metadata/SHAPEFILE/SOURCE_ENCODING")
211 | .and_then(Value::as_str);
212 |
213 | if let Some(encoding) = source_encoding {
214 | if encoding == "" {
215 | return Ok(None);
216 | }
217 | return Ok(Some(encoding.to_string()));
218 | }
219 |
220 | Ok(None)
221 | }
222 |
223 | pub async fn detect_encoding(shape: &PathBuf) -> Result {
224 | let encoding = detect_encoding_ogrinfo(shape).await?;
225 | if let Some(encoding) = encoding {
226 | return Ok(encoding);
227 | }
228 |
229 | if let Some(encoding) = detect_encoding_fallback(shape).await? {
230 | return Ok(encoding);
231 | }
232 |
233 | anyhow::bail!("Failed to detect encoding for {}", shape.display());
234 | }
235 |
236 | #[cfg(test)]
237 | mod tests {
238 |
239 | #[tokio::test]
240 | async fn test_detect_encoding() {
241 | let shape = std::path::PathBuf::from("./test_data/shp/cp932.shp");
242 | let encoding = super::detect_encoding(&shape).await.unwrap();
243 | assert_eq!(encoding, "CP932");
244 |
245 | let shape = std::path::PathBuf::from("./test_data/shp/src_blank.shp");
246 | let encoding = super::detect_encoding(&shape).await.unwrap();
247 | assert_eq!(encoding, "CP932");
248 | }
249 |
250 | #[tokio::test]
251 | async fn test_get_attribute_list() {
252 | let shape = std::path::PathBuf::from("./test_data/shp/cp932.shp");
253 | let attributes = super::get_attribute_list(&shape).await.unwrap();
254 | assert_eq!(attributes, vec!["W09_001", "W09_002", "W09_003", "W09_004"]);
255 | }
256 | }
257 |
--------------------------------------------------------------------------------
/src/loader/load_queue.rs:
--------------------------------------------------------------------------------
1 | use crate::context;
2 | use crate::loader::gdal;
3 | use crate::loader::{mapping, zip_traversal};
4 | use crate::metadata::MetadataConnection;
5 | use crate::scraper::Dataset;
6 | use anyhow::Result;
7 | use async_channel::unbounded;
8 | use indicatif::{ProgressBar, ProgressStyle};
9 | use std::cmp::max;
10 | use std::path::PathBuf;
11 | use std::time::Duration;
12 | use tokio::task;
13 |
14 | use super::Loader;
15 |
16 | async fn load(
17 | dataset: &Dataset,
18 | postgres_url: &str,
19 | skip_if_exists: bool,
20 | metadata_conn: &MetadataConnection,
21 | ) -> Result<()> {
22 | let tmp = context::tmp();
23 | let vrt_tmp = tmp.join("vrt");
24 | tokio::fs::create_dir_all(&vrt_tmp).await?;
25 |
26 | let identifier = &dataset.initial_item.identifier;
27 |
28 | // first, let's get the entries for this dataset from the mapping file
29 | let mappings = mapping::find_mapping_def_for_entry(&identifier).await?;
30 |
31 | for mapping in mappings {
32 | // overwrite the identifier with the one from the mapping file
33 | let identifier = mapping.identifier.clone().to_lowercase();
34 | // println!(
35 | // "Loading dataset: {} - {} - {} as {}",
36 | // mapping.cat1, mapping.cat2, mapping.name, mapping.identifier
37 | // );
38 |
39 | let mut shapefiles: Vec = Vec::new();
40 | for zip_file_path in &dataset.zip_file_paths {
41 | let shapefiles_in_zip =
42 | zip_traversal::matching_shapefiles_in_zip(tmp, zip_file_path, &mapping).await?;
43 | shapefiles.extend(shapefiles_in_zip);
44 | }
45 |
46 | // println!("Found {} shapefiles: {:?}", shapefiles.len(), shapefiles);
47 |
48 | let has_layer = gdal::has_layer(postgres_url, &mapping.identifier).await?;
49 | if skip_if_exists && has_layer {
50 | println!("Table already exists for {}, skipping", mapping.identifier);
51 | } else {
52 | let vrt_path = vrt_tmp.join(&identifier).with_extension("vrt");
53 | gdal::create_vrt(&vrt_path, &shapefiles, &mapping).await?;
54 | gdal::load_to_postgres(&vrt_path, postgres_url).await?;
55 | }
56 |
57 | let metadata = metadata_conn
58 | .build_metadata_from_dataset(&identifier, &mapping, dataset)
59 | .await?;
60 | // println!("Metadata: {:?}", metadata);
61 | metadata_conn.create_dataset(&identifier, &metadata).await?;
62 | }
63 | Ok(())
64 | }
65 |
66 | struct PBStatusUpdateMsg {
67 | added: u64,
68 | finished: u64,
69 | msg: Option,
70 | }
71 |
72 | pub struct LoadQueue {
73 | pb_status_sender: Option>,
74 | sender: Option>,
75 |
76 | set: Option>,
77 | }
78 |
79 | impl LoadQueue {
80 | pub async fn new(loader: &Loader) -> Result {
81 | let Loader {
82 | postgres_url,
83 | skip_if_exists,
84 | ..
85 | } = loader;
86 |
87 | let metadata_conn = MetadataConnection::new(postgres_url).await?;
88 |
89 | let (pb_status_sender, pb_status_receiver) = unbounded::();
90 | let (sender, receiver) = unbounded::();
91 | let mut set = task::JoinSet::new();
92 | let size = max(num_cpus::get() - 1, 1);
93 | for _i in 0..size {
94 | let receiver = receiver.clone();
95 | let pb_sender = pb_status_sender.clone();
96 | let postgres_url = postgres_url.to_string();
97 | let skip_if_exists = *skip_if_exists;
98 | let metadata_conn = metadata_conn.clone();
99 | set.spawn(async move {
100 | while let Ok(item) = receiver.recv().await {
101 | // println!("processor {} loading", _i);
102 | pb_sender
103 | .send(PBStatusUpdateMsg {
104 | added: 0,
105 | finished: 0,
106 | msg: Some(item.initial_item.identifier.clone()),
107 | })
108 | .await
109 | .unwrap();
110 | let result = load(&item, &postgres_url, skip_if_exists, &metadata_conn).await;
111 | if let Err(e) = result {
112 | let identifier = item.initial_item.identifier.clone();
113 | eprintln!(
114 | "Error in loading dataset {}, skipping... {:?}",
115 | identifier, e
116 | );
117 | }
118 | pb_sender
119 | .send(PBStatusUpdateMsg {
120 | added: 0,
121 | finished: 1,
122 | msg: Some(item.initial_item.identifier.clone()),
123 | })
124 | .await
125 | .unwrap();
126 | }
127 | });
128 | }
129 |
130 | set.spawn(async move {
131 | let pb = ProgressBar::new(0);
132 | pb.set_style(
133 | ProgressStyle::with_template(
134 | "{spinner:.green} [{msg}] [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len}",
135 | )
136 | .unwrap()
137 | .progress_chars("=>-"),
138 | );
139 | pb.enable_steady_tick(Duration::from_millis(300));
140 | let mut length = 0;
141 | let mut position = 0;
142 | while let Ok(msg) = pb_status_receiver.recv().await {
143 | length += msg.added;
144 | position += msg.finished;
145 | pb.set_length(length);
146 | pb.set_position(position);
147 | if let Some(msg) = msg.msg {
148 | pb.set_message(msg);
149 | }
150 | }
151 | pb.finish();
152 | println!("DB取り組みが終了しました。");
153 | });
154 |
155 | Ok(Self {
156 | pb_status_sender: Some(pb_status_sender),
157 | sender: Some(sender),
158 | set: Some(set),
159 | })
160 | }
161 |
162 | pub async fn push(&self, item: &Dataset) -> Result<()> {
163 | let Some(sender) = &self.sender else {
164 | return Err(anyhow::anyhow!("LoadQueue is already closed"));
165 | };
166 | let Some(pb_status_sender) = &self.pb_status_sender else {
167 | return Err(anyhow::anyhow!("LoadQueue is already closed"));
168 | };
169 | pb_status_sender
170 | .send(PBStatusUpdateMsg {
171 | added: 1,
172 | finished: 0,
173 | msg: None,
174 | })
175 | .await?;
176 | sender.send(item.clone()).await?;
177 | Ok(())
178 | }
179 |
180 | pub async fn close(&mut self) -> Result<()> {
181 | let Some(_) = self.sender.take() else {
182 | return Err(anyhow::anyhow!("LoadQueue is already closed"));
183 | };
184 | let Some(set) = self.set.take() else {
185 | return Err(anyhow::anyhow!("LoadQueue is already closed"));
186 | };
187 | let Some(_) = self.pb_status_sender.take() else {
188 | return Err(anyhow::anyhow!("LoadQueue is already closed"));
189 | };
190 | set.join_all().await;
191 | Ok(())
192 | }
193 | }
194 |
--------------------------------------------------------------------------------
/src/loader/mapping.rs:
--------------------------------------------------------------------------------
1 | use anyhow::Result;
2 | use calamine::{Reader, Xlsx};
3 | use derive_builder::Builder;
4 | use regex::Regex;
5 | use std::vec;
6 | use tokio::sync::OnceCell;
7 | use url::Url;
8 |
9 | use crate::downloader;
10 |
11 | use super::xslx_helpers::data_to_string;
12 |
13 | #[derive(Builder, Clone, Debug)]
14 | #[builder(derive(Debug))]
15 | pub struct ShapefileMetadata {
16 | #[allow(dead_code)]
17 | pub cat1: String, // 4. 交通
18 | #[allow(dead_code)]
19 | pub cat2: String, // 交通
20 | pub name: String, // 鉄道時系列（ライン）
21 | #[allow(dead_code)]
22 | pub version: String, // 2023年度版
23 | #[allow(dead_code)]
24 | pub data_year: String, // 令和5年度
25 |
26 | /// シェープファイル名
27 | /// （表記中のYYは年次、MMは月、PPは都道府県コード、CCCCCは市区町村コード、AAは支庁コード、mmmmはメッシュコードを示します。）
28 | #[allow(dead_code)]
29 | #[builder(default = "vec![]")]
30 | pub shapefile_matcher: Vec,
31 | // // parsed version of shapefile_matcher; computed from shapefile_matcher
32 | #[builder(setter(skip), default = "self.create_shapefile_name_regex()?")]
33 | pub shapefile_name_regex: Vec,
34 |
35 | pub field_mappings: Vec<(String, String)>,
36 |
37 | /// 元データの識別子
38 | /// インポート識別子はインポート後のテーブル名になります。これは、単一データセットに対して複数テーブルとして扱う場合に必要です。
39 | pub original_identifier: String,
40 | /// インポート識別子
41 | pub identifier: String,
42 | }
43 |
44 | // fn create_shapefile_name_regex(_template_string: String) -> Result {
45 | // Regex::new(r"(?i:(?:\.shp|\.cpg|\.dbf|\.prj|\.qmd|\.shx))$").map_err(|e| e.to_string())
46 | // }
47 |
48 | fn format_name(name: &str) -> String {
49 | let mut formatted_name = name.to_string();
50 | // Remove any parentheses and their contents
51 | let remove_re = Regex::new(r"（[^）]+）").unwrap();
52 | formatted_name = remove_re.replace_all(&formatted_name, "").to_string();
53 | // Remove leading and trailing whitespace
54 | formatted_name = formatted_name.trim().to_string();
55 | formatted_name
56 | }
57 |
58 | fn create_shapefile_name_regex(template_string: String) -> Result {
59 | let remove_re = Regex::new(r"（[^）]+）").unwrap();
60 | let template = remove_re.replace_all(template_string.as_str(), "");
61 | let template = template.trim();
62 |
63 | // If the template ends with ".shp" (case‑insensitive), remove it.
64 | let base_template = if template.to_lowercase().ends_with(".shp") {
65 | // Remove the last 4 characters (".shp")
66 | &template[..template.len() - 4]
67 | } else {
68 | &template
69 | };
70 |
71 | // This regex matches only the allowed placeholders.
72 | // Note: We deliberately list the tokens so that only these are replaced.
73 | let token_pattern = r"(YY|MM|PP|CCCCC|AA|mmmm)";
74 | let re = Regex::new(token_pattern).unwrap();
75 |
76 | let mut result = String::from("(?:^|/)");
77 | let mut last_index = 0;
78 |
79 | // Iterate over each found token in the template.
80 | for mat in re.find_iter(base_template) {
81 | // Escape and append the literal text before the token.
82 | let escaped = regex::escape(&base_template[last_index..mat.start()]);
83 | result.push_str(&escaped);
84 |
85 | // Use the length of the token to determine the number of digits.
86 | let token = mat.as_str();
87 | let replacement = format!("\\d{{{}}}", token.len());
88 | result.push_str(&replacement);
89 |
90 | last_index = mat.end();
91 | }
92 |
93 | // Append and escape any trailing literal text.
94 | result.push_str(®ex::escape(&base_template[last_index..]));
95 | result.push_str(r"(?i:(?:\.shp|\.cpg|\.dbf|\.prj|\.qmd|\.shx))$");
96 |
97 | Regex::new(&result).map_err(|e| e.to_string())
98 | }
99 |
100 | impl ShapefileMetadataBuilder {
101 | fn create_shapefile_name_regex(&self) -> Result, String> {
102 | match &self.shapefile_matcher {
103 | None => {
104 | return Ok(vec![Regex::new(
105 | r"(?i:(?:\.shp|\.cpg|\.dbf|\.prj|\.qmd|\.shx))$",
106 | )
107 | .unwrap()]);
108 | }
109 | Some(ref template_strings) => template_strings
110 | .iter()
111 | .map(|s| create_shapefile_name_regex(s.clone()))
112 | .collect(),
113 | }
114 | }
115 | }
116 |
117 | /// Splits and normalizes a shapefile matcher string into a vector of strings.
118 | fn split_shapefile_matcher(s: &str) -> Vec {
119 | s.replace("\r\n", "\n")
120 | .replace("A38-YY_PP_", "A38-YY_") // 医療圏のshapefile名が間違っている
121 | .split('\n')
122 | .map(|s| s.trim().to_string())
123 | .filter(|s| !s.is_empty())
124 | .collect()
125 | }
126 |
127 | fn should_start_new_metadata_record(
128 | builder: &ShapefileMetadataBuilder,
129 | row: &[calamine::Data],
130 | ) -> bool {
131 | let cat1 = data_to_string(&row[0]);
132 | let cat2 = data_to_string(&row[1]);
133 | let shapefile_names = data_to_string(&row[5])
134 | .map(|s| split_shapefile_matcher(&s))
135 | .and_then(|v| if v.is_empty() { None } else { Some(v) });
136 | let original_identifier = data_to_string(&row[8]);
137 | let mapping_id = data_to_string(&row[7]);
138 | if let (Some(cat1), Some(cat2), Some(shapefile_names)) = (cat1, cat2, shapefile_names) {
139 | if builder.cat1.clone().is_some_and(|s| s != cat1)
140 | || builder.cat2.clone().is_some_and(|s| s != cat2)
141 | || builder
142 | .shapefile_matcher
143 | .clone()
144 | .is_some_and(|s| s != shapefile_names)
145 | {
146 | return true;
147 | }
148 | }
149 | if let (Some(identifier), Some(mapping_id)) = (original_identifier, mapping_id) {
150 | // 例外: 医療圏。１，２，３次医療圏はそれぞれ別テーブルとして扱う。
151 | // -> 識別子はそれぞれA38だが、属性コードの頭4文字が異なる（A38a, A38b, A38c）
152 | if identifier == "A38"
153 | && builder.field_mappings.as_ref().is_some_and(|m| {
154 | m.last().is_some_and(|(_, prev_mapping_id)| {
155 | !mapping_id.starts_with(&prev_mapping_id.chars().take(4).collect::())
156 | })
157 | })
158 | {
159 | return true;
160 | }
161 | }
162 | false
163 | }
164 |
165 | fn extract_identifier_from_row(row: &[calamine::Data]) -> Option {
166 | let identifier = data_to_string(&row[8]);
167 | if let Some(ref id) = identifier {
168 | if id == "A38" {
169 | if let Some(mapping_id) = data_to_string(&row[7]) {
170 | // Take the first 4 characters of mapping_id, or the whole string if shorter
171 | return Some(mapping_id.chars().take(4).collect());
172 | }
173 | }
174 | }
175 | identifier
176 | }
177 |
178 | async fn download_mapping_definition_file() -> Result {
179 | let url = Url::parse("https://nlftp.mlit.go.jp/ksj/gml/codelist/shape_property_table2.xlsx")?;
180 | downloader::download_to_tmp(&url).await
181 | }
182 |
183 | async fn parse_mapping_file() -> Result> {
184 | let file = download_mapping_definition_file().await?;
185 | let path = file.path;
186 | let mut workbook: Xlsx<_> = calamine::open_workbook(&path)?;
187 | let mut out: Vec = Vec::new();
188 | let sheet = workbook.worksheet_range("全データ")?;
189 | let mut data_started = false;
190 |
191 | let mut builder = ShapefileMetadataBuilder::default();
192 | for row in sheet.rows() {
193 | let cat1 = data_to_string(&row[0]);
194 |
195 | if !data_started {
196 | if cat1.is_some_and(|s| s == "大分類") {
197 | data_started = true;
198 | }
199 | continue;
200 | }
201 |
202 | if should_start_new_metadata_record(&builder, row) {
203 | match builder.build() {
204 | Ok(metadata) => out.push(metadata),
205 | Err(e) => panic!("Error: {}, {:?}", e, builder),
206 | }
207 | builder = ShapefileMetadataBuilder::default();
208 | }
209 |
210 | if let Some(original_identifier) = data_to_string(&row[8]) {
211 | if builder.original_identifier.is_none() {
212 | builder.original_identifier(original_identifier);
213 | }
214 | }
215 | if let Some(identifier) = extract_identifier_from_row(row) {
216 | if builder.identifier.is_none() {
217 | builder.identifier(identifier.clone());
218 | }
219 |
220 | let name_override = match identifier.as_str() {
221 | "A38a" => Some("一次医療圏"),
222 | "A38b" => Some("二次医療圏"),
223 | "A38c" => Some("三次医療圏"),
224 | _ => None,
225 | };
226 | if let Some(name) = name_override {
227 | builder.name(name.to_string());
228 | }
229 | }
230 |
231 | if let Some(cat1) = cat1 {
232 | builder.cat1(cat1);
233 | }
234 | if let Some(cat2) = data_to_string(&row[1]) {
235 | builder.cat2(cat2);
236 | }
237 | if let Some(name) = data_to_string(&row[2]) {
238 | if builder.name.is_none() {
239 | builder.name(format_name(&name));
240 | }
241 | }
242 | if let Some(version) = data_to_string(&row[3]) {
243 | builder.version(version);
244 | }
245 | if let Some(data_year) = data_to_string(&row[4]) {
246 | builder.data_year(data_year);
247 | }
248 | if let Some(shapefile_matcher) = data_to_string(&row[5]) {
249 | let mut matchers = builder.shapefile_matcher.clone().unwrap_or(vec![]);
250 | matchers.extend(split_shapefile_matcher(&shapefile_matcher));
251 | builder.shapefile_matcher(matchers);
252 | }
253 |
254 | if let Some((field_name, shape_name)) = data_to_string(&row[6]).zip(data_to_string(&row[7]))
255 | {
256 | let mut mappings = builder.field_mappings.clone().unwrap_or(vec![]);
257 | mappings.push((field_name, shape_name));
258 | builder.field_mappings(mappings);
259 | }
260 | }
261 |
262 | // last row
263 | if let Ok(metadata) = builder.build() {
264 | out.push(metadata);
265 | }
266 |
267 | Ok(out)
268 | }
269 |
270 | static MAPPING_DEFS: OnceCell> = OnceCell::const_new();
271 | pub async fn mapping_defs() -> Result<&'static Vec> {
272 | MAPPING_DEFS
273 | .get_or_try_init(|| async { parse_mapping_file().await })
274 | .await
275 | }
276 |
277 | pub async fn find_mapping_def_for_entry(identifier: &str) -> Result> {
278 | let defs = mapping_defs().await?;
279 | Ok(defs
280 | .iter()
281 | .filter(|def| def.original_identifier == identifier)
282 | .cloned()
283 | .collect::>())
284 | }
285 |
286 | #[cfg(test)]
287 | mod tests {
288 | use super::*;
289 |
290 | #[tokio::test]
291 | async fn test_parse_mapping_file() {
292 | let result = parse_mapping_file().await;
293 | assert!(result.is_ok());
294 | let data = &result.unwrap();
295 |
296 | let metadata = &data[0];
297 | assert_eq!(metadata.cat1, "2. 政策区域");
298 | assert_eq!(metadata.cat2, "大都市圏・条件不利地域");
299 | assert_eq!(metadata.name, "三大都市圏計画区域");
300 | assert_eq!(metadata.version, "2003年度版");
301 | assert_eq!(metadata.data_year, "平成15年度");
302 | assert_eq!(
303 | metadata.shapefile_matcher,
304 | vec!["A03-YY_SYUTO-g_ThreeMajorMetroPlanArea.shp"]
305 | );
306 | assert_eq!(metadata.field_mappings.len(), 8);
307 |
308 | // find 医療圏
309 | let metadata = data
310 | .iter()
311 | .filter(|m| m.name.contains("医療圏"))
312 | .collect::>();
313 | assert_eq!(metadata.len(), 3);
314 | assert_eq!(
315 | metadata.iter().map(|m| m.name.clone()).collect::>(),
316 | vec!["一次医療圏", "二次医療圏", "三次医療圏"]
317 | );
318 | }
319 | }
320 |
--------------------------------------------------------------------------------
/src/loader/mod.rs:
--------------------------------------------------------------------------------
1 | // The loader module is responsible for loading data from ZIP files and in to the database.
2 |
3 | use crate::scraper::Dataset;
4 | use anyhow::Result;
5 | use derive_builder::Builder;
6 |
7 | mod admini_boundary;
8 | mod gdal;
9 | mod load_queue;
10 | pub mod mapping;
11 | mod xslx_helpers;
12 | mod zip_traversal;
13 |
14 | #[derive(Builder)]
15 | pub struct Loader {
16 | datasets: Vec,
17 | postgres_url: String,
18 | skip_if_exists: bool,
19 | }
20 |
21 | impl Loader {
22 | pub async fn load_all(self) -> Result<()> {
23 | let mut load_queue = load_queue::LoadQueue::new(&self).await?;
24 | for dataset in self.datasets {
25 | load_queue.push(&dataset).await?;
26 | }
27 | load_queue.close().await?;
28 | admini_boundary::load_admini_boundary(&self.postgres_url).await?;
29 | Ok(())
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/loader/xslx_helpers.rs:
--------------------------------------------------------------------------------
1 | use calamine::{Data, DataType as _};
2 |
3 | pub fn data_to_string(data: &Data) -> Option {
4 | data.get_string()
5 | .map(|s| s.trim())
6 | .filter(|s| !s.is_empty())
7 | .map(|s| s.to_string())
8 | }
9 |
--------------------------------------------------------------------------------
/src/loader/zip_traversal.rs:
--------------------------------------------------------------------------------
1 | // the module responsible for opening ZIP files and traversing them.
2 | // sometimes, zip files are inside zip files, so when a zip file is encountered, we have to recursively traverse it.
3 | // only extracts shapefiles, to a temporary directory, so ogr2ogr can load them directly to the database.
4 |
5 | use super::mapping::ShapefileMetadata;
6 | use anyhow::{Context, Result};
7 | use regex::Regex;
8 | use std::{fs::File, path::PathBuf};
9 | use zip::ZipArchive;
10 |
11 | fn extract_zip(
12 | outdir: &PathBuf,
13 | zip_path: &PathBuf,
14 | matchers: &Vec,
15 | ) -> Result> {
16 | let mut out = vec![];
17 | let file = File::open(zip_path)?;
18 | let zip_filename = zip_path.file_name().unwrap().to_str().unwrap();
19 | let outdir = outdir.join(zip_filename).with_extension("");
20 | let mut zip = ZipArchive::new(file)?;
21 | // println!("Matchers: {:?}", matchers);
22 | for i in 0..zip.len() {
23 | let mut file = zip.by_index(i)?;
24 | // replace Windows backslashes with forward slashes
25 | let file_name = file.name().to_string().replace("\\", "/");
26 | let dest_path = outdir.join(&file_name);
27 | let basedir = dest_path.parent().unwrap();
28 |
29 | // println!("Extracting: {}", file_name);
30 | if file_name.ends_with(".zip") {
31 | std::fs::create_dir_all(&basedir)?;
32 | std::io::copy(&mut file, &mut File::create(&dest_path)?)?;
33 | out.extend(
34 | extract_zip(&outdir, &dest_path, &matchers)
35 | .with_context(|| format!("when extracting nested {}", dest_path.display()))?,
36 | );
37 | } else if matchers.iter().any(|r| r.is_match(&file_name)) {
38 | if file_name.starts_with("N08-21_GML/utf8/") {
39 | // skip this file, it's a duplicate and contains malformed UTF8
40 | continue;
41 | }
42 | std::fs::create_dir_all(&basedir)?;
43 | std::io::copy(&mut file, &mut File::create(&dest_path)?)?;
44 | out.push(dest_path);
45 | }
46 | }
47 | Ok(out)
48 | }
49 |
50 | pub async fn matching_shapefiles_in_zip(
51 | tmp: &PathBuf,
52 | zip_path: &PathBuf,
53 | mapping: &ShapefileMetadata,
54 | ) -> Result> {
55 | let shp_tmp = tmp.join("shp");
56 | tokio::fs::create_dir_all(&shp_tmp).await?;
57 | let matchers = mapping.shapefile_name_regex.clone();
58 | let zip_path = zip_path.clone();
59 |
60 | let mut all_paths = {
61 | let shp_tmp = shp_tmp.clone();
62 | let zip_path = zip_path.clone();
63 | if mapping.identifier == "A33" {
64 | // A33 shapefiles don't match the regex, so we'll skip this part and
65 | // fall back to the expanded matchers, focusing on Polygon files only
66 | let expanded_matchers = vec![Regex::new(
67 | r"Po?lygon(?i:(?:\.shp|\.cpg|\.dbf|\.prj|\.qmd|\.shx))$",
68 | )?];
69 |
70 | tokio::task::spawn_blocking(move || {
71 | extract_zip(&shp_tmp, &zip_path, &expanded_matchers)
72 | .with_context(|| format!("when extracting {}", zip_path.display()))
73 | })
74 | .await??
75 | } else {
76 | tokio::task::spawn_blocking(move || {
77 | extract_zip(&shp_tmp, &zip_path, &matchers)
78 | .with_context(|| format!("when extracting {}", zip_path.display()))
79 | })
80 | .await??
81 | }
82 | };
83 |
84 | if all_paths.is_empty() {
85 | println!("No shapefiles found in zip file, expanding matchers...");
86 | // since we didn't get any shapefiles this time, let's expand the matchers to see if we can find any
87 | let expanded_matchers = vec![Regex::new(
88 | r"(?i:(?:\.shp|\.cpg|\.dbf|\.prj|\.qmd|\.shx))$",
89 | )?];
90 |
91 | all_paths = tokio::task::spawn_blocking(move || {
92 | extract_zip(&shp_tmp, &zip_path, &expanded_matchers)
93 | .with_context(|| format!("when extracting {}", zip_path.display()))
94 | })
95 | .await??;
96 | }
97 |
98 | // at this point, we have decompressed all shapefiles (and accompanying files)
99 | // however, we only need the `.shp` files for passing to ogr2ogr
100 | let shapefile_paths = all_paths
101 | .iter()
102 | .filter(|p| p.extension().unwrap() == "shp")
103 | .cloned()
104 | .collect::>();
105 |
106 | // println!(
107 | // "Found {} shapefiles: \n{}",
108 | // shapefile_paths.len(),
109 | // shapefile_paths
110 | // .iter()
111 | // .map(|s| format!("- {}", s.display()))
112 | // .collect::>()
113 | // .join("\n")
114 | // );
115 |
116 | Ok(shapefile_paths)
117 | }
118 |
119 | #[cfg(test)]
120 | mod tests {
121 | use super::*;
122 | use std::path::PathBuf;
123 |
124 | #[tokio::test]
125 | async fn test_matching_shapefiles_in_zip() {
126 | let tmp = PathBuf::from("./tmp");
127 | let zip = PathBuf::from("./test_data/zip/A30a5-11_4939-jgd_GML.zip");
128 | let mapping = ShapefileMetadata {
129 | cat1: "cat1".to_string(),
130 | cat2: "cat2".to_string(),
131 | name: "name".to_string(),
132 | version: "version".to_string(),
133 | data_year: "data_year".to_string(),
134 | shapefile_matcher: vec!["A30a5-YY_mmmm_SedimentDisasterAndSnowslide.shp".to_string()],
135 | field_mappings: vec![],
136 | original_identifier: "original_identifier".to_string(),
137 | identifier: "identifier".to_string(),
138 | shapefile_name_regex: vec![Regex::new(
139 | r"A30a5-\d{2}_\d{4}_SedimentDisasterAndSnowslide(?i:(?:\.shp|\.cpg|\.dbf|\.prj|\.qmd|\.shx))$",
140 | )
141 | .unwrap()],
142 | };
143 | let result = matching_shapefiles_in_zip(&tmp, &zip, &mapping).await;
144 | assert!(result.is_ok());
145 | let _ = result.unwrap();
146 | }
147 |
148 | #[tokio::test]
149 | async fn test_matching_shapefiles_in_zip_subdir() {
150 | let tmp = PathBuf::from("./tmp");
151 | let zip = PathBuf::from("./test_data/zip/P23-12_38_GML.zip");
152 | let mapping = ShapefileMetadata {
153 | cat1: "cat1".to_string(),
154 | cat2: "cat2".to_string(),
155 | name: "name".to_string(),
156 | version: "version".to_string(),
157 | data_year: "data_year".to_string(),
158 | shapefile_matcher: vec!["P23a-YY_PP.shp".to_string()],
159 | field_mappings: vec![],
160 | original_identifier: "original_identifier".to_string(),
161 | identifier: "identifier".to_string(),
162 | shapefile_name_regex: vec![Regex::new(
163 | r"(?:^|/)P23a-\d{2}_\d{2}(?i:(?:\.shp|\.cpg|\.dbf|\.prj|\.qmd|\.shx))$",
164 | )
165 | .unwrap()],
166 | };
167 | let result = matching_shapefiles_in_zip(&tmp, &zip, &mapping).await;
168 | assert!(result.is_ok());
169 | let _ = result.unwrap();
170 | }
171 | }
172 |
--------------------------------------------------------------------------------
/src/main.rs:
--------------------------------------------------------------------------------
1 | #![warn(unused_extern_crates)]
2 |
3 | use anyhow::{Context, Result};
4 |
5 | mod cli;
6 | mod context;
7 | mod downloader;
8 | mod loader;
9 | mod metadata;
10 | mod scraper;
11 |
12 | #[tokio::main]
13 | async fn main() -> Result<()> {
14 | let args = cli::main();
15 | if let Some(tmp) = args.tmp_dir {
16 | context::set_tmp(tmp);
17 | }
18 | tokio::fs::create_dir_all(context::tmp()).await?;
19 |
20 | // Download all files first
21 | let scraper = scraper::ScraperBuilder::default()
22 | .skip_dl(args.skip_download)
23 | .filter_identifiers(args.filter_identifiers.clone())
24 | .build()
25 | .context("while building scraper")?;
26 | let datasets = scraper
27 | .download_all()
28 | .await
29 | .with_context(|| format!("while downloading initial data"))?;
30 |
31 | let loader = loader::LoaderBuilder::default()
32 | .datasets(datasets)
33 | .postgres_url(args.postgres_url.clone())
34 | .skip_if_exists(args.skip_sql_if_exists)
35 | .build()
36 | .context("while building loader")?;
37 | loader
38 | .load_all()
39 | .await
40 | .with_context(|| "while loading datasets")?;
41 |
42 | Ok(())
43 | }
44 |
--------------------------------------------------------------------------------
/src/metadata.rs:
--------------------------------------------------------------------------------
1 | use crate::{loader::mapping::ShapefileMetadata, scraper::Dataset};
2 | use anyhow::{Context, Result};
3 | use km_to_sql::{
4 | metadata::{ColumnEnumDetails, ColumnForeignKeyDetails, ColumnMetadata, TableMetadata},
5 | postgres::{init_schema, upsert},
6 | };
7 | use std::sync::Arc;
8 | use tokio_postgres::{Client, NoTls};
9 |
10 | const INIT_SQL: &str = include_str!("../data/schema.sql");
11 |
12 | #[derive(Clone)]
13 | pub struct MetadataConnection {
14 | client: Arc,
15 | }
16 |
17 | impl MetadataConnection {
18 | pub async fn new(connection_str: &str) -> Result {
19 | let (client, connection) = tokio_postgres::connect(connection_str, NoTls)
20 | .await
21 | .with_context(|| "when connecting to PostgreSQL")?;
22 | tokio::spawn(async move {
23 | if let Err(e) = connection.await {
24 | panic!("PostgreSQL connection error: {}", e);
25 | }
26 | });
27 |
28 | client
29 | .batch_execute(INIT_SQL)
30 | .await
31 | .with_context(|| "when initializing PostgreSQL schema")?;
32 | init_schema(&client).await?;
33 |
34 | Ok(MetadataConnection {
35 | client: Arc::new(client),
36 | })
37 | }
38 |
39 | pub async fn build_metadata_from_dataset(
40 | &self,
41 | table_name: &str,
42 | metadata: &ShapefileMetadata,
43 | dataset: &Dataset,
44 | ) -> Result {
45 | let columns_in_db = self
46 | .client
47 | .query(
48 | r#"
49 | SELECT
50 | cols.ordinal_position AS position,
51 | cols.column_name,
52 | cols.udt_name AS underlying_type,
53 | gc.type AS geometry_type,
54 | gc.srid AS geometry_srid
55 | FROM information_schema.columns cols
56 | LEFT JOIN public.geometry_columns gc
57 | ON gc.f_table_schema = cols.table_schema
58 | AND gc.f_table_name = cols.table_name
59 | AND gc.f_geometry_column = cols.column_name
60 | WHERE cols.table_schema = 'public'
61 | AND cols.table_name = $1
62 | ORDER BY cols.ordinal_position
63 | "#,
64 | &[&table_name],
65 | )
66 | .await
67 | .with_context(|| "when querying columns from PostgreSQL")?;
68 |
69 | // println!("[table: {}] Columns in DB: {:?}", table_name, columns_in_db);
70 |
71 | let data_item = &dataset.initial_item;
72 | let data_page = &dataset.page;
73 |
74 | let dp_col_vec: Vec<_> = data_page.metadata.attribute.clone().into_values().collect();
75 |
76 | let mut columns: Vec = vec![];
77 | for db_column in columns_in_db {
78 | let column_name: String = db_column.get(1);
79 | let column_type: String = db_column.get(2);
80 | let geometry_type: Option = db_column.get(3);
81 | let geometry_srid: Option = db_column.get(4);
82 | let column_type = if let Some(geometry_type) = geometry_type {
83 | format!(
84 | "geometry({}, {})",
85 | geometry_type,
86 | geometry_srid.unwrap_or(-1)
87 | )
88 | } else {
89 | column_type
90 | };
91 |
92 | let mut column_metadata = ColumnMetadata {
93 | name: column_name.clone(),
94 | desc: None,
95 | data_type: column_type,
96 | foreign_key: None,
97 | enum_values: None,
98 | };
99 |
100 | if let Some(column) = dp_col_vec.iter().find(|c| c.name == column_name) {
101 | column_metadata.desc = Some(column.description.clone());
102 |
103 | if column.attr_type.contains("行政区域コード") {
104 | column_metadata.foreign_key = Some(ColumnForeignKeyDetails {
105 | foreign_table: "admini_boundary_cd".to_string(),
106 | foreign_column: "改正後のコード".to_string(),
107 | });
108 | }
109 |
110 | use crate::scraper::data_page::RefType;
111 | column_metadata.enum_values = match &column.r#ref {
112 | Some(RefType::Code(map)) => Some(
113 | map.iter()
114 | .map(|(key, value)| ColumnEnumDetails {
115 | value: key.clone(),
116 | desc: Some(value.clone()),
117 | })
118 | .collect(),
119 | ),
120 | Some(RefType::Enum(vec)) => Some(
121 | vec.iter()
122 | .map(|value| ColumnEnumDetails {
123 | value: value.clone(),
124 | desc: None,
125 | })
126 | .collect(),
127 | ),
128 | None => None,
129 | }
130 | }
131 |
132 | columns.push(column_metadata);
133 | }
134 |
135 | let table_metadata = TableMetadata {
136 | name: metadata.name.clone(),
137 | desc: data_page.metadata.fundamental.get("内容").cloned(),
138 | source: Some("国土数値情報".to_string()),
139 | source_url: Some(data_item.url.clone()),
140 | license: Some(data_item.usage.clone()),
141 | license_url: None,
142 | primary_key: Some("ogc_fid".to_string()),
143 | columns,
144 | };
145 |
146 | Ok(table_metadata)
147 | }
148 |
149 | pub async fn create_dataset(&self, identifier: &str, dataset: &TableMetadata) -> Result<()> {
150 | let lowercase_identifier = identifier.to_lowercase();
151 | upsert(&self.client, &lowercase_identifier, dataset).await?;
152 | Ok(())
153 | }
154 | }
155 |
--------------------------------------------------------------------------------
/src/scraper/data_page.rs:
--------------------------------------------------------------------------------
1 | use std::collections::HashMap;
2 |
3 | use anyhow::{anyhow, Context, Result};
4 | use bytesize::ByteSize;
5 | use once_cell::sync::Lazy;
6 | use regex::Regex;
7 | use scraper::{selectable::Selectable, Html, Selector};
8 | use serde::Serialize;
9 | use url::Url;
10 |
11 | use super::table_read::{parse_table, parsed_to_string_array};
12 |
13 | // Compile the regex once for efficiency.
14 | // This regex looks for one or more digits at the very start of the string, immediately followed by '年'.
15 | static YEAR_REGEX: Lazy = Lazy::new(|| Regex::new(r"^(\d+)年").unwrap());
16 |
17 | #[derive(Debug, Serialize)]
18 | pub struct DataPage {
19 | pub url: Url,
20 | pub items: Vec,
21 | pub metadata: DataPageMetadata,
22 | }
23 |
24 | #[derive(Debug, Clone, Serialize)]
25 | pub struct DataItem {
26 | pub area: String,
27 | pub crs: String,
28 | pub bytes: u64,
29 | pub year: Option, // 年
30 | pub nendo: Option, // 年度
31 | pub file_url: Url,
32 | }
33 |
34 | pub async fn scrape(url: &Url) -> Result {
35 | let response = reqwest::get(url.clone()).await?;
36 | let body = response.text().await?;
37 | let document = Html::parse_document(&body);
38 |
39 | let metadata = extract_metadata(&document, &url)
40 | .await
41 | .with_context(|| format!("when accessing {}", url.to_string()))?;
42 |
43 | let td_sel = Selector::parse("td").unwrap();
44 | let data_tr_sel = Selector::parse("table.dataTables tr, table.dataTables-mesh tr").unwrap();
45 | let data_path_re = Regex::new(r"javascript:DownLd\('[^']*',\s*'[^']*',\s*'([^']+)'").unwrap();
46 |
47 | let mut items: Vec = Vec::new();
48 | let mut use_nendo = false;
49 | for row in document.select(&data_tr_sel) {
50 | let tds = row.select(&td_sel).collect::>();
51 | if tds.len() == 0 {
52 | continue;
53 | }
54 | let area = tds[0].text().collect::().trim().to_string();
55 | if area == "地域" {
56 | // header
57 | if tds[2].text().collect::().contains("年度") {
58 | use_nendo = true;
59 | }
60 | continue;
61 | }
62 |
63 | let crs = tds[1].text().collect::().trim().to_string();
64 |
65 | let year_str = tds[2].text().collect::().trim().to_string();
66 | let year = if use_nendo == true {
67 | None
68 | } else {
69 | Some(year_str.clone())
70 | };
71 | let nendo = if use_nendo == false {
72 | None
73 | } else {
74 | Some(year_str)
75 | };
76 | let Some(bytes_str) = tds[3].text().next().map(|s| s.to_string()) else {
77 | continue;
78 | };
79 | // if we couldn't parse the bytes, we'll just use 10MB as a default. It's just used for progress.
80 | let bytes: ByteSize = bytes_str.parse().unwrap_or_else(|_| ByteSize::mb(10));
81 | let Some(file_js_onclick) = tds[5]
82 | .select(&Selector::parse("a").unwrap())
83 | .next()
84 | .and_then(|a| a.value().attr("onclick"))
85 | else {
86 | // panic!("file_js_onclick not found: {:?}", tds[5].html());
87 | continue;
88 | };
89 | let Some(file_url) = data_path_re
90 | .captures(file_js_onclick)
91 | .map(|c| c.get(1).unwrap().as_str())
92 | .map(|s| url.join(s).unwrap())
93 | else {
94 | // panic!("file_url not found: {:?}", file_js_onclick);
95 | continue;
96 | };
97 |
98 | let item = DataItem {
99 | area,
100 | bytes: bytes.0,
101 | crs,
102 | year,
103 | nendo,
104 | file_url,
105 | };
106 | items.push(item);
107 | }
108 |
109 | items = filter_data_items(items);
110 |
111 | Ok(DataPage {
112 | url: url.clone(),
113 | items,
114 | metadata,
115 | })
116 | }
117 |
118 | #[derive(Debug, Clone, Serialize)]
119 | pub enum RefType {
120 | Enum(Vec),
121 | Code(HashMap),
122 | }
123 |
124 | async fn parse_ref_from_url(url: &Url) -> Result> {
125 | if url.to_string().contains("PubFacAdminCd.html") {
126 | return Ok(None);
127 | }
128 |
129 | let response = reqwest::get(url.clone()).await?;
130 | let body = response.text().await?;
131 | let document = Html::parse_document(&body);
132 |
133 | // Selector for cells (| or | )
134 | let td_sel = Selector::parse("td, th").unwrap();
135 | // Selector for table rows
136 | let tr_sel = Selector::parse("table tr").unwrap();
137 |
138 | let mut headers = Vec::new();
139 | // Extract first row
140 | let first_row = document
141 | .select(&tr_sel)
142 | .next()
143 | .ok_or_else(|| anyhow!("no first row found"))?;
144 |
145 | for element in first_row.select(&td_sel) {
146 | headers.push(
147 | element
148 | .text()
149 | .collect::>()
150 | .join(" ")
151 | .trim()
152 | .to_string(),
153 | );
154 | }
155 |
156 | if headers.is_empty() {
157 | return Err(anyhow!("no headers found"));
158 | }
159 |
160 | let code_idx_opt = headers.iter().position(|h| h == "コード");
161 | if let Some(code_idx) = code_idx_opt {
162 | let name_idx = headers
163 | .iter()
164 | .position(|h| {
165 | h == "対応する内容"
166 | || h == "内容"
167 | || h.contains("定義")
168 | || h.contains("分類")
169 | || h.contains("種別")
170 | || h.contains("対象")
171 | || h.contains("区分")
172 | })
173 | .ok_or_else(|| anyhow!("name index not found in headers: {:?}", headers))?;
174 | // code list
175 | let mut code_map = HashMap::new();
176 | for row in document.select(&tr_sel) {
177 | let tds = row.select(&td_sel).collect::>();
178 | if tds.len() < 2 {
179 | continue;
180 | }
181 | // code_idx is the index of the code column
182 | let code = tds
183 | .get(code_idx)
184 | .ok_or(anyhow!("code not found"))?
185 | .text()
186 | .collect::>()
187 | .join(" ")
188 | .trim()
189 | .to_string();
190 | let name = tds
191 | .get(name_idx)
192 | .ok_or(anyhow!("name not found"))?
193 | .text()
194 | .collect::>()
195 | .join(" ")
196 | .trim()
197 | .to_string();
198 | if !code.is_empty() && code != "コード" && !name.is_empty() {
199 | code_map.insert(code, name);
200 | }
201 | }
202 | if code_map.is_empty() {
203 | return Err(anyhow!("no code found"));
204 | }
205 | return Ok(Some(RefType::Code(code_map)));
206 | } else if headers[0].contains("定数") {
207 | // enum list
208 | let mut enum_list = Vec::new();
209 | for cell in document.select(&td_sel) {
210 | let cell_text = cell.text().collect::>().join(" ").trim().to_string();
211 | if !cell_text.is_empty() && cell_text != "定数" {
212 | enum_list.push(cell_text);
213 | }
214 | }
215 | if enum_list.is_empty() {
216 | return Err(anyhow!("no enum found"));
217 | }
218 | return Ok(Some(RefType::Enum(enum_list)));
219 | }
220 |
221 | Err(anyhow!("ref table not found"))
222 | }
223 |
224 | #[derive(Debug, Clone, Serialize)]
225 | pub struct AttributeMetadata {
226 | pub name: String,
227 | pub description: String,
228 | pub attr_type: String,
229 | #[serde(skip_serializing_if = "Option::is_none")]
230 | pub ref_url: Option,
231 | #[serde(skip_serializing_if = "Option::is_none")]
232 | pub r#ref: Option,
233 | }
234 |
235 | #[derive(Default, Debug, Serialize)]
236 | pub struct DataPageMetadata {
237 | pub fundamental: HashMap,
238 | pub attribute: HashMap,
239 | }
240 |
241 | async fn extract_metadata<'a, S: Selectable<'a>>(
242 | html: S,
243 | base_url: &Url,
244 | ) -> Result {
245 | let mut metadata = DataPageMetadata::default();
246 | let table_sel = Selector::parse("table").unwrap();
247 | let t_cell_sel = Selector::parse("th, td").unwrap();
248 | let tables: Vec> = html.select(&table_sel).collect();
249 |
250 | let strip_space_re = Regex::new(r"\s+").unwrap();
251 |
252 | // 「更新履歴」や「内容」が入っているtableを探す
253 | let fundamental_table = tables
254 | .iter()
255 | .find(|table| {
256 | let headers: Vec = table
257 | .select(&t_cell_sel)
258 | .map(|th| th.text().collect::().trim().to_string())
259 | .collect();
260 | headers
261 | .iter()
262 | .any(|h| h.contains("更新履歴") || h.contains("内容"))
263 | })
264 | .ok_or_else(|| anyhow!("基本情報の table が見つかりませんでした"))?
265 | .clone();
266 | let fundamental_parsed = parse_table(fundamental_table);
267 | let fundamental_parsed_str = parsed_to_string_array(fundamental_parsed);
268 | // println!("{:?}", fundamental_parsed);
269 | for row in fundamental_parsed_str.outer_iter() {
270 | if row.len() < 2 {
271 | continue;
272 | }
273 | let key = row[0].as_ref().unwrap().trim().to_string();
274 | let mut value = row[1].as_ref().unwrap().trim().to_string();
275 | value = strip_space_re.replace_all(&value, " ").to_string();
276 | metadata.fundamental.insert(key, value);
277 | }
278 |
279 | if metadata.fundamental.is_empty() {
280 | return Err(anyhow!("基本情報が見つかりませんでした"));
281 | }
282 |
283 | // ※シェープファイルの属性名の後ろに「▲」を付与している項目は、属性値無しのときは、空欄でなく半角アンダーライン（ _ ）を記述している。
284 | // TODO: この処理をハンドリングする?
285 | let attr_key_regex = Regex::new(r"^(.*?)\s*[（(]([a-zA-Z0-9-_]+)▲?[）)]$").unwrap();
286 |
287 | // 「属性情報」や「属性名」が入っているtableを探す
288 | metadata.attribute = tables
289 | .iter()
290 | .find_map(|table| {
291 | // ignore this table if it has any tables inside of it
292 | if table.select(&table_sel).count() > 1 {
293 | return None;
294 | }
295 |
296 | let parsed = parse_table(table.clone());
297 | // 属性名、説明、属性型
298 | let mut attr_indices: Option<(usize, usize, usize)> = None;
299 |
300 | let mut attr_map: HashMap = HashMap::new();
301 |
302 | for row in parsed.outer_iter() {
303 | if row.len() < 3 {
304 | continue;
305 | }
306 | // we've already recognized the indices for the attribute table
307 | if let Some((attr_name_idx, desc_idx, type_idx)) = attr_indices {
308 | // println!("Looking at row: {:?}", row);
309 | let attr_name_str = row[attr_name_idx]
310 | .as_ref()?
311 | .text()
312 | .collect::()
313 | .trim()
314 | .to_string();
315 | let Some(name_match) = attr_key_regex.captures(&attr_name_str) else {
316 | continue;
317 | };
318 | let name_jp = name_match.get(1).unwrap();
319 | let name_id = name_match.get(2).unwrap();
320 |
321 | let mut description = row[desc_idx]
322 | .as_ref()?
323 | .text()
324 | .collect::()
325 | .trim()
326 | .to_string();
327 | description = strip_space_re.replace_all(&description, " ").to_string();
328 | let attr_type_ele = row[type_idx].as_ref().unwrap();
329 | let mut attr_type_str =
330 | attr_type_ele.text().collect::().trim().to_string();
331 | attr_type_str = strip_space_re.replace_all(&attr_type_str, " ").to_string();
332 |
333 | let mut ref_url = None;
334 | if let Some(a) = attr_type_ele.select(&Selector::parse("a").unwrap()).next() {
335 | let href = a.value().attr("href").unwrap();
336 | ref_url = Some(base_url.join(href).unwrap());
337 | }
338 |
339 | attr_map.insert(
340 | name_id.as_str().to_string(),
341 | AttributeMetadata {
342 | name: name_jp.as_str().to_string(),
343 | description,
344 | attr_type: attr_type_str,
345 | ref_url,
346 | r#ref: None,
347 | },
348 | );
349 | } else {
350 | let mut attr_name: Option = None;
351 | let mut attr_desc: Option = None;
352 | let mut attr_type: Option = None;
353 | for (i, cell) in row.iter().enumerate() {
354 | let Some(cell) = cell.as_ref() else {
355 | continue;
356 | };
357 | let cell_str = cell.text().collect::().trim().to_string();
358 | if cell_str.contains("属性名") {
359 | attr_name = Some(i);
360 | } else if cell_str.contains("説明") {
361 | attr_desc = Some(i);
362 | } else if cell_str.contains("属性の型") || cell_str.contains("属性型")
363 | {
364 | attr_type = Some(i);
365 | }
366 | if attr_name.is_some() && attr_desc.is_some() && attr_type.is_some() {
367 | attr_indices =
368 | Some((attr_name.unwrap(), attr_desc.unwrap(), attr_type.unwrap()));
369 | // println!("Found cell indices: {:?}", attr_indices);
370 | break;
371 | }
372 | }
373 | }
374 | }
375 |
376 | if attr_map.is_empty() {
377 | return None;
378 | }
379 | Some(attr_map)
380 | })
381 | .ok_or_else(|| anyhow!("属性情報の table が見つかりませんでした"))?;
382 |
383 | for attr in metadata.attribute.values_mut() {
384 | if let Some(ref_url) = &attr.ref_url {
385 | if ref_url.to_string().contains(".xlsx") {
386 | // AdminiBoundary_CD.xlsx は admini_boundary.rs で対応済み
387 | continue;
388 | }
389 | attr.r#ref = parse_ref_from_url(&ref_url)
390 | .await
391 | .with_context(|| format!("when accessing ref url: {}", &ref_url))?;
392 | }
393 | }
394 |
395 | Ok(metadata)
396 | }
397 |
398 | /// Extracts the numeric year from a field formatted like "2006年（平成18年）".
399 | /// If the field does not match, returns None.
400 | fn extract_year_from_field(field: &str) -> Option {
401 | YEAR_REGEX
402 | .captures(field)
403 | .and_then(|caps| caps.get(1))
404 | .and_then(|m| m.as_str().parse::().ok())
405 | }
406 |
407 | /// Determines the recency value for an item, preferring the `year` field.
408 | /// Falls back to `nendo` if necessary.
409 | fn parse_recency(item: &DataItem) -> Option {
410 | if let Some(ref y) = item.year {
411 | if let Some(year) = extract_year_from_field(y) {
412 | return Some(year);
413 | }
414 | }
415 | if let Some(ref n) = item.nendo {
416 | if let Some(year) = extract_year_from_field(n) {
417 | return Some(year);
418 | }
419 | }
420 | None
421 | }
422 |
423 | /**
424 | * データのリストから、CRSが世界測地系のものを抽出する
425 | * 全国データある場合はそれだけを返す
426 | * ない場合はそのまま帰す（殆どの場合は都道府県別）
427 | */
428 | fn filter_data_items(items: Vec) -> Vec {
429 | // Step 1: Filter items by CRS.
430 | let crs_filtered: Vec = items
431 | .into_iter()
432 | .filter(|item| item.crs == "世界測地系")
433 | .collect();
434 |
435 | // Step 2: Group items by area.
436 | let mut area_groups: HashMap> = HashMap::new();
437 | for item in crs_filtered {
438 | // If 全国 is already in the map, and we aren't in the 全国 group, skip this item.
439 | if area_groups.contains_key("全国") && item.area != "全国" {
440 | continue;
441 | }
442 | area_groups.entry(item.area.clone()).or_default().push(item);
443 | }
444 |
445 | // Step 3: For each area evaluate the max recency and filter items accordingly.
446 | let mut result = Vec::new();
447 | for (_area, group) in area_groups {
448 | let max_recency = group.iter().filter_map(|item| parse_recency(item)).max();
449 | if let Some(max_year) = max_recency {
450 | result.extend(
451 | group
452 | .into_iter()
453 | .filter(|item| parse_recency(item) == Some(max_year)),
454 | );
455 | } else {
456 | result.extend(group);
457 | }
458 | }
459 | result
460 | }
461 |
462 | #[cfg(test)]
463 | mod tests {
464 | use super::*;
465 |
466 | #[tokio::test]
467 | async fn test_scrape_c23() {
468 | let url =
469 | Url::parse("https://nlftp.mlit.go.jp/ksj/gml/datalist/KsjTmplt-C23.html").unwrap();
470 | let page = scrape(&url).await.unwrap();
471 | assert_eq!(page.items.len(), 39);
472 | }
473 |
474 | #[tokio::test]
475 | async fn test_scrape_n03() {
476 | let url =
477 | Url::parse("https://nlftp.mlit.go.jp/ksj/gml/datalist/KsjTmplt-N03-2024.html").unwrap();
478 | let page = scrape(&url).await.unwrap();
479 | // 全国パターン
480 | assert_eq!(page.items.len(), 1);
481 |
482 | let naiyo = page.metadata.fundamental.get("内容").unwrap();
483 | assert!(naiyo.contains("全国の行政界について、都道府県名、"));
484 |
485 | let zahyoukei = page.metadata.fundamental.get("座標系").unwrap();
486 | assert!(zahyoukei.contains("世界測地系"));
487 |
488 | let todoufukenmei = page.metadata.attribute.get("N03_001").unwrap();
489 | assert!(todoufukenmei.name.contains("都道府県名"));
490 | assert!(todoufukenmei.description.contains("都道府県の名称"));
491 | assert!(todoufukenmei.attr_type.contains("文字列"));
492 |
493 | let lg_code = page.metadata.attribute.get("N03_007").unwrap();
494 | assert!(lg_code.name.contains("全国地方公共団体コード"));
495 | assert!(lg_code.description.contains("JIS X 0401"));
496 | assert!(lg_code.attr_type.contains("コードリスト"));
497 | assert!(lg_code
498 | .ref_url
499 | .as_ref()
500 | .is_some_and(|u| u.as_str().contains("AdminiBoundary_CD.xlsx")));
501 | }
502 |
503 | #[tokio::test]
504 | async fn test_scrape_a27() {
505 | let url =
506 | Url::parse("https://nlftp.mlit.go.jp/ksj/gml/datalist/KsjTmplt-A27-2023.html").unwrap();
507 | let page = scrape(&url).await.unwrap();
508 | // 全国パターン
509 | assert_eq!(page.items.len(), 1);
510 |
511 | let a27_001 = page.metadata.attribute.get("A27_001").unwrap();
512 | assert_eq!(a27_001.name, "行政区域コード");
513 | let a27_002 = page.metadata.attribute.get("A27_002").unwrap();
514 | assert_eq!(a27_002.name, "設置主体");
515 | let a27_003 = page.metadata.attribute.get("A27_003").unwrap();
516 | assert_eq!(a27_003.name, "学校コード");
517 | let a27_004 = page.metadata.attribute.get("A27_004").unwrap();
518 | assert_eq!(a27_004.name, "名称");
519 | let a27_005 = page.metadata.attribute.get("A27_005").unwrap();
520 | assert_eq!(a27_005.name, "所在地");
521 | }
522 |
523 | #[tokio::test]
524 | async fn test_scrape_a38() {
525 | let url =
526 | Url::parse("https://nlftp.mlit.go.jp/ksj/gml/datalist/KsjTmplt-A38-2020.html").unwrap();
527 | let page = scrape(&url).await.unwrap();
528 | // 全国パターン
529 | assert_eq!(page.items.len(), 1);
530 |
531 | let a38a_001 = page.metadata.attribute.get("A38a_001").unwrap();
532 | assert_eq!(a38a_001.name, "行政区域コード");
533 | let a38b_001 = page.metadata.attribute.get("A38b_001").unwrap();
534 | assert_eq!(a38b_001.name, "行政区域コード");
535 | let a38c_001 = page.metadata.attribute.get("A38c_001").unwrap();
536 | assert_eq!(a38c_001.name, "都道府県名");
537 | }
538 |
539 | #[tokio::test]
540 | async fn test_parse_ref_enum() {
541 | let url =
542 | Url::parse("https://nlftp.mlit.go.jp/ksj/gml/codelist/L01_v3_2_RoadEnumType.html")
543 | .unwrap();
544 | let ref_enum = parse_ref_from_url(&url).await.unwrap().unwrap();
545 | if let RefType::Enum(ref enum_list) = ref_enum {
546 | assert_eq!(enum_list.len(), 14);
547 | assert_eq!(enum_list[0], "国道");
548 | assert_eq!(enum_list[1], "都道");
549 | } else {
550 | panic!("Expected RefType::Enum, but got something else.");
551 | }
552 | }
553 |
554 | struct TestCase<'a> {
555 | url: &'a str,
556 | expected_len: usize,
557 | expected: HashMap<&'a str, &'a str>,
558 | }
559 |
560 | async fn run_parse_ref_code_test(test_case: TestCase<'_>) {
561 | let url = Url::parse(test_case.url).unwrap();
562 | let ref_enum = parse_ref_from_url(&url).await.unwrap().unwrap();
563 |
564 | match ref_enum {
565 | RefType::Code(ref code_map) => {
566 | assert_eq!(code_map.len(), test_case.expected_len);
567 | for (key, value) in test_case.expected.iter() {
568 | assert_eq!(code_map.get(*key).unwrap(), value);
569 | }
570 | }
571 | _ => panic!("Expected RefType::Code, but got something else."),
572 | }
573 | }
574 |
575 | #[tokio::test]
576 | async fn test_parse_ref_code() {
577 | let test_cases = [
578 | TestCase {
579 | url: "https://nlftp.mlit.go.jp/ksj/gml/codelist/reasonForDesignationCode.html",
580 | expected_len: 7,
581 | expected: HashMap::from([
582 | ("1", "水害（河川）"),
583 | ("2", "水害（海）"),
584 | ("3", "水害（河川・海）"),
585 | ("7", "その他"),
586 | ]),
587 | },
588 | TestCase {
589 | url: "https://nlftp.mlit.go.jp/ksj/gml/codelist/CodeOfPhenomenon.html",
590 | expected_len: 3,
591 | expected: HashMap::from([
592 | ("1", "急傾斜地の崩壊"),
593 | ("2", "土石流"),
594 | ("3", "地滑り"),
595 | ]),
596 | },
597 | TestCase {
598 | url: "https://nlftp.mlit.go.jp/ksj/gml/codelist/MedClassCd.html",
599 | expected_len: 3,
600 | expected: HashMap::from([("1", "病院"), ("2", "診療所"), ("3", "歯科診療所")]),
601 | },
602 | TestCase {
603 | url: "https://nlftp.mlit.go.jp/ksj/gml/codelist/ReferenceDataCd.html",
604 | expected_len: 6,
605 | expected: HashMap::from([
606 | ("1", "10mDEM"),
607 | ("2", "5m空中写真DEM"),
608 | ("3", "5mレーザDEM"),
609 | ("4", "2mDEM"),
610 | ]),
611 | },
612 | TestCase {
613 | url: "https://nlftp.mlit.go.jp/ksj/gml/codelist/LandUseCd-09.html",
614 | expected_len: 17,
615 | expected: HashMap::from([("0100", "田"), ("1100", "河川地及び湖沼")]),
616 | },
617 | TestCase {
618 | url: "https://nlftp.mlit.go.jp/ksj/gml/codelist/welfareInstitution_welfareFacilityMiddleClassificationCode.html",
619 | expected_len: 62,
620 | expected: HashMap::from([
621 | ("0101", "救護施設"),
622 | ("0399", "その他"),
623 | ]),
624 | },
625 | TestCase {
626 | url: "https://nlftp.mlit.go.jp/ksj/gml/codelist/water_depth_code.html",
627 | expected_len: 6,
628 | expected: HashMap::from([
629 | ("1", "0m 以上 0.5m 未満"),
630 | ("6", "20.0m 以上"),
631 | ]),
632 | }
633 | ];
634 |
635 | for test_case in test_cases {
636 | run_parse_ref_code_test(test_case).await;
637 | }
638 | }
639 | }
640 |
--------------------------------------------------------------------------------
/src/scraper/download_queue.rs:
--------------------------------------------------------------------------------
1 | use crate::downloader;
2 | use anyhow::Result;
3 | use async_channel::unbounded;
4 | use indicatif::{ProgressBar, ProgressState, ProgressStyle};
5 | use std::fmt::Write;
6 | use std::time::Duration;
7 | use tokio::task;
8 |
9 | use super::data_page::DataItem;
10 |
11 | const DL_QUEUE_SIZE: usize = 15;
12 |
13 | struct PBStatusUpdateMsg {
14 | added: u64,
15 | finished: u64,
16 | }
17 |
18 | pub struct DownloadQueue {
19 | pb_status_sender: Option>,
20 | sender: Option>,
21 |
22 | set: Option>,
23 | }
24 |
25 | impl DownloadQueue {
26 | pub fn new() -> Self {
27 | let (pb_status_sender, pb_status_receiver) = unbounded::();
28 | let (sender, receiver) = unbounded::();
29 | let mut set = task::JoinSet::new();
30 | for _i in 0..DL_QUEUE_SIZE {
31 | let receiver = receiver.clone();
32 | let pb_sender = pb_status_sender.clone();
33 | set.spawn(async move {
34 | while let Ok(item) = receiver.recv().await {
35 | // println!("processor {} loading: {}", i, item.file_url);
36 | // println!("Downloading: {}", url);
37 | let url = item.file_url;
38 | // TODO: retry the download if it fails
39 | downloader::download_to_tmp(&url).await.unwrap();
40 | pb_sender
41 | .send(PBStatusUpdateMsg {
42 | added: 0,
43 | finished: item.bytes,
44 | })
45 | .await
46 | .unwrap();
47 | }
48 | });
49 | }
50 |
51 | set.spawn(async move {
52 | let pb = ProgressBar::new(0);
53 | pb.set_style(ProgressStyle::with_template("{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} ({eta})")
54 | .unwrap()
55 | .with_key("eta", |state: &ProgressState, w: &mut dyn Write| write!(w, "{:.1}s", state.eta().as_secs_f64()).unwrap())
56 | .progress_chars("=>-"));
57 | pb.enable_steady_tick(Duration::from_millis(300));
58 | let mut length = 0;
59 | let mut position = 0;
60 | while let Ok(msg) = pb_status_receiver.recv().await {
61 | length += msg.added;
62 | position += msg.finished;
63 | pb.set_length(length);
64 | pb.set_position(position);
65 | }
66 | pb.finish();
67 | println!("ダウンロードが終了しました。");
68 | });
69 | Self {
70 | pb_status_sender: Some(pb_status_sender),
71 | sender: Some(sender),
72 | set: Some(set),
73 | }
74 | }
75 |
76 | pub async fn push(&self, item: DataItem) -> Result<()> {
77 | let Some(sender) = &self.sender else {
78 | return Err(anyhow::anyhow!("DownloadQueue is already closed"));
79 | };
80 | let Some(pb_status_sender) = &self.pb_status_sender else {
81 | return Err(anyhow::anyhow!("DownloadQueue is already closed"));
82 | };
83 | pb_status_sender
84 | .send(PBStatusUpdateMsg {
85 | added: item.bytes,
86 | finished: 0,
87 | })
88 | .await?;
89 | sender.send(item).await?;
90 | Ok(())
91 | }
92 |
93 | pub async fn close(&mut self) -> Result<()> {
94 | let Some(_) = self.sender.take() else {
95 | return Err(anyhow::anyhow!("DownloadQueue is already closed"));
96 | };
97 | let Some(set) = self.set.take() else {
98 | return Err(anyhow::anyhow!("DownloadQueue is already closed"));
99 | };
100 | let Some(_) = self.pb_status_sender.take() else {
101 | return Err(anyhow::anyhow!("DownloadQueue is already closed"));
102 | };
103 | set.join_all().await;
104 | Ok(())
105 | }
106 | }
107 |
--------------------------------------------------------------------------------
/src/scraper/initial.rs:
--------------------------------------------------------------------------------
1 | use anyhow::{anyhow, Result};
2 | use regex::Regex;
3 | use serde::Serialize;
4 | use url::Url;
5 |
6 | #[derive(Debug, Clone, Serialize)]
7 | pub struct DataItem {
8 | pub category1_name: String,
9 | pub category2_name: String,
10 | pub name: String,
11 | pub data_source: String,
12 | pub data_accuracy: String,
13 | pub metadata_xml: Url,
14 | pub usage: String,
15 |
16 | pub url: Url,
17 | pub identifier: String,
18 | }
19 |
20 | #[derive(Debug)]
21 | pub struct ScrapeResult {
22 | #[allow(dead_code)]
23 | pub url: Url,
24 | pub data: Vec,
25 | }
26 |
27 | fn extract_identifier_from_metadata_url(url: &str) -> Result {
28 | let re = Regex::new(r"/meta/([a-zA-Z0-9-]+)/")?;
29 | let captures = re
30 | .captures(url)
31 | .ok_or_else(|| anyhow!("Couldn't extract identifier from {}", url))?;
32 | let identifier = captures.get(1).unwrap().as_str().to_string();
33 | Ok(identifier)
34 | }
35 |
36 | pub async fn scrape() -> Result {
37 | let root_url = Url::parse("https://nlftp.mlit.go.jp/ksj/gml/gml_datalist.html")?;
38 | let mut data: Vec = vec![];
39 | let response = reqwest::get(root_url.clone()).await?;
40 | let body = response.text().await?;
41 | let document = scraper::Html::parse_document(&body);
42 | let collapse_sel = scraper::Selector::parse("ul.collapsible").unwrap();
43 | let category1_re = Regex::new(r"\d(?:\.\s*)?([^\s]+)").unwrap();
44 | for collapse in document.select(&collapse_sel) {
45 | let header_sel = scraper::Selector::parse(".collapsible-header").unwrap();
46 | let Some(header) = collapse.select(&header_sel).next() else {
47 | continue;
48 | };
49 | let category1_txt = header.text().collect::().trim().to_string();
50 | let Some(category1_name) = category1_re
51 | .captures(&category1_txt)
52 | .map(|c| c.get(1).unwrap().as_str())
53 | else {
54 | continue;
55 | };
56 |
57 | let table_tr_sel = scraper::Selector::parse("table tr").unwrap();
58 | let mut category2_name: Option = None;
59 | for tr in collapse.select(&table_tr_sel) {
60 | let td_sel = scraper::Selector::parse("td").unwrap();
61 | let a_sel = scraper::Selector::parse("a").unwrap();
62 | let tds = tr.select(&td_sel).collect::>();
63 | if tds.len() == 0 {
64 | continue;
65 | }
66 | let name_td = &tds[0];
67 | let name = name_td.text().collect::().trim().to_string();
68 | if name.starts_with("【") {
69 | category2_name = Some(name);
70 | continue;
71 | }
72 | let Some(url_str) = name_td.select(&a_sel).next().and_then(|u| u.attr("href")) else {
73 | continue;
74 | };
75 | let url = root_url.join(url_str)?;
76 |
77 | let data_source = tds[2].text().collect::().trim().to_string();
78 | let data_accuracy = tds[3].text().collect::().trim().to_string();
79 | let identifier: String;
80 | let metadata_xml_url: Url;
81 | if let Some(url) = tds[4].select(&a_sel).next().and_then(|a| a.attr("href")) {
82 | identifier = extract_identifier_from_metadata_url(&url)?;
83 | metadata_xml_url = root_url.join(url)?;
84 | } else {
85 | continue;
86 | }
87 | let usage = tds[5]
88 | .select(&a_sel)
89 | .next()
90 | .unwrap()
91 | .text()
92 | .collect::()
93 | .trim()
94 | .to_string();
95 |
96 | data.push(DataItem {
97 | category1_name: category1_name.to_string(),
98 | category2_name: category2_name.clone().unwrap_or_default(),
99 | name,
100 | data_source,
101 | data_accuracy,
102 | metadata_xml: metadata_xml_url,
103 | usage,
104 | url,
105 | identifier,
106 | });
107 | }
108 | }
109 |
110 | Ok(ScrapeResult {
111 | url: root_url,
112 | data,
113 | })
114 | }
115 |
116 | #[cfg(test)]
117 | mod tests {
118 | use super::*;
119 |
120 | #[tokio::test]
121 | async fn test_scrape() {
122 | let result = scrape().await.unwrap();
123 | assert_eq!(result.data.len(), 125);
124 | let first = result.data.get(0).unwrap();
125 | assert_eq!(first.name, "海岸線");
126 | assert_eq!(first.identifier, "C23");
127 | }
128 | }
129 |
--------------------------------------------------------------------------------
/src/scraper/mod.rs:
--------------------------------------------------------------------------------
1 | // The scraper module is responsible for downloading the data from the website.
2 | use anyhow::Result;
3 | use derive_builder::Builder;
4 | use std::{fmt, path::PathBuf, sync::Arc};
5 |
6 | use crate::downloader::path_for_url;
7 |
8 | pub mod data_page;
9 | mod download_queue;
10 | pub mod initial;
11 | mod table_read;
12 |
13 | #[derive(Clone)]
14 | pub struct Dataset {
15 | // pub item: data_page::DataItem,
16 | pub initial_item: initial::DataItem,
17 | pub page: Arc,
18 | pub zip_file_paths: Vec,
19 | }
20 |
21 | impl fmt::Display for Dataset {
22 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
23 | write!(
24 | f,
25 | "Dataset identifier={} url={}",
26 | self.initial_item.identifier,
27 | self.page.url.to_string()
28 | )
29 | }
30 | }
31 |
32 | #[derive(Builder)]
33 | pub struct Scraper {
34 | skip_dl: bool,
35 | filter_identifiers: Option>,
36 | }
37 |
38 | impl Scraper {
39 | pub async fn download_all(&self) -> Result> {
40 | let mut dl_queue = download_queue::DownloadQueue::new();
41 | let initial = initial::scrape().await?;
42 | let data_items = initial.data;
43 | let mut out: Vec = Vec::new();
44 | for initial_item in data_items {
45 | // TODO: 非商用を対応
46 | if initial_item.usage == "非商用" {
47 | continue;
48 | }
49 | if let Some(filter_identifiers) = &self.filter_identifiers {
50 | if !filter_identifiers.contains(&initial_item.identifier) {
51 | continue;
52 | }
53 | }
54 |
55 | let page_res = data_page::scrape(&initial_item.url).await;
56 | if let Err(err) = page_res {
57 | println!("[ERROR, skipping...] {:?}", err);
58 | continue;
59 | }
60 | let page = Arc::new(page_res.unwrap());
61 |
62 | let mut zip_file_paths: Vec = Vec::new();
63 | for item in &page.items {
64 | let expected_path = path_for_url(&item.file_url);
65 | zip_file_paths.push(expected_path.0);
66 | if !self.skip_dl {
67 | dl_queue.push(item.clone()).await?;
68 | }
69 | }
70 | out.push(Dataset {
71 | initial_item,
72 | page,
73 | zip_file_paths,
74 | });
75 | }
76 | dl_queue.close().await?;
77 | Ok(out)
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/src/scraper/table_read.rs:
--------------------------------------------------------------------------------
1 | use ndarray::Array2;
2 | use scraper::{selectable::Selectable, ElementRef, Selector};
3 |
4 | /// Parses an HTML table (handling colspan/rowspan) and returns a 2D ndarray
5 | /// where each cell is an Option.
6 | pub fn parse_table<'a, S: Selectable<'a>>(table: S) -> Array2>> {
7 | let tr_selector = Selector::parse("tr").unwrap();
8 | let cell_selector = Selector::parse("th, td").unwrap();
9 |
10 | // We'll accumulate rows in a Vec>>
11 | let mut grid: Vec>>> = Vec::new();
12 | // pending holds cells that span into future rows:
13 | // (target_row, col_index, cell)
14 | let mut pending: Vec<(usize, usize, ElementRef<'a>)> = Vec::new();
15 |
16 | for (row_index, tr) in table.select(&tr_selector).enumerate() {
17 | // Ensure our grid has an entry for this row.
18 | if grid.len() <= row_index {
19 | grid.push(Vec::new());
20 | }
21 | let row = &mut grid[row_index];
22 |
23 | // First, insert any cells carried over from previous rows (rowspan)
24 | for &(_target_row, col_index, ref cell) in
25 | pending.iter().filter(|&&(r, _, _)| r == row_index)
26 | {
27 | while row.len() <= col_index {
28 | row.push(None);
29 | }
30 | row[col_index] = Some(cell.clone());
31 | }
32 | // Remove pending entries that have been used
33 | pending.retain(|&(r, _, _)| r > row_index);
34 |
35 | // Now process the current row’s cells.
36 | let mut col_index = 0;
37 | for cell_node in tr.select(&cell_selector) {
38 | // Skip any columns already filled by pending cells.
39 | while row.get(col_index).is_some() {
40 | col_index += 1;
41 | }
42 | // Extract cell content.
43 | // let content = cell_node
44 | // .text()
45 | // .collect::>()
46 | // .join(" ")
47 | // .trim()
48 | // .to_string();
49 | let content = cell_node;
50 | let colspan = cell_node
51 | .value()
52 | .attr("colspan")
53 | .and_then(|v| v.parse::().ok())
54 | .unwrap_or(1);
55 | let rowspan = cell_node
56 | .value()
57 | .attr("rowspan")
58 | .and_then(|v| v.parse::().ok())
59 | .unwrap_or(1);
60 | let cell = content;
61 |
62 | // Place the cell in the current row for each column it spans.
63 | for offset in 0..colspan {
64 | while row.len() <= col_index + offset {
65 | row.push(None);
66 | }
67 | row[col_index + offset] = Some(cell.clone());
68 | }
69 | // For rowspan > 1, schedule this cell for future rows.
70 | if rowspan > 1 {
71 | for r in 1..rowspan {
72 | pending.push((row_index + r, col_index, cell.clone()));
73 | }
74 | }
75 | col_index += colspan;
76 | }
77 | }
78 |
79 | // Determine the final dimensions.
80 | let nrows = grid.len();
81 | let ncols = grid.iter().map(|r| r.len()).max().unwrap_or(0);
82 |
83 | // Build a single Vec of length nrows*ncols, padding missing cells with None.
84 | let mut data = Vec::with_capacity(nrows * ncols);
85 | for row in grid.iter() {
86 | let mut new_row = row.clone();
87 | new_row.resize(ncols, None);
88 | data.extend(new_row);
89 | }
90 |
91 | // Create the ndarray (note: Array2 is row-major by default).
92 | Array2::from_shape_vec((nrows, ncols), data).expect("Shape mismatch")
93 | }
94 |
95 | pub fn parsed_to_string_array(parsed: Array2>) -> Array2> {
96 | parsed.map(|x| x.map(|y| y.text().collect::()))
97 | }
98 |
99 | #[cfg(test)]
100 | mod tests {
101 | use super::*;
102 |
103 | #[test]
104 | fn test_parse_table() {
105 | // Example HTML (you would replace this with your actual table HTML)
106 | let html_str = r#"
107 |
108 |
109 | | Header 1 |
110 | Header 2 |
111 |
112 |
113 | | A |
114 | B |
115 | C |
116 |
117 |
118 | | D |
119 |
120 |
121 | "#;
122 | let html = scraper::Html::parse_document(&html_str);
123 | let table_array = parse_table(&html);
124 | let table_array_str = parsed_to_string_array(table_array);
125 |
126 | let expected = Array2::from_shape_vec(
127 | (3, 3),
128 | vec![
129 | Some("Header 1".into()),
130 | Some("Header 2".into()),
131 | Some("Header 2".into()),
132 | Some("A".into()),
133 | Some("B".into()),
134 | Some("C".into()),
135 | Some("A".into()),
136 | Some("D".into()),
137 | Some("D".into()),
138 | ],
139 | )
140 | .expect("Failed to create expected array");
141 |
142 | assert_eq!(table_array_str, expected);
143 | }
144 | }
145 |
--------------------------------------------------------------------------------
/test_data/shp/cp932.dbf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/test_data/shp/cp932.dbf
--------------------------------------------------------------------------------
/test_data/shp/cp932.shp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/test_data/shp/cp932.shp
--------------------------------------------------------------------------------
/test_data/shp/cp932.shx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/test_data/shp/cp932.shx
--------------------------------------------------------------------------------
/test_data/shp/src_blank.dbf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/test_data/shp/src_blank.dbf
--------------------------------------------------------------------------------
/test_data/shp/src_blank.prj:
--------------------------------------------------------------------------------
1 | GEOGCS["GCS_JGD_2011",DATUM["D_JGD_2011",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]
2 |
--------------------------------------------------------------------------------
/test_data/shp/src_blank.shp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/test_data/shp/src_blank.shp
--------------------------------------------------------------------------------
/test_data/shp/src_blank.shx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/test_data/shp/src_blank.shx
--------------------------------------------------------------------------------
/test_data/zip/A30a5-11_4939-jgd_GML.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/test_data/zip/A30a5-11_4939-jgd_GML.zip
--------------------------------------------------------------------------------
/test_data/zip/P23-12_38_GML.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/test_data/zip/P23-12_38_GML.zip
--------------------------------------------------------------------------------
/tmp/.gitkeep:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/KotobaMedia/jpksj-to-sql/8ac0a7f12570dffda1b54660539a6545e663adb9/tmp/.gitkeep
--------------------------------------------------------------------------------
|