├── .editorconfig ├── .github └── workflows │ ├── create-data.yml │ ├── create-release.yml │ └── test.yml ├── .gitignore ├── .tool-versions ├── DATA.md ├── LICENSE ├── README.md ├── deploy ├── 01_sync_to_s3.sh ├── 01a_create_archive.sh └── 02_clear_cloudflare_cache.sh ├── docs └── index.html ├── eslint.config.mjs ├── out └── .keep ├── package-lock.json ├── package.json ├── src ├── 01_make_prefecture_city.ts ├── 02_make_machi_aza.ts ├── 03_make_rsdt.ts ├── 04_make_chiban.ts ├── 10_refresh_csv_ranges.ts ├── 99_create_stats.ts ├── address_data.proto ├── address_data.ts ├── data.ts ├── lib │ ├── abr_mlit_merge_tools.ts │ ├── ckan.test.ts │ ├── ckan.ts │ ├── ckan_data │ │ ├── chiban.ts │ │ ├── city.ts │ │ ├── index.test.ts │ │ ├── index.ts │ │ ├── machi_aza.ts │ │ ├── prefecture.ts │ │ └── rsdtdsp_rsdt.ts │ ├── fetch_tools.ts │ ├── mlit_nlftp.test.ts │ ├── mlit_nlftp.ts │ ├── proj.test.ts │ ├── proj.ts │ ├── settings.test.ts │ ├── settings.ts │ ├── zip_tools.test.ts │ └── zip_tools.ts └── processes │ ├── 01_make_prefecture_city.test.ts │ ├── 01_make_prefecture_city.ts │ ├── 02_machi_aza.ts │ ├── 02_make_machi_aza.test.ts │ ├── 02_make_machi_aza.ts │ ├── 03_make_rsdt.test.ts │ ├── 03_make_rsdt.ts │ ├── 04_make_chiban.test.ts │ ├── 04_make_chiban.ts │ ├── 10_refresh_csv_ranges.test.ts │ ├── 10_refresh_csv_ranges.ts │ └── 99_create_stats.ts ├── test └── fixtures │ └── lib │ ├── settings │ └── settings.json │ └── zip_tools │ ├── double_level.csv.zip │ ├── double_level1.csv │ ├── double_level1.csv.zip │ ├── double_level2.csv │ ├── double_level2.csv.zip │ └── single_level.csv.zip ├── tsconfig.dist-cjs.json ├── tsconfig.dist-esm.json └── tsconfig.json /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: https://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | charset = utf-8 11 | indent_style = space 12 | indent_size = 2 13 | trim_trailing_whitespace = true 14 | 15 | [*.md] 16 | indent_size = 4 17 | indent_style = tab 18 | -------------------------------------------------------------------------------- /.github/workflows/create-data.yml: -------------------------------------------------------------------------------- 1 | name: Create Data 2 | 3 | on: 4 | # TODO: Run this workflow every month 5 | # schedule: 6 | # - cron: '0 0 * * *' 7 | workflow_dispatch: 8 | 9 | jobs: 10 | create-data: 11 | runs-on: ubuntu-latest 12 | 13 | # id-token: write is required for AWS authentication 14 | permissions: 15 | id-token: write 16 | contents: read 17 | 18 | steps: 19 | - name: Remove unused software 20 | run: | 21 | echo "Available storage before:" 22 | sudo df -h 23 | echo 24 | sudo rm -rf /usr/share/dotnet 25 | sudo rm -rf /usr/local/lib/android 26 | sudo rm -rf /opt/ghc 27 | sudo rm -rf /opt/hostedtoolcache/CodeQL 28 | echo "Available storage after:" 29 | sudo df -h 30 | echo 31 | 32 | - uses: actions/checkout@v4 33 | 34 | - name: Set up Node.js 35 | uses: actions/setup-node@v4 36 | with: 37 | node-version: 22 38 | cache: 'npm' 39 | 40 | - name: Cache 41 | uses: actions/cache@v4 42 | with: 43 | key: 'v1' 44 | path: cache 45 | 46 | - name: Install dependencies 47 | run: npm ci 48 | 49 | - name: Create all data 50 | run: npm run run:all 51 | 52 | - uses: aws-actions/configure-aws-credentials@v4 53 | with: 54 | role-to-assume: ${{ vars.AWS_ROLE_ARN }} 55 | aws-region: ap-northeast-1 56 | 57 | - name: Deploy to AWS 58 | run: | 59 | ./deploy/01_sync_to_s3.sh 60 | 61 | - name: Create archive 62 | run: | 63 | ./deploy/01a_create_archive.sh 64 | 65 | - name: Upload archive to S3 66 | run: | 67 | aws s3 cp \ 68 | ./out/api.tar.zst s3://japanese-addresses-v2.geoloniamaps.com/experimental/api.tar.zst 69 | 70 | clear-cdn-cache: 71 | needs: 72 | - create-data 73 | 74 | runs-on: ubuntu-latest 75 | 76 | steps: 77 | - uses: actions/checkout@v4 78 | 79 | - name: Clear Cloudflare cache 80 | env: 81 | CLOUDFLARE_ZONE_ID: ${{ vars.CLOUDFLARE_ZONE_ID }} 82 | CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_PURGE_TOKEN }} 83 | run: | 84 | ./deploy/02_clear_cloudflare_cache.sh 85 | -------------------------------------------------------------------------------- /.github/workflows/create-release.yml: -------------------------------------------------------------------------------- 1 | name: Auto Release 2 | 3 | permissions: 4 | contents: write 5 | on: 6 | push: 7 | tags: 8 | - 'v*.*.*' 9 | 10 | jobs: 11 | release: 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - name: Checkout repository 16 | uses: actions/checkout@v4 17 | 18 | - name: Set up Node.js 19 | uses: actions/setup-node@v4 20 | with: 21 | node-version: 22 22 | registry-url: 'https://registry.npmjs.org' 23 | scope: '@geolonia' 24 | cache: 'npm' 25 | 26 | - name: Install dependencies 27 | run: npm ci 28 | 29 | - name: Build 30 | run: npm run build 31 | 32 | - name: Create GitHub release 33 | uses: softprops/action-gh-release@v2 34 | with: 35 | name: ${{ github.ref_name }} 36 | tag_name: ${{ github.ref_name }} 37 | generate_release_notes: true 38 | 39 | - name: Publish to npm 40 | run: npm publish --access=public 41 | env: 42 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 43 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Run Tests 2 | 3 | on: 4 | pull_request: 5 | branches: 6 | - main 7 | push: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | test: 13 | runs-on: ubuntu-latest 14 | 15 | steps: 16 | - uses: actions/checkout@v4 17 | 18 | - name: Set up Node.js 19 | uses: actions/setup-node@v4 20 | with: 21 | node-version: 22 22 | cache: 'npm' 23 | 24 | - name: Cache 25 | uses: actions/cache@v4 26 | with: 27 | key: v1-${{ hashFiles('src/**') }} 28 | path: cache 29 | 30 | - name: Install dependencies 31 | run: npm ci 32 | 33 | - name: Build 34 | run: npm run build 35 | 36 | - name: Lint 37 | run: npm run lint 38 | 39 | - name: Run tests 40 | run: npm run test 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | build 4 | out 5 | cache 6 | .envrc 7 | *.0x 8 | *.7z 9 | /settings.json 10 | explorer 11 | geolonia-japanese-addresses-v2-*.tgz 12 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | nodejs 22.9.0 2 | -------------------------------------------------------------------------------- /DATA.md: -------------------------------------------------------------------------------- 1 | # データ作成について 2 | 3 | 下記元データのリンク 4 | 5 | * [全アドレスデータ](https://catalog.registries.digital.go.jp/rc/dataset/ba000001) (位置参照拡張含む、**地番マスターは含まず**) 6 | * [日本 都道府県マスター データセット `mt_pref_all`](https://catalog.registries.digital.go.jp/rc/dataset/ba-o1-000000_g2-000001) [位置参照拡張 `mt_pref_pos_all`](https://catalog.registries.digital.go.jp/rc/dataset/ba-o1-000000_g2-000012) 7 | * [日本 市区町村マスター データセット `mt_city_all`](https://catalog.registries.digital.go.jp/rc/dataset/ba-o1-000000_g2-000002) [位置参照拡張 `mt_city_pos_all`](https://catalog.registries.digital.go.jp/rc/dataset/ba-o1-000000_g2-000013) 8 | * [日本 町字マスター データセット `mt_town_all`](https://catalog.registries.digital.go.jp/rc/dataset/ba-o1-000000_g2-000003) [位置参照拡張 `mt_town_pos_all`](https://catalog.registries.digital.go.jp/rc/dataset/ba000004) <- こちらの mt_town_pos_all は zip 内に1つのCSVファイルではなく、それぞれの都道府県に分けられた csv.zip が含まれています 9 | * [全国　住居表示-住居マスター　データセット `mt_rsdtdsp_rsdt_all`](https://catalog.registries.digital.go.jp/rc/dataset/ba000003) [位置参照拡張 `mt_rsdtdsp_rsdt_pos_all`](https://catalog.registries.digital.go.jp/rc/dataset/ba000006) 10 | * 地番マスター（全国データなし、それぞれの市区町村個別でダウンロード） 11 | * (都道府県)(市区町村) `mt_parcel_cityXXXXXX` 位置参照拡張 `mt_parcel_pos_cityXXXXXX` 12 | * [北海道北斗市](https://catalog.registries.digital.go.jp/rc/dataset/ba-o1-012360_g2-000010) [位置参照拡張](https://catalog.registries.digital.go.jp/rc/dataset/ba-o1-012360_g2-000011) 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Geolonia, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Geolonia 住所データツール v2 2 | 3 | [![NPM Version](https://img.shields.io/npm/v/%40geolonia%2Fjapanese-addresses-v2)](https://www.npmjs.com/package/@geolonia/japanese-addresses-v2) 4 | 5 | 全国の住所データを HTTP API として公開するためのツールを公開いたします。 6 | 7 | 本データは、デジタル庁が整備する「[アドレス・ベース・レジストリ](https://www.digital.go.jp/policies/base_registry_address)」を元に加工し、様々なアプリケーションから便利に使えるように整理したものとなります。 8 | 9 | 以前、「[Geolonia 住所データ](https://github.com/geolonia/japanese-addresses)」を管理しましたが、v2は従来版と比べて下記の違いがあります。 10 | 11 | * 住居表示住所データと対応（番地・号までのデータが含まれる） 12 | * 地番住所のデータと対応（住居表示住所が導入されていない地域のデータが含まれる） 13 | 14 | [リリースノート](https://github.com/geolonia/japanese-addresses-v2/releases) 15 | 16 | ## API 17 | 18 | このデータを使用した API をご提供しています。現在、制限無しの無料公開をしていますが、様子見ながら公開を停止や変更など行うことがあります。商用稼働は、ご自身でデータを作成しホスティングすることを強くおすすめします。 Geolonia は有償で管理・ホスティングするサービスありますので、ご利用の方はお問い合わせてください。 19 | 20 | #### 都道府県エンドポイント 21 | 22 | ``` 23 | https://japanese-addresses-v2.geoloniamaps.com/api/ja.json 24 | ``` 25 | 26 | 例: [https://japanese-addresses-v2.geoloniamaps.com/api/ja.json](https://japanese-addresses-v2.geoloniamaps.com/api/ja.json) 27 | 28 | ``` 29 | { "meta": { "updated": 00000 }, "data": [ 30 | { 31 | "code": 10006, 32 | "pref": "北海道", 33 | "point": [ 34 | 141.347906782, 35 | 43.0639406375 36 | ] 37 | "cities": [ 38 | { 39 | "code": 11011, 40 | "city": "札幌市", 41 | "ward": "中央区", 42 | "point": [ 43 | 141.35389, 44 | 43.061414 45 | ] 46 | }, 47 | ... 48 | ] 49 | }, 50 | ... 51 | ``` 52 | 53 | #### 町字エンドポイント 54 | 55 | ``` 56 | https://japanese-addresses-v2.geoloniamaps.com/api/ja/<都道府県名>/<市区町村名>.json 57 | ``` 58 | 59 | ※ 都道府県名及び市区町村名は URL エンコードを行ってください。 60 | 61 | 例: [https://japanese-addresses-v2.geoloniamaps.com/api/ja/%E6%9D%B1%E4%BA%AC%E9%83%BD/%E6%96%B0%E5%AE%BF%E5%8C%BA.json](https://japanese-addresses-v2.geoloniamaps.com/api/ja/%E6%9D%B1%E4%BA%AC%E9%83%BD/%E6%96%B0%E5%AE%BF%E5%8C%BA.json) 62 | 63 | ``` 64 | { "meta": { "updated": 00000 }, "data": [ 65 | ... 66 | // 地番情報なしの住居表示未実施大字「新宿区北町」 67 | {"machiaza_id":"0001000","oaza_cho":"北町","point":[139.735037,35.69995]}, 68 | 69 | // 地番情報ありの住居表示実施「新宿区新宿三丁目」 70 | {"machiaza_id":"0020003","oaza_cho":"新宿","chome":"三丁目","rsdt":true,"point":[139.703563,35.691227],"csv_ranges":{"住居表示":{"start":151421,"length":19193},"地番":{"start":189779,"length":8895}}}, 71 | ... 72 | ``` 73 | 74 | ### 注意 75 | 76 | * 町丁目エンドポイントは、すべての地名を網羅しているわけではありません。 77 | * アドレス・ベース・レジストリの整備状況によって住居表示が実施されている町字でも住居表示住所のデータが無いや、地番住所のデータが無いなどのことがあります。また、住居表示や地番の文字列が存在しても位置情報データがまだ存在しないケースもあります。 78 | 79 | ## 住所データ・ API のビルド 80 | 81 | ```shell 82 | $ git clone git@github.com:geolonia/japanese-addresses-v2.git 83 | $ cd japanese-addresses-v2 84 | $ npm install 85 | $ npm run run:all # APIを全て生成する 86 | 87 | # オプション 88 | $ npm run run:01_make_prefecture_city # 都道府県・市区町村のみ作成 89 | $ npm run run:02_make_machi_aza # 町字API作成 90 | $ npm run run:03_make_rsdt # 住居表示住所API作成 (町字APIが先に作らないとエラーになります) 91 | $ npm run run:04_make_chiban # 地番住所API作成 (町字APIが先に作らないとエラーになります) 92 | ``` 93 | 94 | #### APIビルド設定 95 | 96 | `settings.json` に設定を入れてください。内容は `src/lib/settings.ts` を参照してください。 97 | 98 | 例えば、出力を北海道にしぼりたい場合は下記のように設定してください。 99 | 100 | ```json 101 | { 102 | "lgCodes": ["^01"] 103 | } 104 | ``` 105 | 106 | #### アーカイブファイル作成 107 | 108 | `deploy/01a_create_archive.sh` を参照してください 109 | 110 | ### API の構成 111 | 112 | ```shell 113 | └── api 114 |     ├── ja 115 |     │   │── {都道府県名} 116 |     │   │   ├── {市区町村名}-住居表示.txt 117 |     │   │   ├── {市区町村名}-地番.txt 118 |     │ │ └── {市区町村名}.json # 町字一覧 119 |     │ └── {都道府県名}.json # 市区町村一覧 120 |     └── ja.json # 都道府県と市区町村の一覧 121 | ``` 122 | 123 | 各ファイルの詳細な仕様は、 `src/data.ts` の型定義を参照してください。 124 | 125 | #### `txt` ファイル内のフォーマットについて 126 | 127 | `-地番.txt` と `-住居表示.txt` は容量節約のため、市区町村の住所を全て一つのファイルに集約するものです。そのテキストファイルのフォーマットは下記となります。 128 | 129 | 130 | ``` 131 | <町字1>,, 132 | <町字2>,, 133 | ... 134 | =END= 135 | 地番,<町字1> 136 | prc_num1,prc_num2,prc_num3,lng,lat 137 | 1,,,<経度>,<緯度> 138 | ... 139 | ``` 140 | 141 | 論理的な構造としては 142 | 143 | 1. ファイル全体のヘッダー 144 | 1. ファイル内の町字（丁目、小字含む）一覧 145 | 1. それぞれのデータのバイトレンジ（開始バイト数、容量バイト数） 146 | 1. 町字データ（ループ） 147 | 1. 町字データのヘッダー 148 | 1. 地番・住居表示か 149 | 1. カラム名定義 150 | 1. 住所・位置情報データ 151 | 152 | ヘッダーは5万バイトの倍数となります。末尾に `=END=` を挿入し、残りまで `0x20` (半角スペース)で埋めます。クライアントは `=END=` を確認できるまで、5万バイトずつ読み込むことをおすすめします。 153 | 154 | また、 `api/ja/{都道府県名}/{市区町村名}.json` エンドポイントにも、 `csv_ranges` にヘッダーの start / length 情報入っているので、市区町村エンドポイントを既に読み込まれている場合はそのまま利用することをおすすめしております。 155 | 156 | ## 出典 157 | 158 | 本データは、以下のデータを元に、毎月 Geolonia にて更新作業を行っています。 159 | 160 | 「アドレス・ベース・レジストリ」（デジタル庁）[住居表示住所・住居マスターデータセット](https://catalog.registries.digital.go.jp/) をもとに株式会社 Geolonia が作成したものです。 161 | 162 | ## 貢献方法 163 | 164 | * 本データに不具合がある場合には、[Issue](https://github.com/geolonia/japanese-addresses-v2/issues) または[プルリクエスト](https://github.com/geolonia/japanese-addresses-v2/pulls)にてご報告ください。 165 | 166 | ## japanese-addresses-v2を使っているプロジェクト 167 | 168 | * [normalize-japanese-addresses](https://github.com/geolonia/normalize-japanese-addresses) 日本の住所を正規化するライブラリ 169 | 170 | ## スポンサー 171 | 172 | * [一般社団法人 不動産テック協会](https://retechjapan.org/) 173 | 174 | ## 関連情報 175 | 176 | * [【プレスリリース】不動産テック協会、Geolonia と共同で不動産情報の共通 ID 付与の取り組みを開始](https://retechjapan.org/news/archives/pressrelease-20200731/) 177 | * [【プレスリリース】日本全国の住所マスターデータをオープンデータとして無料公開](https://geolonia.com/pressrelease/2020/08/05/japanese-addresses.html) 178 | 179 | ## ライセンス 180 | 181 | Geolonia 住所データのライセンスは以下のとおりです。 182 | 183 | [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/deed.ja) 184 | 185 | 注: リポジトリに同梱されているデータ生成用のスクリプトのライセンスは MIT ライセンスとします。 186 | -------------------------------------------------------------------------------- /deploy/01_sync_to_s3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | aws s3 sync \ 4 | --cache-control="max-age=86400, public" \ 5 | --exclude="*" \ 6 | --include="*.txt" \ 7 | --content-type="text/plain; charset=utf-8" \ 8 | ./out/ s3://japanese-addresses-v2.geoloniamaps.com/ 9 | aws s3 sync \ 10 | --cache-control="max-age=86400, public" \ 11 | --exclude="*" \ 12 | --include="*.json" \ 13 | --content-type="application/json; charset=utf-8" \ 14 | ./out/ s3://japanese-addresses-v2.geoloniamaps.com/ 15 | -------------------------------------------------------------------------------- /deploy/01a_create_archive.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | cd "$(dirname "$0")"/../out 4 | tar -cf "api.tar" ./api 5 | zstd -T0 -19 --rm -z "api.tar" 6 | -------------------------------------------------------------------------------- /deploy/02_clear_cloudflare_cache.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | curl --request POST \ 4 | --url https://api.cloudflare.com/client/v4/zones/$CLOUDFLARE_ZONE_ID/purge_cache \ 5 | --header 'Content-Type: application/json' \ 6 | --header "Authorization: Bearer ${CLOUDFLARE_TOKEN}" \ 7 | --data '{ 8 | "files": [ 9 | "https://japanese-addresses-v2.geoloniamaps.com/api/ja.json" 10 | ] 11 | }' 12 | -------------------------------------------------------------------------------- /docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Geolonia japanese-addresses-v2 7 | 8 | 9 | 61 | 62 | 63 |
64 | 65 |
66 |

@geolonia/japanese-addresses-v2

67 |
68 | 69 | 70 | 71 | 182 | 183 | -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | //@ts-check 2 | import eslint from '@eslint/js'; 3 | import tseslint from 'typescript-eslint'; 4 | 5 | export default tseslint.config( 6 | eslint.configs.recommended, 7 | tseslint.configs.recommended, 8 | tseslint.configs.recommendedTypeChecked, 9 | { 10 | languageOptions: { 11 | parserOptions: { 12 | projectService: true, 13 | tsconfigRootDir: import.meta.dirname, 14 | }, 15 | } 16 | }, 17 | { 18 | ignores: [ 19 | 'cache', 20 | 'node_modules', 21 | 'dist', 22 | 'eslint.config.mjs', 23 | // '**/*.test.ts', 24 | ], 25 | } 26 | ); 27 | -------------------------------------------------------------------------------- /out/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geolonia/japanese-addresses-v2/6fa7674951437a05ad9368fad3b0bac38c463e84/out/.keep -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@geolonia/japanese-addresses-v2", 3 | "type": "module", 4 | "version": "0.0.5", 5 | "description": "", 6 | "exports": { 7 | "import": "./dist/esm/data.mjs", 8 | "require": "./dist/cjs/data.cjs", 9 | "types": "./dist/esm/data.d.ts" 10 | }, 11 | "main": "./dist/cjs/data.cjs", 12 | "types": "./dist/esm/data.d.ts", 13 | "scripts": { 14 | "prepack": "npm run clean && npm run build", 15 | "clean": "shx rm -rf ./dist", 16 | "build:proto": "protoc --plugin=./node_modules/.bin/protoc-gen-ts_proto --ts_proto_out=. ./src/address_data.proto", 17 | "build:dev": "tsc", 18 | "build": "tsc -p tsconfig.dist-cjs.json && tsc -p tsconfig.dist-esm.json && mv ./dist/cjs/data.js ./dist/cjs/data.cjs && mv ./dist/esm/data.js ./dist/esm/data.mjs", 19 | "clear:cache": "shx rm -rf ./cache", 20 | "run:all": "npm run run:01_make_prefecture_city && npm run run:02_make_machi_aza && npm run run:03_make_rsdt && npm run run:04_make_chiban && npm run run:10_refresh_csv_ranges && npm run run:99_create_stats", 21 | "run:01_make_prefecture_city": "tsx ./src/01_make_prefecture_city.ts", 22 | "run:02_make_machi_aza": "tsx ./src/02_make_machi_aza.ts", 23 | "run:03_make_rsdt": "node --max-old-space-size=8192 --import tsx ./src/03_make_rsdt.ts", 24 | "run:04_make_chiban": "node --max-old-space-size=8192 --import tsx ./src/04_make_chiban.ts", 25 | "run:10_refresh_csv_ranges": "tsx ./src/10_refresh_csv_ranges.ts", 26 | "run:99_create_stats": "tsx ./src/99_create_stats.ts", 27 | "create:archive": "rm ./api.7z; 7zz a ./api.7z ./out/api", 28 | "start": "http-server --cors=\"*\" ./out", 29 | "lint": "eslint ./src", 30 | "test": "glob -c \"node --test --import tsx\" \"./src/**/*.test.ts\"" 31 | }, 32 | "keywords": [], 33 | "author": "", 34 | "license": "MIT", 35 | "files": [ 36 | "dist/**/*" 37 | ], 38 | "devDependencies": { 39 | "@eslint/js": "^9.17.0", 40 | "@tsconfig/node22": "^22.0.0", 41 | "@types/better-sqlite3": "^7.6.12", 42 | "@types/cli-progress": "^3.11.6", 43 | "@types/node": "^22.10.2", 44 | "@types/proj4": "^2.5.5", 45 | "@types/unzipper": "^0.10.10", 46 | "better-sqlite3": "^11.7.2", 47 | "cli-progress": "^3.12.0", 48 | "csv-parse": "^5.6.0", 49 | "eslint": "^9.17.0", 50 | "glob": "^11.0.0", 51 | "http-server": "^14.1.1", 52 | "iconv-lite": "^0.6.3", 53 | "lru-cache": "^11.0.2", 54 | "proj4": "^2.12.1", 55 | "shx": "^0.3.4", 56 | "ts-proto": "^2.6.1", 57 | "tsx": "^4.19.1", 58 | "typescript": "^5.7.3", 59 | "typescript-eslint": "^8.19.1", 60 | "undici": "^7.2.1", 61 | "unzipper": "^0.12.3" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/01_make_prefecture_city.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import main from './processes/01_make_prefecture_city.js'; 4 | 5 | main(process.argv) 6 | .then(() => { 7 | process.exit(0); 8 | }) 9 | .catch((e) => { 10 | console.error(e); 11 | process.exit(1); 12 | }); 13 | -------------------------------------------------------------------------------- /src/02_make_machi_aza.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import main from './processes/02_make_machi_aza.js'; 4 | 5 | main(process.argv) 6 | .then(() => { 7 | process.exit(0); 8 | }) 9 | .catch((e) => { 10 | console.error(e); 11 | process.exit(1); 12 | }); 13 | -------------------------------------------------------------------------------- /src/03_make_rsdt.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import main from './processes/03_make_rsdt.js'; 4 | 5 | main(process.argv) 6 | .then(() => { 7 | process.exit(0); 8 | }) 9 | .catch((e) => { 10 | console.error(e); 11 | process.exit(1); 12 | }); 13 | -------------------------------------------------------------------------------- /src/04_make_chiban.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import main from './processes/04_make_chiban.js'; 4 | 5 | main(process.argv) 6 | .then(() => { 7 | process.exit(0); 8 | }) 9 | .catch((e) => { 10 | console.error(e); 11 | process.exit(1); 12 | }); 13 | -------------------------------------------------------------------------------- /src/10_refresh_csv_ranges.ts: -------------------------------------------------------------------------------- 1 | import main from './processes/10_refresh_csv_ranges.js'; 2 | 3 | main(process.argv) 4 | .then(() => { 5 | process.exit(0); 6 | }) 7 | .catch((e) => { 8 | console.error(e); 9 | process.exit(1); 10 | }); 11 | -------------------------------------------------------------------------------- /src/99_create_stats.ts: -------------------------------------------------------------------------------- 1 | import main from './processes/99_create_stats.js'; 2 | 3 | main(process.argv) 4 | .then(() => { 5 | process.exit(0); 6 | }).catch((e) => { 7 | console.error(e); 8 | process.exit(1); 9 | }); 10 | -------------------------------------------------------------------------------- /src/address_data.proto: -------------------------------------------------------------------------------- 1 | message Header { 2 | required Kind kind = 1; 3 | // required string name = 2; 4 | repeated HeaderRow rows = 3; 5 | } 6 | 7 | message HeaderRow { 8 | required string name = 1; 9 | required uint32 offset = 2; 10 | required uint32 length = 3; 11 | } 12 | 13 | enum Kind { 14 | RSDT = 0; 15 | CHIBAN = 1; 16 | } 17 | 18 | message Section { 19 | required Kind kind = 1; 20 | required string name = 2; 21 | 22 | repeated RsdtRow rsdt_rows = 3; 23 | repeated ChibanRow chiban_rows = 4; 24 | } 25 | 26 | message RsdtRow { 27 | optional uint32 blk_num = 1; 28 | required uint32 rsdt_num = 2; 29 | optional uint32 rsdt_num2 = 3; 30 | optional LngLatPoint point = 4; 31 | 32 | /** These strings are only used if the data cannot be expressed by an integer. */ 33 | optional string blk_num_str = 5; 34 | optional string rsdt_num_str = 6; 35 | optional string rsdt_num2_str = 7; 36 | } 37 | 38 | message ChibanRow { 39 | required uint32 prc_num1 = 1; 40 | optional uint32 prc_num2 = 2; 41 | optional uint32 prc_num3 = 3; 42 | optional LngLatPoint point = 4; 43 | 44 | /** These strings are only used if the data cannot be expressed by an integer. */ 45 | optional string prc_num1_str = 5; 46 | optional string prc_num2_str = 6; 47 | optional string prc_num3_str = 7; 48 | } 49 | 50 | message LngLatPoint { 51 | required double lng = 1; 52 | required double lat = 2; 53 | } 54 | -------------------------------------------------------------------------------- /src/address_data.ts: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-ts_proto. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-ts_proto v2.2.0 4 | // protoc v5.28.1 5 | // source: src/address_data.proto 6 | 7 | /* eslint-disable */ 8 | import { BinaryReader, BinaryWriter } from "@bufbuild/protobuf/wire"; 9 | 10 | export const protobufPackage = ""; 11 | 12 | export enum Kind { 13 | RSDT = 0, 14 | CHIBAN = 1, 15 | UNRECOGNIZED = -1, 16 | } 17 | 18 | export function kindFromJSON(object: any): Kind { 19 | switch (object) { 20 | case 0: 21 | case "RSDT": 22 | return Kind.RSDT; 23 | case 1: 24 | case "CHIBAN": 25 | return Kind.CHIBAN; 26 | case -1: 27 | case "UNRECOGNIZED": 28 | default: 29 | return Kind.UNRECOGNIZED; 30 | } 31 | } 32 | 33 | export function kindToJSON(object: Kind): string { 34 | switch (object) { 35 | case Kind.RSDT: 36 | return "RSDT"; 37 | case Kind.CHIBAN: 38 | return "CHIBAN"; 39 | case Kind.UNRECOGNIZED: 40 | default: 41 | return "UNRECOGNIZED"; 42 | } 43 | } 44 | 45 | export interface Header { 46 | kind: Kind; 47 | /** required string name = 2; */ 48 | rows: HeaderRow[]; 49 | } 50 | 51 | export interface HeaderRow { 52 | name: string; 53 | offset: number; 54 | length: number; 55 | } 56 | 57 | export interface Section { 58 | kind: Kind; 59 | name: string; 60 | rsdtRows: RsdtRow[]; 61 | chibanRows: ChibanRow[]; 62 | } 63 | 64 | export interface RsdtRow { 65 | blkNum?: number | undefined; 66 | rsdtNum: number; 67 | rsdtNum2?: number | undefined; 68 | point?: 69 | | LngLatPoint 70 | | undefined; 71 | /** These strings are only used if the data cannot be expressed by an integer. */ 72 | blkNumStr?: string | undefined; 73 | rsdtNumStr?: string | undefined; 74 | rsdtNum2Str?: string | undefined; 75 | } 76 | 77 | export interface ChibanRow { 78 | prcNum1: number; 79 | prcNum2?: number | undefined; 80 | prcNum3?: number | undefined; 81 | point?: 82 | | LngLatPoint 83 | | undefined; 84 | /** These strings are only used if the data cannot be expressed by an integer. */ 85 | prcNum1Str?: string | undefined; 86 | prcNum2Str?: string | undefined; 87 | prcNum3Str?: string | undefined; 88 | } 89 | 90 | export interface LngLatPoint { 91 | lng: number; 92 | lat: number; 93 | } 94 | 95 | function createBaseHeader(): Header { 96 | return { kind: 0, rows: [] }; 97 | } 98 | 99 | export const Header: MessageFns
= { 100 | encode(message: Header, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { 101 | if (message.kind !== 0) { 102 | writer.uint32(8).int32(message.kind); 103 | } 104 | for (const v of message.rows) { 105 | HeaderRow.encode(v!, writer.uint32(26).fork()).join(); 106 | } 107 | return writer; 108 | }, 109 | 110 | decode(input: BinaryReader | Uint8Array, length?: number): Header { 111 | const reader = input instanceof BinaryReader ? input : new BinaryReader(input); 112 | let end = length === undefined ? reader.len : reader.pos + length; 113 | const message = createBaseHeader(); 114 | while (reader.pos < end) { 115 | const tag = reader.uint32(); 116 | switch (tag >>> 3) { 117 | case 1: 118 | if (tag !== 8) { 119 | break; 120 | } 121 | 122 | message.kind = reader.int32() as any; 123 | continue; 124 | case 3: 125 | if (tag !== 26) { 126 | break; 127 | } 128 | 129 | message.rows.push(HeaderRow.decode(reader, reader.uint32())); 130 | continue; 131 | } 132 | if ((tag & 7) === 4 || tag === 0) { 133 | break; 134 | } 135 | reader.skip(tag & 7); 136 | } 137 | return message; 138 | }, 139 | 140 | fromJSON(object: any): Header { 141 | return { 142 | kind: isSet(object.kind) ? kindFromJSON(object.kind) : 0, 143 | rows: globalThis.Array.isArray(object?.rows) ? object.rows.map((e: any) => HeaderRow.fromJSON(e)) : [], 144 | }; 145 | }, 146 | 147 | toJSON(message: Header): unknown { 148 | const obj: any = {}; 149 | if (message.kind !== 0) { 150 | obj.kind = kindToJSON(message.kind); 151 | } 152 | if (message.rows?.length) { 153 | obj.rows = message.rows.map((e) => HeaderRow.toJSON(e)); 154 | } 155 | return obj; 156 | }, 157 | 158 | create, I>>(base?: I): Header { 159 | return Header.fromPartial(base ?? ({} as any)); 160 | }, 161 | fromPartial, I>>(object: I): Header { 162 | const message = createBaseHeader(); 163 | message.kind = object.kind ?? 0; 164 | message.rows = object.rows?.map((e) => HeaderRow.fromPartial(e)) || []; 165 | return message; 166 | }, 167 | }; 168 | 169 | function createBaseHeaderRow(): HeaderRow { 170 | return { name: "", offset: 0, length: 0 }; 171 | } 172 | 173 | export const HeaderRow: MessageFns = { 174 | encode(message: HeaderRow, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { 175 | if (message.name !== "") { 176 | writer.uint32(10).string(message.name); 177 | } 178 | if (message.offset !== 0) { 179 | writer.uint32(16).uint32(message.offset); 180 | } 181 | if (message.length !== 0) { 182 | writer.uint32(24).uint32(message.length); 183 | } 184 | return writer; 185 | }, 186 | 187 | decode(input: BinaryReader | Uint8Array, length?: number): HeaderRow { 188 | const reader = input instanceof BinaryReader ? input : new BinaryReader(input); 189 | let end = length === undefined ? reader.len : reader.pos + length; 190 | const message = createBaseHeaderRow(); 191 | while (reader.pos < end) { 192 | const tag = reader.uint32(); 193 | switch (tag >>> 3) { 194 | case 1: 195 | if (tag !== 10) { 196 | break; 197 | } 198 | 199 | message.name = reader.string(); 200 | continue; 201 | case 2: 202 | if (tag !== 16) { 203 | break; 204 | } 205 | 206 | message.offset = reader.uint32(); 207 | continue; 208 | case 3: 209 | if (tag !== 24) { 210 | break; 211 | } 212 | 213 | message.length = reader.uint32(); 214 | continue; 215 | } 216 | if ((tag & 7) === 4 || tag === 0) { 217 | break; 218 | } 219 | reader.skip(tag & 7); 220 | } 221 | return message; 222 | }, 223 | 224 | fromJSON(object: any): HeaderRow { 225 | return { 226 | name: isSet(object.name) ? globalThis.String(object.name) : "", 227 | offset: isSet(object.offset) ? globalThis.Number(object.offset) : 0, 228 | length: isSet(object.length) ? globalThis.Number(object.length) : 0, 229 | }; 230 | }, 231 | 232 | toJSON(message: HeaderRow): unknown { 233 | const obj: any = {}; 234 | if (message.name !== "") { 235 | obj.name = message.name; 236 | } 237 | if (message.offset !== 0) { 238 | obj.offset = Math.round(message.offset); 239 | } 240 | if (message.length !== 0) { 241 | obj.length = Math.round(message.length); 242 | } 243 | return obj; 244 | }, 245 | 246 | create, I>>(base?: I): HeaderRow { 247 | return HeaderRow.fromPartial(base ?? ({} as any)); 248 | }, 249 | fromPartial, I>>(object: I): HeaderRow { 250 | const message = createBaseHeaderRow(); 251 | message.name = object.name ?? ""; 252 | message.offset = object.offset ?? 0; 253 | message.length = object.length ?? 0; 254 | return message; 255 | }, 256 | }; 257 | 258 | function createBaseSection(): Section { 259 | return { kind: 0, name: "", rsdtRows: [], chibanRows: [] }; 260 | } 261 | 262 | export const Section: MessageFns
= { 263 | encode(message: Section, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { 264 | if (message.kind !== 0) { 265 | writer.uint32(8).int32(message.kind); 266 | } 267 | if (message.name !== "") { 268 | writer.uint32(18).string(message.name); 269 | } 270 | for (const v of message.rsdtRows) { 271 | RsdtRow.encode(v!, writer.uint32(26).fork()).join(); 272 | } 273 | for (const v of message.chibanRows) { 274 | ChibanRow.encode(v!, writer.uint32(34).fork()).join(); 275 | } 276 | return writer; 277 | }, 278 | 279 | decode(input: BinaryReader | Uint8Array, length?: number): Section { 280 | const reader = input instanceof BinaryReader ? input : new BinaryReader(input); 281 | let end = length === undefined ? reader.len : reader.pos + length; 282 | const message = createBaseSection(); 283 | while (reader.pos < end) { 284 | const tag = reader.uint32(); 285 | switch (tag >>> 3) { 286 | case 1: 287 | if (tag !== 8) { 288 | break; 289 | } 290 | 291 | message.kind = reader.int32() as any; 292 | continue; 293 | case 2: 294 | if (tag !== 18) { 295 | break; 296 | } 297 | 298 | message.name = reader.string(); 299 | continue; 300 | case 3: 301 | if (tag !== 26) { 302 | break; 303 | } 304 | 305 | message.rsdtRows.push(RsdtRow.decode(reader, reader.uint32())); 306 | continue; 307 | case 4: 308 | if (tag !== 34) { 309 | break; 310 | } 311 | 312 | message.chibanRows.push(ChibanRow.decode(reader, reader.uint32())); 313 | continue; 314 | } 315 | if ((tag & 7) === 4 || tag === 0) { 316 | break; 317 | } 318 | reader.skip(tag & 7); 319 | } 320 | return message; 321 | }, 322 | 323 | fromJSON(object: any): Section { 324 | return { 325 | kind: isSet(object.kind) ? kindFromJSON(object.kind) : 0, 326 | name: isSet(object.name) ? globalThis.String(object.name) : "", 327 | rsdtRows: globalThis.Array.isArray(object?.rsdtRows) ? object.rsdtRows.map((e: any) => RsdtRow.fromJSON(e)) : [], 328 | chibanRows: globalThis.Array.isArray(object?.chibanRows) 329 | ? object.chibanRows.map((e: any) => ChibanRow.fromJSON(e)) 330 | : [], 331 | }; 332 | }, 333 | 334 | toJSON(message: Section): unknown { 335 | const obj: any = {}; 336 | if (message.kind !== 0) { 337 | obj.kind = kindToJSON(message.kind); 338 | } 339 | if (message.name !== "") { 340 | obj.name = message.name; 341 | } 342 | if (message.rsdtRows?.length) { 343 | obj.rsdtRows = message.rsdtRows.map((e) => RsdtRow.toJSON(e)); 344 | } 345 | if (message.chibanRows?.length) { 346 | obj.chibanRows = message.chibanRows.map((e) => ChibanRow.toJSON(e)); 347 | } 348 | return obj; 349 | }, 350 | 351 | create, I>>(base?: I): Section { 352 | return Section.fromPartial(base ?? ({} as any)); 353 | }, 354 | fromPartial, I>>(object: I): Section { 355 | const message = createBaseSection(); 356 | message.kind = object.kind ?? 0; 357 | message.name = object.name ?? ""; 358 | message.rsdtRows = object.rsdtRows?.map((e) => RsdtRow.fromPartial(e)) || []; 359 | message.chibanRows = object.chibanRows?.map((e) => ChibanRow.fromPartial(e)) || []; 360 | return message; 361 | }, 362 | }; 363 | 364 | function createBaseRsdtRow(): RsdtRow { 365 | return { blkNum: 0, rsdtNum: 0, rsdtNum2: 0, point: undefined, blkNumStr: "", rsdtNumStr: "", rsdtNum2Str: "" }; 366 | } 367 | 368 | export const RsdtRow: MessageFns = { 369 | encode(message: RsdtRow, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { 370 | if (message.blkNum !== undefined && message.blkNum !== 0) { 371 | writer.uint32(8).uint32(message.blkNum); 372 | } 373 | if (message.rsdtNum !== 0) { 374 | writer.uint32(16).uint32(message.rsdtNum); 375 | } 376 | if (message.rsdtNum2 !== undefined && message.rsdtNum2 !== 0) { 377 | writer.uint32(24).uint32(message.rsdtNum2); 378 | } 379 | if (message.point !== undefined) { 380 | LngLatPoint.encode(message.point, writer.uint32(34).fork()).join(); 381 | } 382 | if (message.blkNumStr !== undefined && message.blkNumStr !== "") { 383 | writer.uint32(42).string(message.blkNumStr); 384 | } 385 | if (message.rsdtNumStr !== undefined && message.rsdtNumStr !== "") { 386 | writer.uint32(50).string(message.rsdtNumStr); 387 | } 388 | if (message.rsdtNum2Str !== undefined && message.rsdtNum2Str !== "") { 389 | writer.uint32(58).string(message.rsdtNum2Str); 390 | } 391 | return writer; 392 | }, 393 | 394 | decode(input: BinaryReader | Uint8Array, length?: number): RsdtRow { 395 | const reader = input instanceof BinaryReader ? input : new BinaryReader(input); 396 | let end = length === undefined ? reader.len : reader.pos + length; 397 | const message = createBaseRsdtRow(); 398 | while (reader.pos < end) { 399 | const tag = reader.uint32(); 400 | switch (tag >>> 3) { 401 | case 1: 402 | if (tag !== 8) { 403 | break; 404 | } 405 | 406 | message.blkNum = reader.uint32(); 407 | continue; 408 | case 2: 409 | if (tag !== 16) { 410 | break; 411 | } 412 | 413 | message.rsdtNum = reader.uint32(); 414 | continue; 415 | case 3: 416 | if (tag !== 24) { 417 | break; 418 | } 419 | 420 | message.rsdtNum2 = reader.uint32(); 421 | continue; 422 | case 4: 423 | if (tag !== 34) { 424 | break; 425 | } 426 | 427 | message.point = LngLatPoint.decode(reader, reader.uint32()); 428 | continue; 429 | case 5: 430 | if (tag !== 42) { 431 | break; 432 | } 433 | 434 | message.blkNumStr = reader.string(); 435 | continue; 436 | case 6: 437 | if (tag !== 50) { 438 | break; 439 | } 440 | 441 | message.rsdtNumStr = reader.string(); 442 | continue; 443 | case 7: 444 | if (tag !== 58) { 445 | break; 446 | } 447 | 448 | message.rsdtNum2Str = reader.string(); 449 | continue; 450 | } 451 | if ((tag & 7) === 4 || tag === 0) { 452 | break; 453 | } 454 | reader.skip(tag & 7); 455 | } 456 | return message; 457 | }, 458 | 459 | fromJSON(object: any): RsdtRow { 460 | return { 461 | blkNum: isSet(object.blkNum) ? globalThis.Number(object.blkNum) : 0, 462 | rsdtNum: isSet(object.rsdtNum) ? globalThis.Number(object.rsdtNum) : 0, 463 | rsdtNum2: isSet(object.rsdtNum2) ? globalThis.Number(object.rsdtNum2) : 0, 464 | point: isSet(object.point) ? LngLatPoint.fromJSON(object.point) : undefined, 465 | blkNumStr: isSet(object.blkNumStr) ? globalThis.String(object.blkNumStr) : "", 466 | rsdtNumStr: isSet(object.rsdtNumStr) ? globalThis.String(object.rsdtNumStr) : "", 467 | rsdtNum2Str: isSet(object.rsdtNum2Str) ? globalThis.String(object.rsdtNum2Str) : "", 468 | }; 469 | }, 470 | 471 | toJSON(message: RsdtRow): unknown { 472 | const obj: any = {}; 473 | if (message.blkNum !== undefined && message.blkNum !== 0) { 474 | obj.blkNum = Math.round(message.blkNum); 475 | } 476 | if (message.rsdtNum !== 0) { 477 | obj.rsdtNum = Math.round(message.rsdtNum); 478 | } 479 | if (message.rsdtNum2 !== undefined && message.rsdtNum2 !== 0) { 480 | obj.rsdtNum2 = Math.round(message.rsdtNum2); 481 | } 482 | if (message.point !== undefined) { 483 | obj.point = LngLatPoint.toJSON(message.point); 484 | } 485 | if (message.blkNumStr !== undefined && message.blkNumStr !== "") { 486 | obj.blkNumStr = message.blkNumStr; 487 | } 488 | if (message.rsdtNumStr !== undefined && message.rsdtNumStr !== "") { 489 | obj.rsdtNumStr = message.rsdtNumStr; 490 | } 491 | if (message.rsdtNum2Str !== undefined && message.rsdtNum2Str !== "") { 492 | obj.rsdtNum2Str = message.rsdtNum2Str; 493 | } 494 | return obj; 495 | }, 496 | 497 | create, I>>(base?: I): RsdtRow { 498 | return RsdtRow.fromPartial(base ?? ({} as any)); 499 | }, 500 | fromPartial, I>>(object: I): RsdtRow { 501 | const message = createBaseRsdtRow(); 502 | message.blkNum = object.blkNum ?? 0; 503 | message.rsdtNum = object.rsdtNum ?? 0; 504 | message.rsdtNum2 = object.rsdtNum2 ?? 0; 505 | message.point = (object.point !== undefined && object.point !== null) 506 | ? LngLatPoint.fromPartial(object.point) 507 | : undefined; 508 | message.blkNumStr = object.blkNumStr ?? ""; 509 | message.rsdtNumStr = object.rsdtNumStr ?? ""; 510 | message.rsdtNum2Str = object.rsdtNum2Str ?? ""; 511 | return message; 512 | }, 513 | }; 514 | 515 | function createBaseChibanRow(): ChibanRow { 516 | return { prcNum1: 0, prcNum2: 0, prcNum3: 0, point: undefined, prcNum1Str: "", prcNum2Str: "", prcNum3Str: "" }; 517 | } 518 | 519 | export const ChibanRow: MessageFns = { 520 | encode(message: ChibanRow, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { 521 | if (message.prcNum1 !== 0) { 522 | writer.uint32(8).uint32(message.prcNum1); 523 | } 524 | if (message.prcNum2 !== undefined && message.prcNum2 !== 0) { 525 | writer.uint32(16).uint32(message.prcNum2); 526 | } 527 | if (message.prcNum3 !== undefined && message.prcNum3 !== 0) { 528 | writer.uint32(24).uint32(message.prcNum3); 529 | } 530 | if (message.point !== undefined) { 531 | LngLatPoint.encode(message.point, writer.uint32(34).fork()).join(); 532 | } 533 | if (message.prcNum1Str !== undefined && message.prcNum1Str !== "") { 534 | writer.uint32(42).string(message.prcNum1Str); 535 | } 536 | if (message.prcNum2Str !== undefined && message.prcNum2Str !== "") { 537 | writer.uint32(50).string(message.prcNum2Str); 538 | } 539 | if (message.prcNum3Str !== undefined && message.prcNum3Str !== "") { 540 | writer.uint32(58).string(message.prcNum3Str); 541 | } 542 | return writer; 543 | }, 544 | 545 | decode(input: BinaryReader | Uint8Array, length?: number): ChibanRow { 546 | const reader = input instanceof BinaryReader ? input : new BinaryReader(input); 547 | let end = length === undefined ? reader.len : reader.pos + length; 548 | const message = createBaseChibanRow(); 549 | while (reader.pos < end) { 550 | const tag = reader.uint32(); 551 | switch (tag >>> 3) { 552 | case 1: 553 | if (tag !== 8) { 554 | break; 555 | } 556 | 557 | message.prcNum1 = reader.uint32(); 558 | continue; 559 | case 2: 560 | if (tag !== 16) { 561 | break; 562 | } 563 | 564 | message.prcNum2 = reader.uint32(); 565 | continue; 566 | case 3: 567 | if (tag !== 24) { 568 | break; 569 | } 570 | 571 | message.prcNum3 = reader.uint32(); 572 | continue; 573 | case 4: 574 | if (tag !== 34) { 575 | break; 576 | } 577 | 578 | message.point = LngLatPoint.decode(reader, reader.uint32()); 579 | continue; 580 | case 5: 581 | if (tag !== 42) { 582 | break; 583 | } 584 | 585 | message.prcNum1Str = reader.string(); 586 | continue; 587 | case 6: 588 | if (tag !== 50) { 589 | break; 590 | } 591 | 592 | message.prcNum2Str = reader.string(); 593 | continue; 594 | case 7: 595 | if (tag !== 58) { 596 | break; 597 | } 598 | 599 | message.prcNum3Str = reader.string(); 600 | continue; 601 | } 602 | if ((tag & 7) === 4 || tag === 0) { 603 | break; 604 | } 605 | reader.skip(tag & 7); 606 | } 607 | return message; 608 | }, 609 | 610 | fromJSON(object: any): ChibanRow { 611 | return { 612 | prcNum1: isSet(object.prcNum1) ? globalThis.Number(object.prcNum1) : 0, 613 | prcNum2: isSet(object.prcNum2) ? globalThis.Number(object.prcNum2) : 0, 614 | prcNum3: isSet(object.prcNum3) ? globalThis.Number(object.prcNum3) : 0, 615 | point: isSet(object.point) ? LngLatPoint.fromJSON(object.point) : undefined, 616 | prcNum1Str: isSet(object.prcNum1Str) ? globalThis.String(object.prcNum1Str) : "", 617 | prcNum2Str: isSet(object.prcNum2Str) ? globalThis.String(object.prcNum2Str) : "", 618 | prcNum3Str: isSet(object.prcNum3Str) ? globalThis.String(object.prcNum3Str) : "", 619 | }; 620 | }, 621 | 622 | toJSON(message: ChibanRow): unknown { 623 | const obj: any = {}; 624 | if (message.prcNum1 !== 0) { 625 | obj.prcNum1 = Math.round(message.prcNum1); 626 | } 627 | if (message.prcNum2 !== undefined && message.prcNum2 !== 0) { 628 | obj.prcNum2 = Math.round(message.prcNum2); 629 | } 630 | if (message.prcNum3 !== undefined && message.prcNum3 !== 0) { 631 | obj.prcNum3 = Math.round(message.prcNum3); 632 | } 633 | if (message.point !== undefined) { 634 | obj.point = LngLatPoint.toJSON(message.point); 635 | } 636 | if (message.prcNum1Str !== undefined && message.prcNum1Str !== "") { 637 | obj.prcNum1Str = message.prcNum1Str; 638 | } 639 | if (message.prcNum2Str !== undefined && message.prcNum2Str !== "") { 640 | obj.prcNum2Str = message.prcNum2Str; 641 | } 642 | if (message.prcNum3Str !== undefined && message.prcNum3Str !== "") { 643 | obj.prcNum3Str = message.prcNum3Str; 644 | } 645 | return obj; 646 | }, 647 | 648 | create, I>>(base?: I): ChibanRow { 649 | return ChibanRow.fromPartial(base ?? ({} as any)); 650 | }, 651 | fromPartial, I>>(object: I): ChibanRow { 652 | const message = createBaseChibanRow(); 653 | message.prcNum1 = object.prcNum1 ?? 0; 654 | message.prcNum2 = object.prcNum2 ?? 0; 655 | message.prcNum3 = object.prcNum3 ?? 0; 656 | message.point = (object.point !== undefined && object.point !== null) 657 | ? LngLatPoint.fromPartial(object.point) 658 | : undefined; 659 | message.prcNum1Str = object.prcNum1Str ?? ""; 660 | message.prcNum2Str = object.prcNum2Str ?? ""; 661 | message.prcNum3Str = object.prcNum3Str ?? ""; 662 | return message; 663 | }, 664 | }; 665 | 666 | function createBaseLngLatPoint(): LngLatPoint { 667 | return { lng: 0, lat: 0 }; 668 | } 669 | 670 | export const LngLatPoint: MessageFns = { 671 | encode(message: LngLatPoint, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { 672 | if (message.lng !== 0) { 673 | writer.uint32(9).double(message.lng); 674 | } 675 | if (message.lat !== 0) { 676 | writer.uint32(17).double(message.lat); 677 | } 678 | return writer; 679 | }, 680 | 681 | decode(input: BinaryReader | Uint8Array, length?: number): LngLatPoint { 682 | const reader = input instanceof BinaryReader ? input : new BinaryReader(input); 683 | let end = length === undefined ? reader.len : reader.pos + length; 684 | const message = createBaseLngLatPoint(); 685 | while (reader.pos < end) { 686 | const tag = reader.uint32(); 687 | switch (tag >>> 3) { 688 | case 1: 689 | if (tag !== 9) { 690 | break; 691 | } 692 | 693 | message.lng = reader.double(); 694 | continue; 695 | case 2: 696 | if (tag !== 17) { 697 | break; 698 | } 699 | 700 | message.lat = reader.double(); 701 | continue; 702 | } 703 | if ((tag & 7) === 4 || tag === 0) { 704 | break; 705 | } 706 | reader.skip(tag & 7); 707 | } 708 | return message; 709 | }, 710 | 711 | fromJSON(object: any): LngLatPoint { 712 | return { 713 | lng: isSet(object.lng) ? globalThis.Number(object.lng) : 0, 714 | lat: isSet(object.lat) ? globalThis.Number(object.lat) : 0, 715 | }; 716 | }, 717 | 718 | toJSON(message: LngLatPoint): unknown { 719 | const obj: any = {}; 720 | if (message.lng !== 0) { 721 | obj.lng = message.lng; 722 | } 723 | if (message.lat !== 0) { 724 | obj.lat = message.lat; 725 | } 726 | return obj; 727 | }, 728 | 729 | create, I>>(base?: I): LngLatPoint { 730 | return LngLatPoint.fromPartial(base ?? ({} as any)); 731 | }, 732 | fromPartial, I>>(object: I): LngLatPoint { 733 | const message = createBaseLngLatPoint(); 734 | message.lng = object.lng ?? 0; 735 | message.lat = object.lat ?? 0; 736 | return message; 737 | }, 738 | }; 739 | 740 | type Builtin = Date | Function | Uint8Array | string | number | boolean | undefined; 741 | 742 | export type DeepPartial = T extends Builtin ? T 743 | : T extends globalThis.Array ? globalThis.Array> 744 | : T extends ReadonlyArray ? ReadonlyArray> 745 | : T extends {} ? { [K in keyof T]?: DeepPartial } 746 | : Partial; 747 | 748 | type KeysOfUnion = T extends T ? keyof T : never; 749 | export type Exact = P extends Builtin ? P 750 | : P & { [K in keyof P]: Exact } & { [K in Exclude>]: never }; 751 | 752 | function isSet(value: any): boolean { 753 | return value !== null && value !== undefined; 754 | } 755 | 756 | export interface MessageFns { 757 | encode(message: T, writer?: BinaryWriter): BinaryWriter; 758 | decode(input: BinaryReader | Uint8Array, length?: number): T; 759 | fromJSON(object: any): T; 760 | toJSON(message: T): unknown; 761 | create, I>>(base?: I): T; 762 | fromPartial, I>>(object: I): T; 763 | } 764 | -------------------------------------------------------------------------------- /src/data.ts: -------------------------------------------------------------------------------- 1 | /** 注意: [経度, 緯度] の順 */ 2 | export type LngLat = [number, number]; 3 | 4 | export type SinglePrefecture = { 5 | /** 全国地方公共団体コード */ 6 | code: number; 7 | /** 都道府県名 */ 8 | pref: string; 9 | 10 | /** 都道府県名 (カナ) */ 11 | pref_k: string; 12 | 13 | /** 都道府県名 (ローマ字) */ 14 | pref_r: string; 15 | 16 | /** 代表点 (県庁の位置) */ 17 | point: LngLat; 18 | 19 | cities: SingleCity[]; 20 | }; 21 | 22 | /** 23 | * SinglePrefecture から都道府県名を取得します。 24 | * @param pref SinglePrefecture 25 | * @returns string 26 | */ 27 | export function prefectureName(pref: SinglePrefecture): string { 28 | return pref.pref; 29 | } 30 | 31 | type Api = { 32 | meta: { 33 | /** データ更新日(UNIX時間; 秒) */ 34 | updated: number; 35 | }; 36 | data: T; 37 | } 38 | 39 | /** 40 | * 都道府県、市区町村一覧API 41 | * 政令都市の場合は区で区切ります 42 | * @file api/ja.json 43 | */ 44 | export type PrefectureApi = Api; 45 | 46 | export type SingleCity = { 47 | /** 全国地方公共団体コード */ 48 | code: number; 49 | /** 郡名 */ 50 | county?: string; 51 | /** 郡名 (カナ) */ 52 | county_k?: string; 53 | /** 郡名 (ローマ字) */ 54 | county_r?: string; 55 | 56 | /** 市区町村名 */ 57 | city: string; 58 | /** 市区町村名 (カナ) */ 59 | city_k: string; 60 | /** 市区町村名 (ローマ字) */ 61 | city_r: string; 62 | 63 | /** 政令市区名 */ 64 | ward?: string; 65 | /** 政令市区名 (カナ) */ 66 | ward_k?: string; 67 | /** 政令市区名 (ローマ字) */ 68 | ward_r?: string; 69 | 70 | /** 代表点 (自治体役場の位置) */ 71 | point: LngLat; 72 | }; 73 | 74 | /** 75 | * SingleCity から市区町村名を取得します。郡名と政令市区名を含めます。 76 | * @param city SingleCity 77 | * @returns string 78 | */ 79 | export function cityName(city: SingleCity): string { 80 | return `${city.county || ''}${city.city}${city.ward || ''}`; 81 | } 82 | 83 | /** 84 | * 市区町村一覧API 85 | * @file api/ja/{都道府県名}.json 86 | */ 87 | export type CityApi = Api; 88 | 89 | export type SingleMachiAza = { 90 | /** ABR上の「町字ID」 */ 91 | machiaza_id: string; 92 | 93 | /** 大字・町名 */ 94 | oaza_cho?: string; 95 | /** 大字・町名 (カナ) */ 96 | oaza_cho_k?: string; 97 | /** 大字・町名 (ローマ字) */ 98 | oaza_cho_r?: string; 99 | 100 | /** 丁目名 */ 101 | chome?: string; 102 | /** 丁目名 (数字) */ 103 | chome_n?: number; 104 | 105 | /** 小字名 */ 106 | koaza?: string; 107 | /** 小字名 (カナ) */ 108 | koaza_k?: string; 109 | /** 小字名 (ローマ字) */ 110 | koaza_r?: string; 111 | 112 | /** 住居表示住所の情報の存在。値が存在しない場合は、住居表示住所の情報は存在しません。 */ 113 | rsdt?: true; 114 | 115 | /** 代表点 */ 116 | point?: LngLat; 117 | 118 | /** CSV APIに付加情報が存在する場合、この町字のバイト範囲を指定します。 */ 119 | csv_ranges?: { 120 | ["住居表示"]?: { start: number; length: number; }; 121 | ["地番"]?: { start: number; length: number; }; 122 | } 123 | }; 124 | 125 | /** 126 | * SingleMachiAza から町字名を取得します。大字・丁目・小字を含めます。 127 | * @param machiAza SingleMachiAza 128 | * @returns string 129 | */ 130 | export function machiAzaName(machiAza: SingleMachiAza): string { 131 | return `${machiAza.oaza_cho || ''}${machiAza.chome || ''}${machiAza.koaza || ''}`; 132 | } 133 | 134 | /** 135 | * 町字一覧API 136 | * @file api/ja/{都道府県名}/{市区町村名}.json 137 | */ 138 | export type MachiAzaApi = Api; 139 | 140 | export type SingleRsdt = { 141 | /** 街区符号 */ 142 | blk_num?: string; 143 | /** 住居番号 */ 144 | rsdt_num: string; 145 | /** 住居番号2 */ 146 | rsdt_num2?: string; 147 | 148 | /** 代表点 */ 149 | point?: LngLat; 150 | }; 151 | 152 | /** 153 | * SingleRsdt の街区符号・住居番号・住居番号2を `-` で区切った文字列を返します。 154 | * @param rsdt SingleRsdt 155 | * @returns string 156 | */ 157 | export function rsdtToString(rsdt: SingleRsdt): string { 158 | return [rsdt.blk_num, rsdt.rsdt_num, rsdt.rsdt_num2].filter(Boolean).join('-'); 159 | } 160 | 161 | /** 162 | * {市区町村名}-住居表示.json は類似なデータフォーマットを使います。 163 | * @file api/ja/{都道府県名}/{市区町村名}-住居表示.json 164 | */ 165 | export type RsdtApi = { 166 | machiAza: SingleMachiAza; 167 | rsdts: SingleRsdt[]; 168 | }[]; 169 | 170 | export type SingleChiban = { 171 | /** 地番1 */ 172 | prc_num1: string; 173 | /** 地番2 */ 174 | prc_num2?: string; 175 | /** 地番3 */ 176 | prc_num3?: string; 177 | 178 | /** 代表点 */ 179 | point?: LngLat; 180 | }; 181 | 182 | /** 183 | * SingleChiban の地番1・地番2・地番3を `-` で区切った文字列を返します。 184 | * @param chiban SingleChiban 185 | * @returns string 186 | */ 187 | export function chibanToString(chiban: SingleChiban): string { 188 | return [chiban.prc_num1, chiban.prc_num2, chiban.prc_num3] 189 | .filter(Boolean) 190 | .join('-'); 191 | } 192 | -------------------------------------------------------------------------------- /src/lib/abr_mlit_merge_tools.ts: -------------------------------------------------------------------------------- 1 | import { SingleMachiAza } from "../data.js"; 2 | import { NlftpMlitDataRow } from "./mlit_nlftp.js"; 3 | 4 | export function filterMlitDataByPrefCity(mlitData: NlftpMlitDataRow[], prefName: string, cityName: string): NlftpMlitDataRow[] { 5 | return mlitData.filter(row => row.pref_name === prefName && row.city_name === cityName); 6 | } 7 | 8 | export function createMergedApiData(abrData: SingleMachiAza[], mlitData: NlftpMlitDataRow[]): SingleMachiAza[] { 9 | const out = abrData; 10 | 11 | for (const row of mlitData) { 12 | // ABRデータに重複があるかのチェック 13 | if (abrData.find(a => ( 14 | (a.oaza_cho === row.oaza_cho && a.chome === row.chome) || // 大字と丁目が一致する場合 15 | (a.koaza === row.oaza_cho) || // 小字が一致する場合 16 | ((a.oaza_cho || '') + (a.koaza || '') === row.oaza_cho) // 大字と小字を結合したものが一致する場合 17 | ))) { 18 | continue; 19 | } 20 | out.push({ 21 | machiaza_id: row.machiaza_id, 22 | oaza_cho: row.oaza_cho, 23 | chome: row.chome, 24 | point: row.point, 25 | }) 26 | } 27 | 28 | return out; 29 | } 30 | -------------------------------------------------------------------------------- /src/lib/ckan.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import * as ckan from './ckan.js'; 5 | 6 | await test.describe('ckan', async () => { 7 | await test('ckanPackageSearch works', async () => { 8 | const res = await ckan.ckanPackageSearch('香川県高松市'); 9 | assert.ok(res.length > 0); 10 | }); 11 | 12 | await test('getCkanPackageById works', async () => { 13 | const res = await ckan.getCkanPackageById('ba-o1-000000_g2-000001'); 14 | assert.strictEqual(res.name, 'ba-o1-000000_g2-000001'); 15 | }); 16 | 17 | await test.describe('downloadAndExtract', async () => { 18 | await test('should download, unzip, and parse the CSV file', async () => { 19 | const res = ckan.downloadAndExtract>('https://catalog.registries.digital.go.jp/rsc/address/mt_town_city372013.csv.zip'); 20 | let count = 0; 21 | for await (const row of res) { 22 | count += 1; 23 | // make sure all rows are parsed, and the header row is not in the results 24 | assert.strictEqual(row['lg_code'], '372013'); 25 | } 26 | assert.ok(count > 0); 27 | }); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /src/lib/ckan.ts: -------------------------------------------------------------------------------- 1 | import path from 'node:path'; 2 | import fs from 'node:fs'; 3 | 4 | import { parse as csvParse } from 'csv-parse'; 5 | 6 | import { fetch } from 'undici'; 7 | import { unzipAndExtractZipBuffer } from './zip_tools.js'; 8 | import { getDownloadStream } from './fetch_tools.js'; 9 | import { lgCodeMatch, loadSettings } from './settings.js'; 10 | 11 | const CKAN_BASE_REGISTRY_URL = `https://catalog.registries.digital.go.jp/rc` 12 | const USER_AGENT = 'curl/8.7.1'; 13 | const CACHE_DIR = path.join(import.meta.dirname, '..', '..', 'cache'); 14 | 15 | export type CKANResponse = { 16 | success: false 17 | } | { 18 | success: true 19 | result: T 20 | } 21 | 22 | type CKANResponseInner = ( 23 | CKANPackageSearchResultList | 24 | CKANPackageSearchResult 25 | ); 26 | 27 | export type CKANPackageSearchResultList = { 28 | count: number, 29 | sort: string, 30 | results: CKANPackageSearchResult[] 31 | } 32 | 33 | export type CKANPackageSearchResult = { 34 | id: string 35 | metadata_created: string, 36 | metadata_modified: string, 37 | name: string, 38 | notes: string, 39 | num_resources: number, 40 | num_tags: number, 41 | title: string, 42 | type: string, 43 | extras: { [key: string]: string }[], 44 | resources: CKANPackageResource[], 45 | } 46 | 47 | export type CKANPackageResource = { 48 | description: string 49 | last_modified: string 50 | id: string 51 | url: string 52 | format: string 53 | } 54 | 55 | export async function ckanPackageSearch(query: string): Promise { 56 | const cacheKey = `package_search_${query}.json`; 57 | const cacheFile = path.join(CACHE_DIR, 'ckan', cacheKey); 58 | 59 | let json: CKANResponse; 60 | if (fs.existsSync(cacheFile)) { 61 | json = await fs.promises.readFile(cacheFile, 'utf-8') 62 | .then((data) => JSON.parse(data) as CKANResponse); 63 | } else { 64 | const url = new URL(`${CKAN_BASE_REGISTRY_URL}/api/3/action/package_search`); 65 | url.searchParams.set('q', query); 66 | const res = await fetch(url.toString(), { 67 | headers: { 68 | 'User-Agent': USER_AGENT, 69 | }, 70 | }); 71 | json = await res.json() as CKANResponse; 72 | 73 | await fs.promises.mkdir(path.dirname(cacheFile), { recursive: true }); 74 | await fs.promises.writeFile(cacheFile, JSON.stringify(json)); 75 | } 76 | 77 | if (!json.success) { 78 | throw new Error('CKAN API returned an error: ' + JSON.stringify(json)); 79 | } 80 | 81 | return json.result.results; 82 | } 83 | 84 | export async function getCkanPackageById(id: string): Promise { 85 | const cacheKey = `package_show_${id}.json`; 86 | const cacheFile = path.join(CACHE_DIR, 'ckan', cacheKey); 87 | 88 | let json: CKANResponse; 89 | if (fs.existsSync(cacheFile)) { 90 | json = await fs.promises.readFile(cacheFile, 'utf-8') 91 | .then((data) => JSON.parse(data) as CKANResponse); 92 | } else { 93 | const url = new URL(`${CKAN_BASE_REGISTRY_URL}/api/3/action/package_show`); 94 | url.searchParams.set('id', id); 95 | const res = await fetch(url.toString(), { 96 | headers: { 97 | 'User-Agent': USER_AGENT, 98 | }, 99 | }); 100 | json = await res.json() as CKANResponse; 101 | 102 | await fs.promises.mkdir(path.dirname(cacheFile), { recursive: true }); 103 | await fs.promises.writeFile(cacheFile, JSON.stringify(json)); 104 | } 105 | if (!json.success) { 106 | throw new Error('CKAN API returned an error: ' + JSON.stringify(json)); 107 | } 108 | return json.result; 109 | } 110 | 111 | export function getUrlForCSVResource(res: CKANPackageSearchResult): string | undefined { 112 | return res.resources.find((resource) => resource.format.startsWith('CSV'))?.url; 113 | } 114 | 115 | export type CSVParserIterator = AsyncIterableIterator; 116 | 117 | export async function *combineCSVParserIterators(...iterators: CSVParserIterator[]): CSVParserIterator { 118 | for (const i of iterators) { 119 | yield* i; 120 | } 121 | } 122 | 123 | export async function *downloadAndExtract(url: string): CSVParserIterator { 124 | const bodyStream = await getDownloadStream(url); 125 | const fileEntries = unzipAndExtractZipBuffer(bodyStream); 126 | for await (const entry of fileEntries) { 127 | const csvParser = csvParse(entry, { quote: false }); 128 | let header: string[] | undefined = undefined; 129 | for await (const r of csvParser) { 130 | const record = r as string[]; 131 | // save header 132 | if (typeof header === 'undefined') { 133 | header = record; 134 | continue; 135 | } 136 | yield record.reduce>((acc, value, index) => { 137 | acc[header![index]] = value; 138 | return acc; 139 | }, {}) as T; 140 | } 141 | } 142 | } 143 | 144 | export async function *getAndStreamCSVDataForId>(id: string): CSVParserIterator { 145 | const res = await getCkanPackageById(id); 146 | const url = getUrlForCSVResource(res); 147 | if (!url) { 148 | throw new Error('No CSV resource found'); 149 | } 150 | const settings = await loadSettings(); 151 | for await (const record of downloadAndExtract(url)) { 152 | const lgCode = (record as {'lg_code': string})['lg_code']; 153 | if (!lgCodeMatch(settings, lgCode)) { continue; } 154 | yield record; 155 | } 156 | } 157 | 158 | export async function getAndParseCSVDataForId>(id: string): Promise { 159 | return Array.fromAsync(getAndStreamCSVDataForId(id)); 160 | } 161 | 162 | export function findResultByTypeAndArea(results: CKANPackageSearchResult[], dataType: string, area: string): CKANPackageSearchResult | undefined { 163 | return results.find((result) => ( 164 | result.extras.findIndex((extra) => (extra.key === "データセット種別" && extra.value === dataType)) > 0 && 165 | result.extras.findIndex((extra) => (extra.key === "対象地域" && extra.value === area)) > 0 166 | )); 167 | } 168 | -------------------------------------------------------------------------------- /src/lib/ckan_data/chiban.ts: -------------------------------------------------------------------------------- 1 | export type ChibanData = { 2 | /// 全国地方公共団体コード 3 | lg_code: string 4 | /// 町字ID 5 | machiaza_id: string 6 | /// 地番ID 7 | prc_id: string 8 | /// 市区町村名 9 | city: string 10 | /// 政令市区名 11 | ward: string 12 | /// 大字・町名 13 | oaza_cho: string 14 | /// 丁目名 15 | chome: string 16 | /// 小字名 17 | koaza: string 18 | /// 地番1 19 | prc_num1: string 20 | /// 地番2 21 | prc_num2: string 22 | /// 地番3 23 | prc_num3: string 24 | /// 住居表示フラグ 25 | rsdt_addr_flg: string 26 | /// 地番レコード区分フラグ 27 | prc_rec_flg: string 28 | /// 地番区域コード 29 | prc_area_code: string 30 | /// 効力発生日 31 | efct_date: string 32 | /// 廃止日 33 | ablt_date: string 34 | /// 原典資料コード 35 | src_code: string 36 | /// 備考 37 | remarks: string 38 | /// 不動産番号 39 | real_prop_num: string 40 | }; 41 | 42 | export type ChibanPosData = { 43 | /// 全国地方公共団体コード 44 | lg_code: string; 45 | /// 町字ID 46 | machiaza_id: string; 47 | /// 地番ID 48 | prc_id: string; 49 | /// 代表点_経度 50 | rep_lon: string; 51 | /// 代表点_緯度 52 | rep_lat: string; 53 | /// 代表点_座標参照系 54 | rep_srid: string; 55 | /// 代表点_地図情報レベル 56 | rep_scale: string; 57 | /// 代表点_原典資料コード 58 | rep_src_code: string; 59 | /// ポリゴン_ファイル名 60 | plygn_fname: string; 61 | /// ポリゴン_キーコード 62 | plygn_kcode: string; 63 | /// ポリゴン_データフォーマット 64 | plygn_fmt: string; 65 | /// ポリゴン_座標参照系 66 | plygn_srid: string; 67 | /// ポリゴン_地図情報レベル 68 | plygn_scale: string; 69 | /// ポリゴン_原典資料コード 70 | plygn_src_code: string; 71 | /// 法務省地図_市区町村コード 72 | moj_map_city_code: string; 73 | /// 法務省地図_大字コード 74 | moj_map_oaza_code: string; 75 | /// 法務省地図_丁目コード 76 | moj_map_chome_code: string; 77 | /// 法務省地図_小字コード 78 | moj_map_koaza_code: string; 79 | /// 法務省地図_予備コード 80 | moj_map_spare_code: string; 81 | /// 法務省地図_筆id 82 | moj_map_brushid: string; 83 | }; 84 | -------------------------------------------------------------------------------- /src/lib/ckan_data/city.ts: -------------------------------------------------------------------------------- 1 | export type CityData = { 2 | /// 全国地方公共団体コード 3 | lg_code: string; 4 | /// 都道府県名 5 | pref: string; 6 | /// 都道府県名_カナ 7 | pref_kana: string; 8 | /// 都道府県名_英字 9 | pref_roma: string; 10 | /// 郡名 11 | county: string; 12 | /// 郡名_カナ 13 | county_kana: string; 14 | /// 郡名_英字 15 | county_roma: string; 16 | /// 市区町村名 17 | city: string; 18 | /// 市区町村名_カナ 19 | city_kana: string; 20 | /// 市区町村名_英字 21 | city_roma: string; 22 | /// 政令市区名 23 | ward: string; 24 | /// 政令市区名_カナ 25 | ward_kana: string; 26 | /// 政令市区名_英字 27 | ward_roma: string; 28 | /// 効力発生日 29 | efct_date: string; 30 | /// 廃止日 31 | ablt_date: string; 32 | /// 備考 33 | remarks: string; 34 | }; 35 | 36 | export type CityPosData = { 37 | /// 全国地方公共団体コード 38 | lg_code: string; 39 | /// 代表点_経度 40 | rep_lon: string; 41 | /// 代表点_緯度 42 | rep_lat: string; 43 | /// 代表点_座標参照系 44 | rep_srid: string; 45 | /// 代表点_地図情報レベル 46 | rep_scale: string; 47 | /// ポリゴン_ファイル名 48 | plygn_fname: string; 49 | /// ポリゴン_キーコード 50 | plygn_kcode: string; 51 | /// ポリゴン_データフォーマット 52 | plygn_fmt: string; 53 | /// ポリゴン_座標参照系 54 | plygn_srid: string; 55 | /// ポリゴン_地図情報レベル 56 | plygn_scale: string; 57 | }; 58 | 59 | export type CityDataWithPos = CityData & CityPosData; 60 | 61 | export function mergeCityData(cityData: CityData[], cityPosData: CityPosData[]): CityDataWithPos[] { 62 | const out: CityDataWithPos[] = []; 63 | for (const city of cityData) { 64 | const pos = cityPosData.find((pos) => pos.lg_code === city.lg_code); 65 | if (pos) { 66 | out.push({ ...city, ...pos }); 67 | } 68 | } 69 | return out; 70 | } 71 | -------------------------------------------------------------------------------- /src/lib/ckan_data/index.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test, { describe } from 'node:test'; 3 | 4 | import * as index from './index.js'; 5 | 6 | await describe('ckan_data/index', async () => { 7 | await describe('joinAsyncIterators', async () => { 8 | await test('it correctly joins two async iterators when they are ordered', async () => { 9 | const one = async function*(){ 10 | await new Promise((resolve) => setTimeout(resolve, 10)); 11 | yield *[ 12 | { id: 100, name: 'Alice' }, 13 | { id: 101, name: 'Bob' } 14 | ]; 15 | }; 16 | const two = async function*(){ 17 | await new Promise((resolve) => setTimeout(resolve, 10)); 18 | yield *[ 19 | { id: 100, age: 500 }, 20 | { id: 101, age: 501 } 21 | ]; 22 | }; 23 | 24 | const res = await Array.fromAsync( 25 | index.mergeDataLeftJoin(one(), two(), ['id']) 26 | ); 27 | 28 | assert.deepStrictEqual(res, [ 29 | { id: 100, name: 'Alice', age: 500 }, 30 | { id: 101, name: 'Bob', age: 501 }, 31 | ]); 32 | }); 33 | 34 | await test('it correctly joins two async iterators when they are out of order', async () => { 35 | const one = async function *() { 36 | await new Promise((resolve) => setTimeout(resolve, 10)); 37 | yield *[{ id: 1, name: 'Alice' }, { id: 2, name: 'Bob' }, { id: 3, name: 'Charlie' }]; 38 | }; 39 | const two = async function *() { 40 | await new Promise((resolve) => setTimeout(resolve, 10)); 41 | yield *[{ id: 2, age: 30 }, { id: 1, age: 25 }, { id: 4, age: 35 }]; 42 | }; 43 | 44 | const res = await Array.fromAsync( 45 | index.mergeDataLeftJoin(one(), two(), ['id']) 46 | ); 47 | 48 | assert.deepStrictEqual(res, [ 49 | { id: 1, name: 'Alice', age: 25 }, 50 | { id: 2, name: 'Bob', age: 30 }, 51 | { id: 3, name: 'Charlie' }, 52 | ]); 53 | }); 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /src/lib/ckan_data/index.ts: -------------------------------------------------------------------------------- 1 | import Database from "better-sqlite3"; 2 | import fs from "node:fs/promises"; 3 | import os from "node:os"; 4 | import path from "node:path"; 5 | import { pipeline } from "node:stream/promises"; 6 | 7 | // eslint-disable-next-line @typescript-eslint/no-explicit-any 8 | function _createKey(data: any, keys: string[]): string { 9 | // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access 10 | return keys.map((key) => `${data[key]}`).join("|"); 11 | } 12 | 13 | export async function *mergeDataLeftJoin(left: AsyncIterableIterator, right: AsyncIterableIterator, keys: string[], memory: boolean = false): AsyncIterableIterator<(T | T & U)> { 14 | let tmpDbPath = ":memory:"; 15 | 16 | if (!memory) { 17 | const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "merge-data-left-join-")); 18 | tmpDbPath = path.join(tmpDir, "db.sqlite3"); 19 | console.log(`Creating temporary database: ${tmpDbPath}`); 20 | } 21 | 22 | const db = new Database(tmpDbPath); 23 | db.pragma("synchronous = OFF"); 24 | db.pragma("journal_mode = MEMORY"); 25 | db.exec(` 26 | CREATE TABLE l ( 27 | key TEXT, 28 | data JSONB 29 | ); 30 | CREATE TABLE r ( 31 | key TEXT, 32 | data JSONB 33 | ); 34 | `); 35 | const stmt1 = db.prepare("INSERT INTO l VALUES (?, ?)"); 36 | const stmt2 = db.prepare("INSERT INTO r VALUES (?, ?)"); 37 | 38 | await Promise.all([ 39 | pipeline(left, async function (source) { 40 | for await (const data of source) { 41 | stmt1.run(_createKey(data, keys), JSON.stringify(data)); 42 | } 43 | }), 44 | pipeline(right, async function (source) { 45 | for await (const data of source) { 46 | stmt2.run(_createKey(data, keys), JSON.stringify(data)); 47 | } 48 | }), 49 | ]); 50 | db.exec(` 51 | CREATE INDEX l_key ON l(key); 52 | CREATE INDEX r_key ON r(key); 53 | `); 54 | 55 | const select = db.prepare(` 56 | SELECT 57 | json_patch(l.data, coalesce(r.data, '{}')) AS d01 58 | FROM 59 | l 60 | LEFT JOIN r ON l.key = r.key 61 | `); 62 | for (const data of select.iterate()) { 63 | yield JSON.parse(data.d01); 64 | } 65 | 66 | db.close(); 67 | if (!memory) { 68 | await fs.rm(path.dirname(tmpDbPath), { recursive: true }); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/lib/ckan_data/machi_aza.ts: -------------------------------------------------------------------------------- 1 | export type MachiAzaData = { 2 | /// 全国地方公共団体コード 3 | lg_code: string; 4 | /// 町字ID 5 | machiaza_id: string; 6 | /// 町字区分コード 7 | machiaza_type: string; 8 | /// 都道府県名 9 | pref: string; 10 | /// 都道府県名_カナ 11 | pref_kana: string; 12 | /// 都道府県名_英字 13 | pref_roma: string; 14 | /// 郡名 15 | county: string; 16 | /// 郡名_カナ 17 | county_kana: string; 18 | /// 郡名_英字 19 | county_roma: string; 20 | /// 市区町村名 21 | city: string; 22 | /// 市区町村名_カナ 23 | city_kana: string; 24 | /// 市区町村名_英字 25 | city_roma: string; 26 | /// 政令市区名 27 | ward: string; 28 | /// 政令市区名_カナ 29 | ward_kana: string; 30 | /// 政令市区名_英字 31 | ward_roma: string; 32 | /// 大字・町名 33 | oaza_cho: string; 34 | /// 大字・町名_カナ 35 | oaza_cho_kana: string; 36 | /// 大字・町名_英字 37 | oaza_cho_roma: string; 38 | /// 丁目名 39 | chome: string; 40 | /// 丁目名_カナ 41 | chome_kana: string; 42 | /// 丁目名_数字 43 | chome_number: string; 44 | /// 小字名 45 | koaza: string; 46 | /// 小字名_カナ 47 | koaza_kana: string; 48 | /// 小字名_英字 49 | koaza_roma: string; 50 | /// 同一町字識別情報 51 | machiaza_dist: string; 52 | /// 住居表示フラグ 53 | rsdt_addr_flg: string; 54 | /// 住居表示方式コード 55 | rsdt_addr_mtd_code: string; 56 | /// 大字・町名_通称フラグ 57 | oaza_cho_aka_flg: string; 58 | /// 小字名_通称コード 59 | koaza_aka_code: string; 60 | /// 大字・町名_電子国土基本図外字 61 | oaza_cho_gsi_uncmn: string; 62 | /// 小字名_電子国土基本図外字 63 | koaza_gsi_uncmn: string; 64 | /// 状態フラグ 65 | status_flg: string; 66 | /// 起番フラグ 67 | wake_num_flg: string; 68 | /// 効力発生日 69 | efct_date: string; 70 | /// 廃止日 71 | ablt_date: string; 72 | /// 原典資料コード 73 | src_code: string; 74 | /// 郵便番号 75 | post_code: string; 76 | /// 備考 77 | remarks: string; 78 | }; 79 | 80 | export type MachiAzaPosData = { 81 | /// 全国地方公共団体コード 82 | lg_code: string; 83 | /// 町字ID 84 | machiaza_id: string; 85 | /// 住居表示フラグ 86 | rsdt_addr_flg: string; 87 | /// 代表点_経度 88 | rep_lon: string; 89 | /// 代表点_緯度 90 | rep_lat: string; 91 | /// 代表点_座標参照系 92 | rep_srid: string; 93 | /// 代表点_地図情報レベル 94 | rep_scale: string; 95 | /// 代表点_原典資料コード 96 | rep_src_code: string; 97 | /// ポリゴン_ファイル名 98 | plygn_fname: string; 99 | /// ポリゴン_キーコード 100 | plygn_kcode: string; 101 | /// ポリゴン_データフォーマット 102 | plygn_fmt: string; 103 | /// ポリゴン_座標参照系 104 | plygn_srid: string; 105 | /// ポリゴン_地図情報レベル 106 | plygn_scale: string; 107 | /// ポリゴン_原典資料コード 108 | plygn_src_code: string; 109 | /// 位置参照情報_大字町丁目コード 110 | pos_oaza_cho_chome_code: string; 111 | /// 位置参照情報_データ整備年度 112 | pos_data_mnt_year: string; 113 | /// 国勢調査_境界_小地域（町丁・字等別）_KEY_CODE 114 | cns_bnd_s_area_kcode: string; 115 | /// 国勢調査_境界_データ整備年度 116 | cns_bnd_year: string; 117 | }; 118 | -------------------------------------------------------------------------------- /src/lib/ckan_data/prefecture.ts: -------------------------------------------------------------------------------- 1 | export type PrefData = { 2 | /// 全国地方公共団体コード 3 | lg_code: string; 4 | /// 都道府県名 5 | pref: string; 6 | /// 都道府県名_カナ 7 | pref_kana: string; 8 | /// 都道府県名_英字 9 | pref_roma: string; 10 | /// 効力発生日 11 | efct_date: string; 12 | /// 廃止日 13 | ablt_date: string; 14 | /// 備考 15 | remarks: string; 16 | }; 17 | 18 | export type PrefPosData = { 19 | /// 全国地方公共団体コード 20 | lg_code: string; 21 | /// 代表点_経度 22 | rep_lon: string; 23 | /// 代表点_緯度 24 | rep_lat: string; 25 | /// 代表点_座標参照系 26 | rep_srid: string; 27 | /// 代表点_地図情報レベル 28 | rep_scale: string; 29 | /// ポリゴン_ファイル名 30 | plygn_fname: string; 31 | /// ポリゴン_キーコード 32 | plygn_kcode: string; 33 | /// ポリゴン_データフォーマット 34 | plygn_fmt: string; 35 | /// ポリゴン_座標参照系 36 | plygn_srid: string; 37 | /// ポリゴン_地図情報レベル 38 | plygn_scale: string; 39 | }; 40 | 41 | export type PrefDataWithPos = PrefData & PrefPosData; 42 | 43 | export function mergePrefectureData(prefData: PrefData[], prefPosData: PrefPosData[]): PrefDataWithPos[] { 44 | const out: PrefDataWithPos[] = []; 45 | for (const pref of prefData) { 46 | const pos = prefPosData.find((pos) => pos.lg_code === pref.lg_code); 47 | if (pos) { 48 | out.push({ ...pref, ...pos }); 49 | } 50 | } 51 | return out; 52 | } 53 | -------------------------------------------------------------------------------- /src/lib/ckan_data/rsdtdsp_rsdt.ts: -------------------------------------------------------------------------------- 1 | import { mergeDataLeftJoin } from "./index.js"; 2 | 3 | export type RsdtdspRsdtData = { 4 | /// 全国地方公共団体コード 5 | lg_code: string; 6 | /// 町字ID 7 | machiaza_id: string; 8 | /// 街区ID 9 | blk_id: string; 10 | /// 住居ID 11 | rsdt_id: string; 12 | /// 住居2ID 13 | rsdt2_id: string; 14 | /// 市区町村名 15 | city: string; 16 | /// 政令市区名 17 | ward: string; 18 | /// 大字・町名 19 | oaza_cho: string; 20 | /// 丁目名 21 | chome: string; 22 | /// 小字名 23 | koaza: string; 24 | /// 街区符号 25 | blk_num: string; 26 | /// 住居番号 27 | rsdt_num: string; 28 | /// 住居番号2 29 | rsdt_num2: string; 30 | /// 基礎番号・住居番号区分 31 | basic_rsdt_div: string; 32 | /// 住居表示フラグ 33 | rsdt_addr_flg: string; 34 | /// 住居表示方式コード 35 | rsdt_addr_mtd_code: string; 36 | /// 状態フラグ 37 | status_flg: string; 38 | /// 効力発生日 39 | efct_date: string; 40 | /// 廃止日 41 | ablt_date: string; 42 | /// 原典資料コード 43 | src_code: string; 44 | /// 備考 45 | remarks: string; 46 | }; 47 | 48 | export type RsdtdspRsdtPosData = { 49 | /// 全国地方公共団体コード 50 | lg_code: string; 51 | /// 町字ID 52 | machiaza_id: string; 53 | /// 街区ID 54 | blk_id: string; 55 | /// 住居ID 56 | rsdt_id: string; 57 | /// 住居2ID 58 | rsdt2_id: string; 59 | /// 住居表示フラグ 60 | rsdt_addr_flg: string; 61 | /// 住居表示方式コード 62 | rsdt_addr_mtd_code: string; 63 | /// 代表点_経度 64 | rep_lon: string; 65 | /// 代表点_緯度 66 | rep_lat: string; 67 | /// 代表点_座標参照系 68 | rep_srid: string; 69 | /// 代表点_地図情報レベル 70 | rep_scale: string; 71 | /// 代表点_原典資料コード 72 | rep_src_code: string; 73 | /// 電子国土基本図（地名情報）「住居表示住所」_住所コード（可読） 74 | rsdt_addr_code_rdbl: string; 75 | /// 電子国土基本図（地名情報）「住居表示住所」_データ整備日 76 | rsdt_addr_data_mnt_date: string; 77 | /// 基礎番号・住居番号区分 78 | basic_rsdt_div: string; 79 | }; 80 | 81 | export type RsdtdspRsdtDataWithPos = RsdtdspRsdtData | RsdtdspRsdtData & RsdtdspRsdtPosData; 82 | 83 | export function mergeRsdtdspRsdtData( 84 | rsdtdspRsdtData: AsyncIterableIterator, 85 | rsdtdspRsdtPosData: AsyncIterableIterator 86 | ): AsyncIterableIterator { 87 | return mergeDataLeftJoin(rsdtdspRsdtData, rsdtdspRsdtPosData, ["lg_code", "machiaza_id", "blk_id", "rsdt_id", "rsdt2_id"]); 88 | } 89 | -------------------------------------------------------------------------------- /src/lib/fetch_tools.ts: -------------------------------------------------------------------------------- 1 | import path from 'node:path'; 2 | import fs from 'node:fs'; 3 | 4 | import { fetch } from 'undici'; 5 | 6 | const USER_AGENT = 'curl/8.7.1'; 7 | const CACHE_DIR = path.join(import.meta.dirname, '..', '..', 'cache'); 8 | 9 | export async function getDownloadStream(url: string): Promise { 10 | const cacheKey = url.replace(/[^a-zA-Z0-9]/g, '_'); 11 | const cacheFile = path.join(CACHE_DIR, 'files', cacheKey); 12 | 13 | let buffer: Buffer; 14 | if (!fs.existsSync(cacheFile)) { 15 | // console.log(`Downloading ${url}`); 16 | const res = await fetch(url, { 17 | headers: { 18 | 'User-Agent': USER_AGENT, 19 | }, 20 | }); 21 | 22 | if (!res.ok) { 23 | throw new Error(`HTTP ${res.status}: ${res.statusText}`); 24 | } 25 | 26 | await fs.promises.mkdir(path.dirname(cacheFile), { recursive: true }); 27 | const body = await res.arrayBuffer(); 28 | buffer = Buffer.from(body); 29 | await fs.promises.writeFile(cacheFile, buffer); 30 | } else { 31 | buffer = await fs.promises.readFile(cacheFile); 32 | } 33 | 34 | return buffer; 35 | } 36 | -------------------------------------------------------------------------------- /src/lib/mlit_nlftp.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import { 5 | downloadAndExtractNlftpMlitFile, 6 | } from './mlit_nlftp.js'; 7 | 8 | await test.describe('downloadAndExtractNlftpMlitFile', async () => { 9 | await test('it works', async () => { 10 | // 沖縄県 11 | const data = await downloadAndExtractNlftpMlitFile('47'); 12 | assert.strictEqual(data.length, 1228); 13 | assert.strictEqual(data[0].machiaza_id, 'MLIT:472010001001'); 14 | assert.strictEqual(data[0].pref_name, '沖縄県'); 15 | assert.strictEqual(data[0].city_name, '那覇市'); 16 | assert.strictEqual(data[0].oaza_cho, '古波蔵'); 17 | assert.strictEqual(data[0].chome, '一丁目'); 18 | assert.strictEqual(data[0].point.length, 2); 19 | }); 20 | }); 21 | -------------------------------------------------------------------------------- /src/lib/mlit_nlftp.ts: -------------------------------------------------------------------------------- 1 | import { parse as csvParse } from 'csv-parse'; 2 | import iconv from 'iconv-lite'; 3 | 4 | import { unzipAndExtractZipBuffer } from './zip_tools.js'; 5 | import { getDownloadStream } from './fetch_tools.js'; 6 | 7 | const SKIP_ROWS = new Set([ 8 | // 国土数値情報では「柿さき町」で登録されているが、ABRには「柿崎町」として登録されている。 9 | // 柿崎町を優先するので、「柿さき町」はスキップする。 10 | '愛知県/安城市/柿さき町', 11 | ]); 12 | 13 | export type NlftpMlitDataRow = { 14 | machiaza_id: string 15 | 16 | pref_name: string 17 | city_name: string 18 | 19 | oaza_cho: string 20 | chome?: string 21 | point: [number, number] 22 | } 23 | 24 | // type NlftpMlitCsvRow = { 25 | // 0 "都道府県コード": string 26 | // 1 "都道府県名": string 27 | // 2 "市区町村コード": string 28 | // 3 "市区町村名": string 29 | // 4 "大字町丁目コード": string 30 | // 5 "大字町丁目名": string 31 | // 6 "緯度": string 32 | // 7 "経度": string 33 | // 8 "原典資料コード": string 34 | // 9 "大字・字・丁目区分コード": string 35 | // } 36 | 37 | function parseRows(rows: string[][]): NlftpMlitDataRow[] { 38 | // remove header row 39 | rows.shift(); 40 | // sort by code (should already be sorted, just in case) 41 | rows.sort((a, b) => { 42 | return a[4].localeCompare(b[4]); 43 | }); 44 | 45 | const result: NlftpMlitDataRow[] = []; 46 | for (const row of rows) { 47 | if (SKIP_ROWS.has(`${row[1]}/${row[3]}/${row[5]}`)) continue; 48 | 49 | let oaza_cho = row[5]; 50 | let chome: string | undefined = undefined; 51 | const chomeMatch = oaza_cho.match(/^(.*?)([一二三四五六七八九十]+丁目)$/); 52 | if (chomeMatch) { 53 | oaza_cho = chomeMatch[1]; 54 | chome = chomeMatch[2]; 55 | } 56 | 57 | result.push({ 58 | machiaza_id: `MLIT:${row[4]}`, 59 | pref_name: row[1], 60 | city_name: row[3], 61 | oaza_cho, 62 | chome, 63 | point: [ 64 | parseFloat(row[7]), // longitude 65 | parseFloat(row[6]), // latitude 66 | ], 67 | }); 68 | } 69 | 70 | return result; 71 | } 72 | 73 | /** 74 | * 国土数値情報からデータをダウンロードしパースします。 75 | * 参照: https://nlftp.mlit.go.jp/isj/ 76 | */ 77 | export async function downloadAndExtractNlftpMlitFile(prefCode: string): Promise { 78 | const version = '17.0b'; // 大字・町丁目レベル位置参照情報 79 | // 22.0a は街区レベル位置参照情報なので、ここには必要ない 80 | const url = `https://nlftp.mlit.go.jp/isj/dls/data/${version}/${prefCode}000-${version}.zip`; 81 | const bodyStream = await getDownloadStream(url); 82 | const entries = unzipAndExtractZipBuffer(bodyStream); 83 | for await (const entry of entries) { 84 | if (entry.path.slice(-4) !== '.csv') continue; 85 | const decoded = iconv.decode(entry, 'Shift_JIS'); 86 | const rows = await Array.fromAsync( 87 | csvParse(decoded) 88 | ); 89 | return parseRows(rows); 90 | } 91 | throw new Error('No CSV file detected in archive file'); 92 | } 93 | -------------------------------------------------------------------------------- /src/lib/proj.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import proj from './proj.js'; 5 | 6 | await test.describe('proj', async () => { 7 | await test('should project coordinates from EPSG:6668 to EPSG:4326', () => { 8 | const coords = [139.6917, 35.6895]; 9 | const projected = proj('EPSG:6668', 'EPSG:4326', coords); 10 | assert.deepStrictEqual(projected, [139.6917, 35.6895]); 11 | }); 12 | }); 13 | -------------------------------------------------------------------------------- /src/lib/proj.ts: -------------------------------------------------------------------------------- 1 | import proj4 from "proj4"; 2 | import { LngLat } from "../data.js"; 3 | 4 | proj4.defs("EPSG:4612","+proj=longlat +ellps=GRS80 +no_defs +type=crs"); 5 | proj4.defs("EPSG:6668","+proj=longlat +ellps=GRS80 +no_defs +type=crs"); 6 | 7 | export default proj4; 8 | 9 | type DataRowWithGeometry = { 10 | rep_lon: string 11 | rep_lat: string 12 | rep_srid: string 13 | }; 14 | 15 | /** 16 | * Reprojects Address Base Registry lon/lat data to EPSG:4326 17 | * @returns 18 | */ 19 | export function projectABRData(dataRow: DataRowWithGeometry): LngLat { 20 | const input: LngLat = [parseFloat(dataRow.rep_lon), parseFloat(dataRow.rep_lat)]; 21 | return proj4(dataRow.rep_srid, "EPSG:4326", input); 22 | } 23 | -------------------------------------------------------------------------------- /src/lib/settings.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test, { describe, before, after } from 'node:test'; 3 | 4 | import path from 'node:path'; 5 | import fs from 'node:fs/promises'; 6 | 7 | import * as settings from './settings.js'; 8 | 9 | const fixtureDir = path.join(import.meta.dirname, '..', '..', 'test', 'fixtures', 'lib', 'settings'); 10 | 11 | before(async () => { 12 | await fs.copyFile(path.join(fixtureDir, 'settings.json'), path.join(process.cwd(), 'settings.json')); 13 | }); 14 | 15 | after(async () => { 16 | await fs.rm(path.join(process.cwd(), 'settings.json')); 17 | delete process.env.SETTINGS_JSON; 18 | delete process.env.SETTINGS_PATH; 19 | }); 20 | 21 | await test('loadSettings', async () => { 22 | const parsedSettings = await settings.loadSettings(); 23 | assert.equal(parsedSettings.lgCodes.length, 1); 24 | assert.ok(parsedSettings.lgCodes[0].test('011002')); 25 | assert.ok(!parsedSettings.lgCodes[0].test('131002')); 26 | }); 27 | 28 | await describe('lgCodeMatch', async () => { 29 | await test('basic settings', () => { 30 | const settingsData = settings.parseSettings({ 31 | lgCodes: ['^01', '^13', '472018'], 32 | }); 33 | assert.ok(settings.lgCodeMatch(settingsData, '011002')); 34 | assert.ok(settings.lgCodeMatch(settingsData, '131002')); 35 | assert.ok(!settings.lgCodeMatch(settingsData, '465054')); 36 | assert.ok(settings.lgCodeMatch(settingsData, '472018')); 37 | }); 38 | 39 | await test('市区町村まで指定されているときは、都道府県全体に対してマッチする', () => { 40 | const settingsData = settings.parseSettings({ 41 | lgCodes: ['131002', '472018'], 42 | }); 43 | assert.ok(settings.lgCodeMatch(settingsData, '131002')); 44 | assert.ok(settings.lgCodeMatch(settingsData, '130001')); 45 | 46 | assert.ok(settings.lgCodeMatch(settingsData, '472018')); 47 | assert.ok(settings.lgCodeMatch(settingsData, '470007')); 48 | 49 | assert.ok(!settings.lgCodeMatch(settingsData, '011002')); 50 | assert.ok(!settings.lgCodeMatch(settingsData, '460001')); 51 | }); 52 | }); 53 | 54 | await test('loadSettings with overwritten JSON', async () => { 55 | process.env.SETTINGS_JSON = JSON.stringify({ lgCodes: ['^99'] }); 56 | const parsedSettings = await settings.loadSettings(); 57 | assert.equal(parsedSettings.lgCodes.length, 1); 58 | assert.ok(parsedSettings.lgCodes[0].test('990000')); 59 | }); 60 | -------------------------------------------------------------------------------- /src/lib/settings.ts: -------------------------------------------------------------------------------- 1 | import { LRUCache } from 'lru-cache'; 2 | import path from 'node:path'; 3 | import fs from 'node:fs/promises'; 4 | 5 | const settingsPath = () => { 6 | if (process.env.SETTINGS_JSON) return "json:" + process.env.SETTINGS_JSON; 7 | if (process.env.SETTINGS_PATH) return process.env.SETTINGS_PATH; 8 | return path.join(process.cwd(), "settings.json"); 9 | } 10 | 11 | /** settings.json */ 12 | type Settings = { 13 | /** 14 | * 出力する自治体のデータを制限するためのフィルター 15 | * 全国地方公共団体コードをマッチする正規表現の文字列を配列で指定してください。 16 | * OR条件で指定されたコードのいずれかに一致するデータのみ出力されます。 17 | * 18 | * 設定されていない場合は、全てのデータが出力されます。 19 | * 20 | * 例: ["011002", "012025"] は、北海道札幌市と北海道函館市のデータのみ出力されます。 21 | * 例: ["^01"] は、北海道の全ての自治体のデータのみ出力されます。 22 | */ 23 | lgCodes?: string[]; 24 | } 25 | 26 | const DEFAULT_SETTINGS: Settings = {}; 27 | 28 | // --- 29 | 30 | async function loadRawSettings(input: string): Promise { 31 | if (input.startsWith("json:")) { 32 | return JSON.parse(input.slice(5)) as Settings; 33 | } 34 | 35 | try { 36 | const settingsData = await fs.readFile(input, "utf-8"); 37 | return JSON.parse(settingsData) as Settings; 38 | } catch (e) { 39 | if ((e as NodeJS.ErrnoException).code === "ENOENT") { 40 | return DEFAULT_SETTINGS; 41 | } 42 | throw e; 43 | } 44 | } 45 | 46 | export function parseSettings(settings: Settings): ParsedSettings { 47 | return { 48 | lgCodes: settings.lgCodes?.map((code) => new RegExp(code)) || [], 49 | }; 50 | } 51 | 52 | const settingsCache = new LRUCache({ 53 | max: 10, 54 | fetchMethod: async (key) => { 55 | const rawSettings = await loadRawSettings(key); 56 | return parseSettings(rawSettings); 57 | }, 58 | }); 59 | 60 | export type ParsedSettings = { 61 | lgCodes: RegExp[]; 62 | } 63 | 64 | export async function loadSettings(): Promise { 65 | const settings = await settingsCache.fetch(settingsPath()); 66 | if (!settings) { 67 | return { lgCodes: [] }; 68 | } 69 | return settings; 70 | } 71 | 72 | export function lgCodeMatch(settings: ParsedSettings, lgCode: string): boolean { 73 | if (settings.lgCodes.length === 0) { 74 | return true; 75 | } 76 | for (const re of settings.lgCodes) { 77 | if (re.test(lgCode)) { 78 | return true; 79 | } 80 | 81 | // re が市区町村まで指定されている場合は、都道府県全体に対してマッチする 82 | const cityCodeMatch = re.source.match(/^\^?(\d{2})\d{3}/); 83 | if (cityCodeMatch) { 84 | const prefCode = cityCodeMatch[1]; 85 | if (lgCode.startsWith(prefCode + '000')) { 86 | return true; 87 | } 88 | } 89 | } 90 | return false; 91 | } 92 | -------------------------------------------------------------------------------- /src/lib/zip_tools.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import path from 'node:path'; 5 | import fs from 'node:fs'; 6 | 7 | import * as zip_tools from './zip_tools.js'; 8 | 9 | const fixtureDir = path.join(import.meta.dirname, '..', '..', 'test', 'fixtures', 'lib', 'zip_tools'); 10 | 11 | await test.describe('unzipAndExtractZipFile', async () => { 12 | await test('it works for a single layer of zip', async () => { 13 | const filePath = path.join(fixtureDir, 'single_level.csv.zip'); 14 | const stream = fs.createReadStream(filePath); 15 | const files = await Array.fromAsync(zip_tools.unzipAndExtractZipFile(stream)); 16 | assert.strictEqual(files.length, 1); 17 | const file0 = Buffer.concat(await Array.fromAsync(files[0])).toString('utf-8'); 18 | assert.strictEqual(file0, `It,works!\n\n`); 19 | }); 20 | 21 | await test('it works for a double layer of zip', async () => { 22 | const filePath = path.join(fixtureDir, 'double_level.csv.zip'); 23 | const stream = fs.createReadStream(filePath); 24 | const files = await Array.fromAsync(zip_tools.unzipAndExtractZipFile(stream)); 25 | assert.strictEqual(files.length, 2); 26 | 27 | const file0 = Buffer.concat(await Array.fromAsync(files[0])).toString('utf-8'); 28 | assert.strictEqual(file0, `It,works!\nDouble,1\n`); 29 | 30 | const file1 = Buffer.concat(await Array.fromAsync(files[1])).toString('utf-8'); 31 | assert.strictEqual(file1, `It,works!\nDouble,2\n`); 32 | }); 33 | }); 34 | 35 | await test.describe('unzipAndExtractZipBuffer', async () => { 36 | await test('it works for a single layer of zip', async () => { 37 | const filePath = path.join(fixtureDir, 'single_level.csv.zip'); 38 | const buffer = await fs.promises.readFile(filePath); 39 | const files = await Array.fromAsync(zip_tools.unzipAndExtractZipBuffer(buffer)); 40 | assert.strictEqual(files.length, 1); 41 | const file0 = files[0].toString('utf-8'); 42 | assert.strictEqual(file0, `It,works!\n\n`); 43 | }); 44 | 45 | await test('it works for a double layer of zip', async () => { 46 | const filePath = path.join(fixtureDir, 'double_level.csv.zip'); 47 | const buffer = await fs.promises.readFile(filePath); 48 | const files = await Array.fromAsync(zip_tools.unzipAndExtractZipBuffer(buffer)); 49 | assert.strictEqual(files.length, 2); 50 | 51 | const file0 = files[0].toString('utf-8'); 52 | assert.strictEqual(file0, `It,works!\nDouble,1\n`); 53 | 54 | const file1 = files[1].toString('utf-8'); 55 | assert.strictEqual(file1, `It,works!\nDouble,2\n`); 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /src/lib/zip_tools.ts: -------------------------------------------------------------------------------- 1 | import { Readable } from 'node:stream'; 2 | import unzipper, { Entry } from 'unzipper'; 3 | 4 | /** 5 | * A function to unzip a file that may or may not contain another zip file. 6 | * The zip file is extracted and each file is returned as a readable stream. 7 | */ 8 | export async function *unzipAndExtractZipFile(zipFile: Readable): AsyncGenerator { 9 | const files = zipFile.pipe(unzipper.Parse({forceStream: true})); 10 | for await (const entry_ of files) { 11 | const entry = entry_ as Entry; 12 | if (entry.type === 'File' && entry.path.endsWith('.zip')) { 13 | yield *unzipAndExtractZipFile(entry); 14 | } else if (entry.type === 'File' && entry.path.endsWith('.csv')) { 15 | yield entry; 16 | } else { 17 | entry.autodrain(); 18 | } 19 | } 20 | } 21 | 22 | export async function *unzipAndExtractZipBuffer(zipFile: Buffer): AsyncGenerator { 23 | const directory = await unzipper.Open.buffer(zipFile); 24 | for (const file of directory.files) { 25 | if (file.type === 'File' && file.path.endsWith('.zip')) { 26 | const content = await file.buffer(); 27 | yield *unzipAndExtractZipBuffer(content); 28 | } else if (file.type === 'File' && file.path.endsWith('.csv')) { 29 | const content = await file.buffer(); 30 | yield Object.assign(content, {path: file.path}); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/processes/01_make_prefecture_city.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import fs from 'node:fs/promises'; 5 | import main from './01_make_prefecture_city.js'; 6 | import { PrefectureApi } from '../data.js'; 7 | 8 | await test.describe('with Hokkaido filter', async () => { 9 | test.before(() => { 10 | process.env.SETTINGS_JSON = JSON.stringify({ lgCodes: ['^01'] }); 11 | }); 12 | 13 | test.after(() => { 14 | delete process.env.SETTINGS_JSON; 15 | }); 16 | 17 | await test('it generates the API', async () => { 18 | await fs.rm('./out/api_hokkaido', { recursive: true, force: true }); 19 | await main(['', '', './out/api_hokkaido']); 20 | assert.ok(true); 21 | 22 | const ja = JSON.parse(await fs.readFile('./out/api_hokkaido/ja.json', 'utf-8')) as PrefectureApi; 23 | const jaData = ja.data; 24 | assert.equal(jaData.length, 1); 25 | const hokkaido = jaData[0]; 26 | assert.equal(hokkaido.pref, '北海道'); 27 | const cities = hokkaido.cities; 28 | assert.equal(cities.length, 194); 29 | assert.equal(cities[0].city, '札幌市'); 30 | }); 31 | }); 32 | 33 | -------------------------------------------------------------------------------- /src/processes/01_make_prefecture_city.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs'; 2 | import path from 'node:path'; 3 | 4 | import { getAndParseCSVDataForId } from '../lib/ckan.js'; 5 | import { CityApi, PrefectureApi, SingleCity, SinglePrefecture } from '../data.js'; 6 | import { projectABRData } from '../lib/proj.js'; 7 | import { CityData, CityPosData, mergeCityData } from '../lib/ckan_data/city.js'; 8 | import { mergePrefectureData, PrefData, PrefPosData } from '../lib/ckan_data/prefecture.js'; 9 | 10 | function outputCityData(outDir: string, prefName: string, apiData: CityApi, prefectureApi: PrefectureApi) { 11 | // 政令都市の「区名」が無い場合は出力から除外する 12 | const filteredApiData = apiData.data.filter((city) => { 13 | return ( 14 | // 区名がある場合はそのまま出力 15 | city.ward !== undefined || 16 | // 区名が無い場合は、同じ市区町村名で区名があるデータが無いか確認（ある＝政令都市のため、出力しない。ない＝政令都市ではない） 17 | apiData.data.filter((c2) => c2.city === city.city).every((c2) => c2.ward === undefined) 18 | ); 19 | }); 20 | apiData.data = filteredApiData; 21 | 22 | prefectureApi.data.find((pref) => pref.pref === prefName)!.cities = filteredApiData; 23 | 24 | const outFile = path.join(outDir, 'ja', `${prefName}.json`); 25 | fs.mkdirSync(path.dirname(outFile), { recursive: true }); 26 | fs.writeFileSync(outFile, JSON.stringify(apiData)); 27 | console.log(`${prefName.padEnd(4, '　')}: ${filteredApiData.length.toString(10).padEnd(3, ' ')} 件の市区町村を出力した`); 28 | } 29 | 30 | async function main(argv: string[]) { 31 | const updated = Math.floor(Date.now() / 1000); 32 | const outDir = argv[2] || path.join(import.meta.dirname, '..', '..', 'out', 'api'); 33 | fs.mkdirSync(outDir, { recursive: true }); 34 | 35 | const [ 36 | prefMain, 37 | prefPos, 38 | 39 | main, 40 | pos, 41 | ] = await Promise.all([ 42 | getAndParseCSVDataForId('ba-o1-000000_g2-000001'), // 都道府県 43 | getAndParseCSVDataForId('ba-o1-000000_g2-000012'), // 位置参照拡張 44 | 45 | getAndParseCSVDataForId('ba-o1-000000_g2-000002'), // 市区町村 46 | getAndParseCSVDataForId('ba-o1-000000_g2-000013'), // 位置参照拡張 47 | ]); 48 | const rawData = mergeCityData(main, pos); 49 | 50 | const prefApiData: SinglePrefecture[] = []; 51 | const rawPrefData = mergePrefectureData(prefMain, prefPos); 52 | 53 | for (const raw of rawPrefData) { 54 | prefApiData.push({ 55 | code: parseInt(raw.lg_code), 56 | pref: raw.pref, 57 | pref_k: raw.pref_kana, 58 | pref_r: raw.pref_roma, 59 | point: projectABRData(raw), 60 | cities: [], 61 | }); 62 | } 63 | 64 | const prefApi: PrefectureApi = { 65 | meta: { 66 | updated, 67 | }, 68 | data: prefApiData, 69 | }; 70 | 71 | let lastPref: string | undefined = undefined; 72 | let allCount = 0; 73 | const processedLgCodes: Set = new Set(); 74 | let apiData: SingleCity[] = []; 75 | for (const raw of rawData) { 76 | allCount++; 77 | if (lastPref !== raw.pref && lastPref !== undefined) { 78 | const api: CityApi = { 79 | meta: { 80 | updated, 81 | }, 82 | data: apiData, 83 | }; 84 | outputCityData(outDir, lastPref, api, prefApi); 85 | apiData = []; 86 | } 87 | if (lastPref !== raw.pref) { 88 | lastPref = raw.pref; 89 | } 90 | apiData.push({ 91 | code: parseInt(raw.lg_code), 92 | county: raw.county === '' ? undefined : raw.county, 93 | county_k: raw.county_kana === '' ? undefined : raw.county_kana, 94 | county_r: raw.county_roma === '' ? undefined : raw.county_roma, 95 | city: raw.city, 96 | city_k: raw.city_kana, 97 | city_r: raw.city_roma, 98 | ward: raw.ward === '' ? undefined : raw.ward, 99 | ward_k: raw.ward_kana === '' ? undefined : raw.ward_kana, 100 | ward_r: raw.ward_roma === '' ? undefined : raw.ward_roma, 101 | point: projectABRData(raw), 102 | }); 103 | processedLgCodes.add(raw.lg_code); 104 | } 105 | if (lastPref) { 106 | const api: CityApi = { 107 | meta: { 108 | updated, 109 | }, 110 | data: apiData, 111 | }; 112 | outputCityData(outDir, lastPref, api, prefApi); 113 | } 114 | 115 | const outFile = path.join(outDir, 'ja.json'); 116 | fs.writeFileSync(outFile, JSON.stringify(prefApi)); 117 | 118 | console.log(`全国: ${allCount} ${processedLgCodes.size} 件の市区町村を出力した`); 119 | } 120 | 121 | export default main; 122 | -------------------------------------------------------------------------------- /src/processes/02_machi_aza.ts: -------------------------------------------------------------------------------- 1 | import { SingleMachiAza } from "../data.js"; 2 | import { MachiAzaData, MachiAzaPosData } from "../lib/ckan_data/machi_aza.js"; 3 | import { projectABRData } from "../lib/proj.js"; 4 | 5 | export function rawToMachiAza(raw: MachiAzaData | (MachiAzaData & MachiAzaPosData)): SingleMachiAza { 6 | return { 7 | machiaza_id: raw.machiaza_id, 8 | 9 | oaza_cho: raw.oaza_cho === '' ? undefined : raw.oaza_cho, 10 | oaza_cho_k: raw.oaza_cho_kana === '' ? undefined : raw.oaza_cho_kana, 11 | oaza_cho_r: raw.oaza_cho_roma === '' ? undefined : raw.oaza_cho_roma, 12 | 13 | chome: raw.chome === '' ? undefined : raw.chome, 14 | chome_n: raw.chome_number === '' ? undefined : parseInt(raw.chome_number, 10), 15 | 16 | koaza: raw.koaza === '' ? undefined : raw.koaza, 17 | koaza_k: raw.koaza_kana === '' ? undefined : raw.koaza_kana, 18 | koaza_r: raw.koaza_roma === '' ? undefined : raw.koaza_roma, 19 | 20 | rsdt: raw.rsdt_addr_flg === '1' ? true : undefined, 21 | point: 'rep_srid' in raw ? projectABRData(raw) : undefined, 22 | }; 23 | } 24 | -------------------------------------------------------------------------------- /src/processes/02_make_machi_aza.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import fs from 'node:fs/promises'; 5 | import main from './02_make_machi_aza.js'; 6 | import { MachiAzaApi } from '../data.js'; 7 | 8 | await test.describe('with filter for 452092 (宮崎県えびの市)', async () => { 9 | test.before(() => { 10 | process.env.SETTINGS_JSON = JSON.stringify({ lgCodes: ['452092'] }); 11 | }); 12 | 13 | test.after(() => { 14 | delete process.env.SETTINGS_JSON; 15 | }); 16 | 17 | await test('it generates the API', async () => { 18 | await fs.rm('./out/api_miyazaki_ebino', { recursive: true, force: true }); 19 | await main(['', '', './out/api_miyazaki_ebino']); 20 | assert.ok(true); 21 | 22 | const e = JSON.parse(await fs.readFile('./out/api_miyazaki_ebino/ja/宮崎県/えびの市.json', 'utf-8')) as MachiAzaApi; 23 | const eData = e.data; 24 | assert(eData.length > 100); 25 | assert(eData.find((city) => city.machiaza_id === '0000110')?.koaza === '下村'); 26 | }); 27 | }); 28 | 29 | -------------------------------------------------------------------------------- /src/processes/02_make_machi_aza.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs'; 2 | import path from 'node:path'; 3 | 4 | import { getAndStreamCSVDataForId } from '../lib/ckan.js'; 5 | import { MachiAzaApi, SingleMachiAza } from '../data.js'; 6 | import { MachiAzaData, MachiAzaPosData } from '../lib/ckan_data/machi_aza.js'; 7 | import { mergeDataLeftJoin } from '../lib/ckan_data/index.js'; 8 | import { rawToMachiAza } from './02_machi_aza.js'; 9 | import { downloadAndExtractNlftpMlitFile, NlftpMlitDataRow } from '../lib/mlit_nlftp.js'; 10 | import { createMergedApiData, filterMlitDataByPrefCity } from '../lib/abr_mlit_merge_tools.js'; 11 | 12 | async function outputMachiAzaData( 13 | outDir: string, 14 | prefName: string, 15 | cityName: string, 16 | api: MachiAzaApi, 17 | ): Promise { 18 | const outFile = path.join(outDir, 'ja', prefName, `${cityName}.json`); 19 | await fs.promises.mkdir(path.dirname(outFile), { recursive: true }); 20 | await fs.promises.writeFile(outFile, JSON.stringify(api)); 21 | console.log(`${prefName.padEnd(4, '　')} ${cityName.padEnd(10, '　')}: ${api.data.length.toString(10).padEnd(4, ' ')} 件の町字を出力した`); 22 | return api.data.length; 23 | } 24 | 25 | async function main(argv: string[]) { 26 | const updated = Math.floor(Date.now() / 1000); 27 | const outDir = argv[2] || path.join(import.meta.dirname, '..', '..', 'out', 'api'); 28 | fs.mkdirSync(outDir, { recursive: true }); 29 | 30 | 31 | const mainStream = getAndStreamCSVDataForId('ba-o1-000000_g2-000003'); 32 | const posStream = getAndStreamCSVDataForId('ba000004'); 33 | const rawData = mergeDataLeftJoin(mainStream, posStream, ['lg_code', 'machiaza_id']); 34 | // const rawData = mergeMachiAzaData(mainStream, posStream); 35 | 36 | let lastLGCode: string | undefined = undefined; 37 | let lastPrefName: string | undefined = undefined; 38 | let lastCityName: string | undefined = undefined; 39 | let lastMlitData: NlftpMlitDataRow[] | undefined = undefined; 40 | let allCount = 0; 41 | let apiData: SingleMachiAza[] = []; 42 | for await (const raw of rawData) { 43 | if (lastLGCode !== raw.lg_code && lastLGCode !== undefined) { 44 | const filteredMlit = filterMlitDataByPrefCity(lastMlitData!, lastPrefName!, lastCityName!); 45 | apiData = createMergedApiData(apiData, filteredMlit); 46 | const api: MachiAzaApi = { meta: { updated }, data: apiData }; 47 | allCount += await outputMachiAzaData(outDir, lastPrefName!, lastCityName!, api); 48 | apiData = []; 49 | } 50 | if (lastPrefName !== raw.pref) { 51 | // 都道府県が変わったので、都道府県レベルの新しいデータを取得する 52 | lastMlitData = await downloadAndExtractNlftpMlitFile(raw.lg_code.slice(0, 2)); 53 | } 54 | if (lastLGCode !== raw.lg_code) { 55 | lastLGCode = raw.lg_code; 56 | lastPrefName = raw.pref; 57 | lastCityName = `${raw.county}${raw.city}${raw.ward}`; 58 | } 59 | 60 | apiData.push(rawToMachiAza(raw)); 61 | } 62 | if (lastLGCode) { 63 | const filteredMlit = filterMlitDataByPrefCity(lastMlitData!, lastPrefName!, lastCityName!); 64 | apiData = createMergedApiData(apiData, filteredMlit); 65 | const api: MachiAzaApi = { meta: { updated }, data: apiData }; 66 | allCount += await outputMachiAzaData(outDir, lastPrefName!, lastCityName!, api); 67 | } 68 | 69 | console.log(`全国: ${allCount} 件の町字を出力した`); 70 | } 71 | 72 | export default main; 73 | -------------------------------------------------------------------------------- /src/processes/03_make_rsdt.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import fs from 'node:fs/promises'; 5 | import main from './03_make_rsdt.js'; 6 | import { getRangesFromCSV } from './10_refresh_csv_ranges.js'; 7 | 8 | await test.describe('with filter for 131059 (東京都文京区)', async () => { 9 | test.before(() => { 10 | process.env.SETTINGS_JSON = JSON.stringify({ lgCodes: ['131059'] }); 11 | }); 12 | 13 | test.after(() => { 14 | delete process.env.SETTINGS_JSON; 15 | }); 16 | 17 | await test('it generates the API', async () => { 18 | await fs.rm('./out/api_tokyo_bunkyo', { recursive: true, force: true }); 19 | await main(['', '', './out/api_tokyo_bunkyo']); 20 | assert.ok(true); 21 | 22 | const headers = await getRangesFromCSV('./out/api_tokyo_bunkyo/ja/東京都/文京区-住居表示.txt'); 23 | assert(typeof headers !== 'undefined'); 24 | assert.equal(headers[0].name, '白山一丁目'); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/processes/03_make_rsdt.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs'; 2 | import path from 'node:path'; 3 | import { ckanPackageSearch, combineCSVParserIterators, CSVParserIterator, findResultByTypeAndArea, getAndParseCSVDataForId, getAndStreamCSVDataForId } from '../lib/ckan.js'; 4 | import { mergeRsdtdspRsdtData, RsdtdspRsdtData, RsdtdspRsdtPosData } from '../lib/ckan_data/rsdtdsp_rsdt.js'; 5 | import { machiAzaName, RsdtApi, SingleRsdt } from '../data.js'; 6 | import { projectABRData } from '../lib/proj.js'; 7 | import { MachiAzaData } from '../lib/ckan_data/machi_aza.js'; 8 | import { rawToMachiAza } from './02_machi_aza.js'; 9 | import { loadSettings } from '../lib/settings.js'; 10 | 11 | const HEADER_CHUNK_SIZE = 50_000; 12 | // const HEADER_PBF_CHUNK_SIZE = 8_192; 13 | 14 | function getOutPath(ma: MachiAzaData) { 15 | return path.join( 16 | ma.pref, 17 | `${ma.county}${ma.city}${ma.ward}`, 18 | ); 19 | } 20 | 21 | type HeaderRow = { 22 | name: string; 23 | offset: number; 24 | length: number; 25 | } 26 | 27 | function serializeApiDataTxt(apiData: RsdtApi): { headerIterations: number, headerData: HeaderRow[], data: Buffer } { 28 | const outSections: Buffer[] = []; 29 | for ( const { machiAza, rsdts } of apiData ) { 30 | let outSection = `住居表示,${machiAzaName(machiAza)}\n` + 31 | `blk_num,rsdt_num,rsdt_num2,lng,lat\n`; 32 | for (const rsdt of rsdts) { 33 | outSection += `${rsdt.blk_num || ''},${rsdt.rsdt_num},${rsdt.rsdt_num2 || ''},${rsdt.point?.[0] || ''},${rsdt.point?.[1] || ''}\n`; 34 | } 35 | outSections.push(Buffer.from(outSection, 'utf8')); 36 | } 37 | 38 | const createHeader = (iterations = 1) => { 39 | let header = ''; 40 | const headerMaxSize = HEADER_CHUNK_SIZE * iterations; 41 | let lastBytePos = headerMaxSize; 42 | const headerData: HeaderRow[] = []; 43 | for (const [index, section] of outSections.entries()) { 44 | const ma = apiData[index].machiAza; 45 | 46 | header += `${machiAzaName(ma)},${lastBytePos},${section.length}\n`; 47 | headerData.push({ 48 | name: machiAzaName(ma), 49 | offset: lastBytePos, 50 | length: section.length, 51 | }); 52 | lastBytePos += section.length; 53 | } 54 | const headerBuf = Buffer.from(header + '=END=\n', 'utf8'); 55 | if (headerBuf.length > headerMaxSize) { 56 | return createHeader(iterations + 1); 57 | } else { 58 | const padding = Buffer.alloc(headerMaxSize - headerBuf.length); 59 | padding.fill(0x20); 60 | return { 61 | iterations, 62 | data: headerData, 63 | buffer: Buffer.concat([headerBuf, padding]) 64 | }; 65 | } 66 | }; 67 | 68 | const header = createHeader(); 69 | return { 70 | headerIterations: header.iterations, 71 | headerData: header.data, 72 | data: Buffer.concat([header.buffer, ...outSections]), 73 | }; 74 | } 75 | 76 | // function _stringIfNotInteger(value: string | undefined) { 77 | // if (!value) { return undefined; } 78 | // return /^\d+$/.test(value) ? undefined : value; 79 | // } 80 | 81 | // function serializeApiDataPbf(apiData: RsdtApi): Buffer { 82 | // let outSections: Buffer[] = []; 83 | // for ( const { machiAza, rsdts } of apiData ) { 84 | // const section: AddrData.Section = { 85 | // kind: AddrData.Kind.RSDT, 86 | // name: machiAzaName(machiAza), 87 | // rsdtRows: [], 88 | // chibanRows: [], 89 | // } 90 | // for (const rsdt of rsdts) { 91 | // section.rsdtRows.push({ 92 | // blkNum: rsdt.blk_num ? parseInt(rsdt.blk_num, 10) : undefined, 93 | // rsdtNum: parseInt(rsdt.rsdt_num, 10), 94 | // rsdtNum2: rsdt.rsdt_num2 ? parseInt(rsdt.rsdt_num2, 10) : undefined, 95 | // point: rsdt.point ? { lng: rsdt.point[0], lat: rsdt.point[1] } : undefined, 96 | // blkNumStr: _stringIfNotInteger(rsdt.blk_num), 97 | // rsdtNumStr: _stringIfNotInteger(rsdt.rsdt_num), 98 | // rsdtNum2Str: _stringIfNotInteger(rsdt.rsdt_num2), 99 | // }); 100 | // } 101 | // const sectionBuf = Buffer.from(AddrData.Section.encode(section).finish()); 102 | // outSections.push(sectionBuf); 103 | // } 104 | 105 | // const createHeader = (iterations = 1) => { 106 | // const header: AddrData.Header = { 107 | // kind: AddrData.Kind.RSDT, 108 | // rows: [], 109 | // }; 110 | // const headerMaxSize = HEADER_PBF_CHUNK_SIZE * iterations; 111 | // let lastBytePos = headerMaxSize; 112 | // for (const [index, section] of outSections.entries()) { 113 | // const ma = apiData[index].machiAza; 114 | 115 | // header.rows.push({ 116 | // name: machiAzaName(ma), 117 | // offset: lastBytePos, 118 | // length: section.length, 119 | // }); 120 | // lastBytePos += section.length; 121 | // } 122 | // const headerBuf = Buffer.from(AddrData.Header.encode(header).finish()); 123 | // if (headerBuf.length > headerMaxSize) { 124 | // return createHeader(iterations + 1); 125 | // } else { 126 | // const padding = Buffer.alloc(headerMaxSize - headerBuf.length); 127 | // padding.fill(0x00); 128 | // return Buffer.concat([headerBuf, padding]); 129 | // } 130 | // }; 131 | 132 | // const header = createHeader(); 133 | // return Buffer.concat([header, ...outSections]); 134 | // } 135 | 136 | async function outputRsdtData(outDir: string, outFilename: string, apiData: RsdtApi) { 137 | // const machiAzaJSON = path.join(outDir, 'ja', outFilename + '.json'); 138 | // fs.mkdirSync(path.dirname(machiAzaJSON), { recursive: true }); 139 | // fs.writeFileSync(outFileJSON, JSON.stringify(apiData)); 140 | 141 | const outFileTXT = path.join(outDir, 'ja', outFilename + '-住居表示.txt'); 142 | const txt = serializeApiDataTxt(apiData); 143 | await fs.promises.mkdir(path.dirname(outFileTXT), { recursive: true }); 144 | await fs.promises.writeFile(outFileTXT, txt.data); 145 | 146 | // const outFilePbf = path.join(outDir, 'ja', outFilename + '.pbf'); 147 | // fs.writeFileSync(outFilePbf, serializeApiDataPbf(apiData)); 148 | 149 | console.log(`${outFilename}-住居表示: ${apiData.length.toString(10).padEnd(4, ' ')} 件の町字を出力した`); 150 | } 151 | 152 | async function main(argv: string[]) { 153 | const outDir = argv[2] || path.join(import.meta.dirname, '..', '..', 'out', 'api'); 154 | fs.mkdirSync(outDir, { recursive: true }); 155 | 156 | const machiAzaData = await getAndParseCSVDataForId('ba-o1-000000_g2-000003'); // 市区町村 & 町字 157 | const machiAzaDataByCode = new Map(machiAzaData.map((city) => [ 158 | `${city.lg_code}|${city.machiaza_id}`, 159 | city 160 | ])); 161 | 162 | // 鹿児島県 163 | // const mainStream = getAndStreamCSVDataForId('ba-o1-460001_g2-000005'); 164 | // const posStream = getAndStreamCSVDataForId('ba-o1-460001_g2-000008'); 165 | 166 | const hasFilter = (await loadSettings()).lgCodes.length > 0; 167 | 168 | let mainStream: CSVParserIterator; 169 | let posStream: CSVParserIterator; 170 | if (!hasFilter) { 171 | mainStream = getAndStreamCSVDataForId('ba000003'); 172 | posStream = getAndStreamCSVDataForId('ba000006'); 173 | } else { 174 | // machiAzaData が既にフィルターされているので、そこからユニークな都道府県のみ抽出し、そのストリームのみ読み込むようにする 175 | const prefs = new Set(machiAzaData.map((ma) => ma.pref)); 176 | 177 | const mainStreams: CSVParserIterator[] = []; 178 | const posStreams: CSVParserIterator[] = []; 179 | for (const pref of prefs) { 180 | const mainSearchQuery = `${pref} 住居表示-住居マスター データセット`; 181 | const mainResults = await ckanPackageSearch(mainSearchQuery); 182 | const main = findResultByTypeAndArea(mainResults, '住居表示-住居マスター（都道府県）', pref); 183 | if (!main) { 184 | throw new Error(`「${pref}」の住居表示-住居マスター データセットが見つかりませんでした`); 185 | } 186 | mainStreams.push(getAndStreamCSVDataForId(main.id)); 187 | 188 | const posSearchQuery = `${pref} 住居表示-住居マスター位置参照拡張 データセット`; 189 | const posResults = await ckanPackageSearch(posSearchQuery); 190 | const pos = findResultByTypeAndArea(posResults, '住居表示-住居マスター位置参照拡張（都道府県）', pref); 191 | if (!pos) { 192 | throw new Error(`「${pref}」の住居表示-住居マスター位置参照拡張 データセットが見つかりませんでした`); 193 | } 194 | posStreams.push(getAndStreamCSVDataForId(pos.id)); 195 | } 196 | mainStream = combineCSVParserIterators(...mainStreams); 197 | posStream = combineCSVParserIterators(...posStreams); 198 | } 199 | const rawData = mergeRsdtdspRsdtData(mainStream, posStream); 200 | 201 | let lastOutPath: string | undefined = undefined; 202 | 203 | let apiData: RsdtApi = []; 204 | let currentRsdtList: SingleRsdt[] = []; 205 | let currentMachiAza: MachiAzaData | undefined = undefined; 206 | for await (const raw of rawData) { 207 | const ma = machiAzaDataByCode.get(`${raw.lg_code}|${raw.machiaza_id}`); 208 | if (!ma) { 209 | continue; 210 | } 211 | const thisOutPath = getOutPath(ma); 212 | if (currentMachiAza && (currentMachiAza.machiaza_id !== ma.machiaza_id || currentMachiAza.lg_code !== ma.lg_code)) { 213 | if (currentRsdtList.length > 0) { 214 | apiData.push({ 215 | machiAza: rawToMachiAza(currentMachiAza), 216 | rsdts: currentRsdtList, 217 | }); 218 | } 219 | currentMachiAza = ma; 220 | currentRsdtList = []; 221 | } 222 | if (lastOutPath !== thisOutPath && lastOutPath !== undefined) { 223 | await outputRsdtData(outDir, lastOutPath, apiData); 224 | apiData = []; 225 | } 226 | if (lastOutPath !== thisOutPath) { 227 | lastOutPath = thisOutPath; 228 | } 229 | if (!currentMachiAza) { 230 | currentMachiAza = ma; 231 | } 232 | 233 | currentRsdtList.push({ 234 | blk_num: raw.blk_num === '' ? undefined : raw.blk_num, 235 | rsdt_num: raw.rsdt_num, 236 | rsdt_num2: raw.rsdt_num2 === '' ? undefined : raw.rsdt_num2, 237 | point: 'rep_srid' in raw ? projectABRData(raw) : undefined, 238 | }); 239 | } 240 | if (currentMachiAza && currentRsdtList.length > 0) { 241 | apiData.push({ 242 | machiAza: rawToMachiAza(currentMachiAza), 243 | rsdts: currentRsdtList, 244 | }); 245 | } 246 | if (lastOutPath) { 247 | await outputRsdtData(outDir, lastOutPath, apiData); 248 | } 249 | } 250 | 251 | export default main; 252 | -------------------------------------------------------------------------------- /src/processes/04_make_chiban.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import fs from 'node:fs/promises'; 5 | import main from './04_make_chiban.js'; 6 | import { getRangesFromCSV } from './10_refresh_csv_ranges.js'; 7 | 8 | await test.describe('with filter for 465054 (鹿児島県熊毛郡屋久島町)', async () => { 9 | test.before(() => { 10 | process.env.SETTINGS_JSON = JSON.stringify({ lgCodes: ['465054'] }); 11 | }); 12 | 13 | test.after(() => { 14 | delete process.env.SETTINGS_JSON; 15 | }); 16 | 17 | await test('it generates the API', async () => { 18 | await fs.rm('./out/api_kagoshima_yakushima', { recursive: true, force: true }); 19 | await main(['', '', './out/api_kagoshima_yakushima']); 20 | assert.ok(true); 21 | 22 | const headers = await getRangesFromCSV('./out/api_kagoshima_yakushima/ja/鹿児島県/熊毛郡屋久島町-地番.txt'); 23 | assert(typeof headers !== 'undefined'); 24 | assert.equal(headers[0].name, '安房'); 25 | }); 26 | }); 27 | -------------------------------------------------------------------------------- /src/processes/04_make_chiban.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | import fs from 'node:fs'; 4 | import path from 'node:path'; 5 | 6 | import cliProgress from 'cli-progress'; 7 | 8 | import { ckanPackageSearch, findResultByTypeAndArea, getAndParseCSVDataForId, getAndStreamCSVDataForId } from '../lib/ckan.js'; 9 | import { machiAzaName, SingleChiban, SingleMachiAza } from '../data.js'; 10 | import { projectABRData } from '../lib/proj.js'; 11 | import { MachiAzaData } from '../lib/ckan_data/machi_aza.js'; 12 | import { ChibanData, ChibanPosData } from '../lib/ckan_data/chiban.js'; 13 | import { mergeDataLeftJoin } from '../lib/ckan_data/index.js'; 14 | 15 | const HEADER_CHUNK_SIZE = 50_000; 16 | 17 | type ChibanApi = { 18 | machiAza: SingleMachiAza; 19 | chibans: SingleChiban[]; 20 | }[]; 21 | 22 | type HeaderRow = { 23 | name: string; 24 | offset: number; 25 | length: number; 26 | } 27 | 28 | function serializeApiDataTxt(apiData: ChibanApi): { headerIterations: number, headerData: HeaderRow[], data: Buffer } { 29 | const outSections: Buffer[] = []; 30 | for ( const { machiAza, chibans } of apiData ) { 31 | let outSection = `地番,${machiAzaName(machiAza)}\n` + 32 | `prc_num1,prc_num2,prc_num3,lng,lat\n`; 33 | for (const chiban of chibans) { 34 | outSection += `${chiban.prc_num1},${chiban.prc_num2 || ''},${chiban.prc_num3 || ''},${chiban.point?.[0] || ''},${chiban.point?.[1] || ''}\n`; 35 | } 36 | outSections.push(Buffer.from(outSection, 'utf8')); 37 | } 38 | 39 | const createHeader = (iterations = 1) => { 40 | let header = ''; 41 | const headerMaxSize = HEADER_CHUNK_SIZE * iterations; 42 | let lastBytePos = headerMaxSize; 43 | const headerData: HeaderRow[] = []; 44 | for (const [index, section] of outSections.entries()) { 45 | const ma = apiData[index].machiAza; 46 | 47 | header += `${machiAzaName(ma)},${lastBytePos},${section.length}\n`; 48 | headerData.push({ 49 | name: machiAzaName(ma), 50 | offset: lastBytePos, 51 | length: section.length, 52 | }); 53 | 54 | lastBytePos += section.length; 55 | } 56 | const headerBuf = Buffer.from(header + '=END=\n', 'utf8'); 57 | if (headerBuf.length > headerMaxSize) { 58 | return createHeader(iterations + 1); 59 | } else { 60 | const padding = Buffer.alloc(headerMaxSize - headerBuf.length); 61 | padding.fill(0x20); 62 | return { 63 | iterations, 64 | data: headerData, 65 | buffer: Buffer.concat([headerBuf, padding]) 66 | }; 67 | } 68 | }; 69 | 70 | const header = createHeader(); 71 | return { 72 | headerIterations: header.iterations, 73 | headerData: header.data, 74 | data: Buffer.concat([header.buffer, ...outSections]), 75 | }; 76 | } 77 | 78 | async function outputChibanData(outDir: string, outFilename: string, apiData: ChibanApi) { 79 | if (apiData.length === 0) { 80 | return; 81 | } 82 | // const machiAzaJSON = path.join(outDir, 'ja', outFilename + '.json'); 83 | // await fs.promises.writeFile(outFile, JSON.stringify(apiData, null, 2)); 84 | 85 | const outFileTXT = path.join(outDir, 'ja', outFilename + '-地番.txt'); 86 | const txt = serializeApiDataTxt(apiData); 87 | await fs.promises.mkdir(path.dirname(outFileTXT), { recursive: true }); 88 | await fs.promises.writeFile(outFileTXT, txt.data); 89 | 90 | console.log(`${outFilename}: ${apiData.length.toString(10).padEnd(4, ' ')} 件の町字の地番を出力した`); 91 | } 92 | 93 | async function main(argv: string[]) { 94 | const outDir = argv[2] || path.join(import.meta.dirname, '..', '..', 'out', 'api'); 95 | fs.mkdirSync(outDir, { recursive: true }); 96 | 97 | console.log('事前準備: 町字データを取得中...'); 98 | const machiAzaData = await getAndParseCSVDataForId('ba-o1-000000_g2-000003'); // 市区町村 & 町字 99 | const machiAzaDataByCode = new Map(machiAzaData.map((ma) => [ 100 | `${ma.lg_code}|${ma.machiaza_id}`, 101 | ma 102 | ])); 103 | const machiAzas: MachiAzaData[] = []; 104 | for (const ma of machiAzaData) { 105 | if (machiAzas.findIndex((c) => c.lg_code === ma.lg_code) > 0) { 106 | continue; 107 | } 108 | machiAzas.push(ma); 109 | } 110 | console.log('事前準備: 町字データを取得しました'); 111 | 112 | const progress = new cliProgress.SingleBar({ 113 | format: ' {bar} {percentage}% | ETA: {eta_formatted} | {value}/{total}', 114 | barCompleteChar: '\u2588', 115 | barIncompleteChar: '\u2591', 116 | etaBuffer: 30, 117 | fps: 2, 118 | // No-TTY output is required for CI/CD environments 119 | noTTYOutput: true, 120 | }); 121 | progress.start(machiAzas.length, 0); 122 | try { 123 | 124 | let currentLgCode: string | undefined = undefined; 125 | for (const ma of machiAzas) { 126 | if (currentLgCode && ma.lg_code === currentLgCode) { 127 | // we have already processed this lg_code, so we can skip it 128 | progress.increment(); 129 | continue; 130 | } else if (currentLgCode !== ma.lg_code) { 131 | currentLgCode = ma.lg_code; 132 | } 133 | let area = `${ma.pref} ${ma.county}${ma.city}`; 134 | if (ma.ward !== '') { 135 | area += ` ${ma.ward}`; 136 | } 137 | const searchQuery = `${area} 地番マスター`; 138 | const results = await ckanPackageSearch(searchQuery); 139 | const chibanDataRef = findResultByTypeAndArea(results, '地番マスター（市区町村）', area); 140 | const chibanPosDataRef = findResultByTypeAndArea(results, '地番マスター位置参照拡張（市区町村）', area); 141 | if (!chibanDataRef) { 142 | console.error(`Insufficient data found for ${searchQuery} (地番マスター)`); 143 | progress.increment(); 144 | continue; 145 | } 146 | 147 | const mainStream = getAndStreamCSVDataForId(chibanDataRef.name); 148 | const posStream = chibanPosDataRef ? 149 | getAndStreamCSVDataForId(chibanPosDataRef.name) 150 | : 151 | // 位置参照拡張データが無い場合もある 152 | (async function*() {})(); 153 | 154 | const rawData = mergeDataLeftJoin(mainStream, posStream, ['lg_code', 'machiaza_id', 'prc_id'], true); 155 | // console.log(`処理: ${ma.pref} ${ma.county}${ma.city} ${ma.ward} の地番データを処理中...`); 156 | 157 | let currentMachiAza: MachiAzaData | undefined = undefined; 158 | const apiData: ChibanApi = []; 159 | let currentChibanList: SingleChiban[] = []; 160 | for await (const raw of rawData) { 161 | const ma = machiAzaDataByCode.get(`${raw.lg_code}|${raw.machiaza_id}`); 162 | if (!ma) { 163 | continue; 164 | } 165 | if (currentMachiAza && (currentMachiAza.machiaza_id !== ma.machiaza_id || currentMachiAza.lg_code !== ma.lg_code)) { 166 | apiData.push({ 167 | machiAza: currentMachiAza, 168 | chibans: currentChibanList, 169 | }); 170 | currentChibanList = []; 171 | currentMachiAza = ma; 172 | } 173 | if (!currentMachiAza) { 174 | currentMachiAza = ma; 175 | } 176 | 177 | currentChibanList.push({ 178 | prc_num1: raw.prc_num1, 179 | prc_num2: raw.prc_num2 !== '' ? raw.prc_num2 : undefined, 180 | prc_num3: raw.prc_num3 !== '' ? raw.prc_num3 : undefined, 181 | point: 'rep_srid' in raw ? projectABRData(raw) : undefined, 182 | }); 183 | } 184 | if (currentMachiAza && currentChibanList.length > 0) { 185 | apiData.push({ 186 | machiAza: currentMachiAza, 187 | chibans: currentChibanList, 188 | }); 189 | } 190 | await outputChibanData(outDir, path.join( 191 | ma.pref, 192 | `${ma.county}${ma.city}${ma.ward}`, 193 | ), apiData); 194 | progress.increment(); 195 | } 196 | } finally { 197 | progress.stop(); 198 | } 199 | } 200 | 201 | export default main; 202 | -------------------------------------------------------------------------------- /src/processes/10_refresh_csv_ranges.test.ts: -------------------------------------------------------------------------------- 1 | import assert from 'node:assert'; 2 | import test from 'node:test'; 3 | 4 | import fs from 'node:fs/promises'; 5 | 6 | import mainPrefCity from './01_make_prefecture_city.js'; 7 | import mainMachiAza from './02_make_machi_aza.js'; 8 | import mainRsdt from './03_make_rsdt.js'; 9 | import mainChiban from './04_make_chiban.js'; 10 | 11 | import main from './10_refresh_csv_ranges.js' 12 | import { MachiAzaApi } from '../data.js'; 13 | 14 | await test.describe('with filter for 302015 (和歌山県和歌山市)', async () => { 15 | test.before(() => { 16 | process.env.SETTINGS_JSON = JSON.stringify({ lgCodes: ['302015'] }); 17 | }); 18 | 19 | test.after(() => { 20 | delete process.env.SETTINGS_JSON; 21 | }); 22 | 23 | await test('it generates the API', async () => { 24 | await fs.rm('./out/api_wakayama_wakayama', { recursive: true, force: true }); 25 | await mainPrefCity(['', '', './out/api_wakayama_wakayama']); 26 | await mainMachiAza(['', '', './out/api_wakayama_wakayama']); 27 | await mainRsdt(['', '', './out/api_wakayama_wakayama']); 28 | await mainChiban(['', '', './out/api_wakayama_wakayama']); 29 | await main(['', '', './out/api_wakayama_wakayama']); 30 | assert.ok(true); 31 | 32 | const machiAzaApi = JSON.parse(await fs.readFile('./out/api_wakayama_wakayama/ja/和歌山県/和歌山市.json', 'utf-8')) as MachiAzaApi; 33 | const data = machiAzaApi.data; 34 | assert(data.length > 100); 35 | assert.equal(data[0].oaza_cho, '葵町'); 36 | assert('地番' in (data[0].csv_ranges || {})); 37 | assert('住居表示' in (data[0].csv_ranges || {})); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /src/processes/10_refresh_csv_ranges.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs'; 2 | import path from 'node:path'; 3 | 4 | import { parse as csvParse } from 'csv-parse'; 5 | import cliProgress from 'cli-progress'; 6 | 7 | import { cityName, MachiAzaApi, machiAzaName, PrefectureApi, prefectureName, SingleCity, SinglePrefecture } from '../data.js'; 8 | import { HeaderRow } from '../address_data.js'; 9 | 10 | function readUntilHeaderEnd(path: string): Promise { 11 | const HEADER_END = Buffer.from('=END=\n', 'utf8'); 12 | return new Promise((resolve, reject) => { 13 | const readStream = fs.createReadStream(path); 14 | const chunks: Buffer[] = []; 15 | let foundEnd = false; 16 | readStream.on('data', (chunk: Buffer) => { 17 | if (foundEnd) { 18 | return; 19 | } 20 | const headerEndPos = chunk.indexOf(HEADER_END); 21 | if (headerEndPos !== -1) { 22 | foundEnd = true; 23 | readStream.close(); 24 | chunks.push(chunk.subarray(0, headerEndPos)); 25 | } else { 26 | chunks.push(chunk); 27 | } 28 | }); 29 | readStream.on('close', () => { 30 | resolve(Buffer.concat(chunks)); 31 | }); 32 | readStream.on('error', (err) => { 33 | reject(err); 34 | }); 35 | }); 36 | } 37 | 38 | export async function getRangesFromCSV(path: string): Promise { 39 | try { 40 | const headerData = await readUntilHeaderEnd(path); 41 | const headerStream = csvParse(headerData); 42 | const rows: HeaderRow[] = []; 43 | for await (const line_ of headerStream) { 44 | const line = line_ as [string, string, string]; 45 | if (line[0] === '=END=') { 46 | break; 47 | } 48 | const [name, start, length] = line; 49 | rows.push({ 50 | name, 51 | offset: parseInt(start), 52 | length: parseInt(length), 53 | }); 54 | } 55 | return rows; 56 | } catch (e) { 57 | if ((e as NodeJS.ErrnoException).code === 'ENOENT') { 58 | return undefined; 59 | } 60 | throw e; 61 | } 62 | } 63 | 64 | async function main(argv: string[]) { 65 | const apiDir = argv[2] || path.join(import.meta.dirname, '..', '..', 'out', 'api'); 66 | const jaFile = path.join(apiDir, 'ja.json'); 67 | const api = JSON.parse(fs.readFileSync(jaFile, 'utf-8')) as PrefectureApi; 68 | 69 | const progress = new cliProgress.SingleBar({ 70 | format: ' {bar} {percentage}% | ETA: {eta_formatted} | {value}/{total}', 71 | barCompleteChar: '\u2588', 72 | barIncompleteChar: '\u2591', 73 | etaBuffer: 30, 74 | fps: 2, 75 | // No-TTY output is required for CI/CD environments 76 | noTTYOutput: true, 77 | }); 78 | 79 | const flatCities: [SinglePrefecture, SingleCity][] = []; 80 | for (const pref of api.data) { 81 | for (const city of pref.cities) { 82 | flatCities.push([pref, city]); 83 | } 84 | } 85 | 86 | progress.start(flatCities.length, 0); 87 | try { 88 | for (const [pref, city] of flatCities) { 89 | const cityPrefix = path.join(apiDir, 'ja', prefectureName(pref), cityName(city)); 90 | const [ 91 | chibanHeader, 92 | rsdtHeader, 93 | ] = await Promise.all([ 94 | getRangesFromCSV(`${cityPrefix}-地番.txt`), 95 | getRangesFromCSV(`${cityPrefix}-住居表示.txt`), 96 | ]); 97 | 98 | if (!chibanHeader && !rsdtHeader) { 99 | progress.increment(); 100 | continue; 101 | } 102 | 103 | const maData = JSON.parse(await fs.promises.readFile(`${cityPrefix}.json`, 'utf8')) as MachiAzaApi; 104 | for (const headerRow of chibanHeader || []) { 105 | const ma = maData.data.find((ma) => machiAzaName(ma) === headerRow.name); 106 | if (ma) { 107 | ma.csv_ranges = ma.csv_ranges || {}; 108 | ma.csv_ranges['地番'] = { start: headerRow.offset, length: headerRow.length }; 109 | } 110 | } 111 | for (const headerRow of rsdtHeader || []) { 112 | const ma = maData.data.find((ma) => machiAzaName(ma) === headerRow.name); 113 | if (ma) { 114 | ma.csv_ranges = ma.csv_ranges || {}; 115 | ma.csv_ranges['住居表示'] = { start: headerRow.offset, length: headerRow.length }; 116 | } 117 | } 118 | 119 | await fs.promises.writeFile(`${cityPrefix}.json`, JSON.stringify(maData)); 120 | 121 | progress.increment(); 122 | } 123 | } finally { 124 | progress.stop(); 125 | } 126 | } 127 | 128 | export default main; 129 | -------------------------------------------------------------------------------- /src/processes/99_create_stats.ts: -------------------------------------------------------------------------------- 1 | import fs from 'node:fs'; 2 | import path from 'node:path'; 3 | import { parse as csvParse } from 'csv-parse'; 4 | import { pipeline } from 'node:stream/promises'; 5 | import { cityName, MachiAzaApi, PrefectureApi, prefectureName } from '../data.js'; 6 | 7 | type Stats = { 8 | prefCount: number; 9 | lgCount: number; 10 | machiAzaCount: number; 11 | machiAzaWithChibanCount: number; 12 | machiAzaWithRsdtCount: number; 13 | machiAzaWithChibanAndRsdtCount: number; 14 | rsdtCount: number; 15 | rsdtWithPosCount: number; 16 | chibanCount: number; 17 | chibanWithPosCount: number; 18 | } 19 | 20 | async function getCountForCSVRange(path: string, range?: { start: number, length: number }): Promise<[number, number]> { 21 | if (!range) { return [0, 0]; } 22 | try { 23 | const { start, length } = range; 24 | const fd = await fs.promises.open(path, 'r'); 25 | const stream = fd.createReadStream({ 26 | start, 27 | end: start + length - 1, 28 | }); 29 | const parser = csvParse({ 30 | columns: true, 31 | from_line: 2, 32 | }); 33 | let count = 0; 34 | let countWithPos = 0; 35 | await pipeline(stream, parser, async (source: AsyncIterable<{lat?: string, lng?: string}>) => { 36 | for await (const record of source) { 37 | count++; 38 | if (record['lng'] && record['lat']) { 39 | countWithPos++; 40 | } 41 | } 42 | }); 43 | return [count, countWithPos]; 44 | } catch (e) { 45 | if ((e as NodeJS.ErrnoException).code === 'ENOENT') { 46 | return [0, 0]; 47 | } 48 | throw e; 49 | } 50 | } 51 | 52 | async function main(argv: string[]) { 53 | const dataDir = argv[2] || path.join(import.meta.dirname, '..', '..', 'out', 'api'); 54 | 55 | const jaJSON = await fs.promises.readFile(path.join(dataDir, 'ja.json'), 'utf8'); 56 | const ja = JSON.parse(jaJSON) as PrefectureApi; 57 | 58 | const stats: Stats = { 59 | prefCount: ja.data.length, 60 | lgCount: ja.data.reduce((acc, pref) => acc + pref.cities.length, 0), 61 | machiAzaCount: 0, 62 | machiAzaWithRsdtCount: 0, 63 | machiAzaWithChibanCount: 0, 64 | machiAzaWithChibanAndRsdtCount: 0, 65 | rsdtCount: 0, 66 | rsdtWithPosCount: 0, 67 | chibanCount: 0, 68 | chibanWithPosCount: 0, 69 | }; 70 | 71 | for (const pref of ja.data) { 72 | for (const lg of pref.cities) { 73 | const machiAzaJSON = await fs.promises.readFile(path.join( 74 | dataDir, 'ja', prefectureName(pref), `${cityName(lg)}.json` 75 | ), 'utf-8'); 76 | const machiAza = JSON.parse(machiAzaJSON) as MachiAzaApi; 77 | stats.machiAzaCount += machiAza.data.length; 78 | 79 | for (const ma of machiAza.data) { 80 | if ((ma.csv_ranges || {})['地番']) { 81 | stats.machiAzaWithChibanCount++; 82 | } 83 | if ((ma.csv_ranges || {})['住居表示']) { 84 | stats.machiAzaWithRsdtCount++; 85 | } 86 | if ((ma.csv_ranges || {})['地番'] && (ma.csv_ranges || {})['住居表示']) { 87 | stats.machiAzaWithChibanAndRsdtCount++; 88 | } 89 | 90 | const [ chibanCount, chibanWithPosCount ] = await getCountForCSVRange(path.join( 91 | dataDir, 'ja', prefectureName(pref), `${cityName(lg)}-地番.txt` 92 | ), (ma.csv_ranges || {})['地番']); 93 | stats.chibanCount += chibanCount; 94 | stats.chibanWithPosCount += chibanWithPosCount; 95 | 96 | const [ rsdtCount, rsdtWithPosCount] = await getCountForCSVRange(path.join( 97 | dataDir, 'ja', prefectureName(pref), `${cityName(lg)}-住居表示.txt` 98 | ), (ma.csv_ranges || {})['住居表示']); 99 | stats.rsdtCount += rsdtCount; 100 | stats.rsdtWithPosCount += rsdtWithPosCount; 101 | } 102 | } 103 | } 104 | 105 | await fs.promises.writeFile(path.join(dataDir, 'stats.json'), JSON.stringify(stats, null, 2)); 106 | } 107 | 108 | export default main; 109 | -------------------------------------------------------------------------------- /test/fixtures/lib/settings/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "lgCodes": ["^01"] 3 | } 4 | -------------------------------------------------------------------------------- /test/fixtures/lib/zip_tools/double_level.csv.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geolonia/japanese-addresses-v2/6fa7674951437a05ad9368fad3b0bac38c463e84/test/fixtures/lib/zip_tools/double_level.csv.zip -------------------------------------------------------------------------------- /test/fixtures/lib/zip_tools/double_level1.csv: -------------------------------------------------------------------------------- 1 | It,works! 2 | Double,1 3 | -------------------------------------------------------------------------------- /test/fixtures/lib/zip_tools/double_level1.csv.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geolonia/japanese-addresses-v2/6fa7674951437a05ad9368fad3b0bac38c463e84/test/fixtures/lib/zip_tools/double_level1.csv.zip -------------------------------------------------------------------------------- /test/fixtures/lib/zip_tools/double_level2.csv: -------------------------------------------------------------------------------- 1 | It,works! 2 | Double,2 3 | -------------------------------------------------------------------------------- /test/fixtures/lib/zip_tools/double_level2.csv.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geolonia/japanese-addresses-v2/6fa7674951437a05ad9368fad3b0bac38c463e84/test/fixtures/lib/zip_tools/double_level2.csv.zip -------------------------------------------------------------------------------- /test/fixtures/lib/zip_tools/single_level.csv.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/geolonia/japanese-addresses-v2/6fa7674951437a05ad9368fad3b0bac38c463e84/test/fixtures/lib/zip_tools/single_level.csv.zip -------------------------------------------------------------------------------- /tsconfig.dist-cjs.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/node22/tsconfig.json", 3 | "compileOnSave": false, 4 | "compilerOptions": { 5 | "lib": ["ESNext"], 6 | "module": "commonjs", 7 | "moduleResolution": "Classic", 8 | "rootDir": "src", 9 | "outDir": "dist/cjs", 10 | "types": ["node"], 11 | "declaration": false 12 | }, 13 | "files": [ 14 | "src/data.ts" 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /tsconfig.dist-esm.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/node22/tsconfig.json", 3 | "compileOnSave": false, 4 | "compilerOptions": { 5 | "lib": ["ESNext"], 6 | "rootDir": "src", 7 | "outDir": "dist/esm", 8 | "types": ["node"], 9 | "declaration": true 10 | }, 11 | "files": [ 12 | "src/data.ts" 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@tsconfig/node22/tsconfig.json", 3 | "compileOnSave": false, 4 | "compilerOptions": { 5 | "lib": ["ESNext"], 6 | "rootDir": "src", 7 | "outDir": "build", 8 | "types": ["node"], 9 | "declaration": true 10 | }, 11 | "filesGlob": ["./src/**/*.ts"], 12 | "include": ["src/**/*.ts"], 13 | "exclude": ["dist", "build", "node_modules"] 14 | } 15 | --------------------------------------------------------------------------------