├── tag ├── .npmignore ├── .gitignore ├── dist.js.LICENSE.txt ├── webpack.config.js ├── package.json ├── test.html └── style.css ├── .vscode └── settings.json ├── lib ├── .npmignore ├── .eslintignore ├── .gitignore ├── babel.config.json ├── .vscode │ └── settings.json ├── bus-icon.png ├── bus_stop_icon.png ├── jest.config.js ├── dist.js.LICENSE.txt ├── webpack.config.js ├── package.json ├── test2.html ├── ds.txt ├── readme.md ├── realtime-bus-locations.html ├── test │ └── helper.test.js ├── src │ └── internal.js ├── test.html └── test_v1.html ├── .gitignore ├── api ├── .gitignore ├── wrangler.toml ├── webpack.config.js ├── package.json ├── readme.md ├── src │ ├── index.js │ └── functionParams.js └── test.js ├── storage ├── caddy_config │ └── .gitignore ├── caddy_data │ └── .gitignore ├── Caddyfile ├── addSuccess.png ├── addStoragePage.png ├── .gitignore ├── go.mod ├── docker-compose.yml ├── go.md ├── go.sum ├── fileServer.go ├── fileServer-v1.go ├── serverUpdater.go ├── s3.go ├── readme.md ├── ftps.go ├── dataUpdater.go └── dataUpdater-v1.go ├── babel.config.json ├── about ├── public │ ├── bus-icon.png │ ├── butter.png │ ├── favicon.ico │ ├── bus_stop_icon.png │ ├── marker-shadow.png │ ├── marker-icon-2x.png │ ├── v0.0.0 │ │ └── root.json │ └── index.html ├── babel.config.js ├── src │ ├── plugins │ │ └── vuetify.js │ ├── main.js │ ├── App.vue │ └── components │ │ ├── SampleCode.vue │ │ └── HelloWorld.vue ├── vue.config.js ├── docker-compose.yml ├── Dockerfile ├── .gitignore ├── jsconfig.json ├── README.md └── package.json ├── cmd ├── requirements.txt ├── docker-compose.yml ├── .gitignore ├── cmd.sh ├── 0_rm.go ├── 4_unziper.py ├── subdirectory_fix.py ├── Dockerfile ├── go.mod ├── helper │ ├── file.go │ ├── sign.go │ └── tar.go ├── 3_downloader.py ├── index.html ├── 6_add_gtfs_id_info.go ├── 9_addSign.go ├── check.go ├── sender.go ├── 8_add_stopdata.py ├── 8_add_stopdata_feed.py ├── 999_uploadOriginData.go ├── 999_uploadOriginData_feed.go ├── receiver.go ├── 7_add_datalist.go ├── 2_line2obj.py ├── go.sum ├── 1_getDataList.py └── 5_split.go ├── tag-maker ├── public │ ├── bus-icon.png │ ├── favicon.ico │ ├── bus_stop_icon.png │ ├── marker-icon-2x.png │ ├── marker-shadow.png │ ├── v0.0.0 │ │ └── root.json │ └── index.html ├── babel.config.js ├── src │ ├── plugins │ │ └── vuetify.js │ ├── main.js │ └── App.vue ├── vue.config.js ├── docker-compose.yml ├── Dockerfile ├── .gitignore ├── jsconfig.json ├── README.md └── package.json ├── timetable-app ├── public │ ├── favicon.ico │ ├── bus-icon.png │ ├── bus_stop_icon.png │ ├── marker-icon-2x.png │ ├── marker-shadow.png │ ├── icon │ │ ├── timetable-app-192.png │ │ └── timetable-app-512.png │ ├── manifest.json │ ├── v0.0.0 │ │ └── root.json │ ├── sw.js │ └── index.html ├── babel.config.js ├── src │ ├── plugins │ │ └── vuetify.js │ ├── components │ │ ├── HelloWorld.vue │ │ └── SampleCode.vue │ ├── main.js │ └── App.vue ├── vue.config.js ├── docker-compose.yml ├── Dockerfile ├── .gitignore ├── jsconfig.json ├── README.md └── package.json ├── jest.config.js ├── docker-compose.yml ├── .eslintrc.json ├── package.json ├── LICENSE.txt └── README.md /tag/.npmignore: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tag/.gitignore: -------------------------------------------------------------------------------- 1 | dist.js -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | } -------------------------------------------------------------------------------- /lib/.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist/* 3 | -------------------------------------------------------------------------------- /api/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist -------------------------------------------------------------------------------- /lib/.eslintignore: -------------------------------------------------------------------------------- 1 | dist.js 2 | *.test.js 3 | -------------------------------------------------------------------------------- /lib/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | dist.js -------------------------------------------------------------------------------- /storage/caddy_config/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /storage/caddy_data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["@babel/preset-env"] 3 | } -------------------------------------------------------------------------------- /lib/babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["@babel/preset-env"] 3 | } -------------------------------------------------------------------------------- /lib/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "liveServer.settings.port": 5501 3 | } -------------------------------------------------------------------------------- /storage/Caddyfile: -------------------------------------------------------------------------------- 1 | your-domain.com { 2 | reverse_proxy file-server:8000 3 | } 4 | -------------------------------------------------------------------------------- /lib/bus-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/lib/bus-icon.png -------------------------------------------------------------------------------- /tag/dist.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /*! For license information please see dist.js.LICENSE.txt */ 2 | -------------------------------------------------------------------------------- /lib/bus_stop_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/lib/bus_stop_icon.png -------------------------------------------------------------------------------- /storage/addSuccess.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/storage/addSuccess.png -------------------------------------------------------------------------------- /about/public/bus-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/about/public/bus-icon.png -------------------------------------------------------------------------------- /about/public/butter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/about/public/butter.png -------------------------------------------------------------------------------- /about/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/about/public/favicon.ico -------------------------------------------------------------------------------- /cmd/requirements.txt: -------------------------------------------------------------------------------- 1 | h3==3.7.6 2 | pandas==2.0.0 3 | nltk==3.8.1 4 | requests==2.31.0 5 | numpy==1.24.3 6 | -------------------------------------------------------------------------------- /storage/addStoragePage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/storage/addStoragePage.png -------------------------------------------------------------------------------- /about/public/bus_stop_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/about/public/bus_stop_icon.png -------------------------------------------------------------------------------- /about/public/marker-shadow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/about/public/marker-shadow.png -------------------------------------------------------------------------------- /tag-maker/public/bus-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/tag-maker/public/bus-icon.png -------------------------------------------------------------------------------- /tag-maker/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/tag-maker/public/favicon.ico -------------------------------------------------------------------------------- /about/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | '@vue/cli-plugin-babel/preset' 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /about/public/marker-icon-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/about/public/marker-icon-2x.png -------------------------------------------------------------------------------- /tag-maker/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | '@vue/cli-plugin-babel/preset' 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /timetable-app/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/timetable-app/public/favicon.ico -------------------------------------------------------------------------------- /tag-maker/public/bus_stop_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/tag-maker/public/bus_stop_icon.png -------------------------------------------------------------------------------- /tag-maker/public/marker-icon-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/tag-maker/public/marker-icon-2x.png -------------------------------------------------------------------------------- /tag-maker/public/marker-shadow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/tag-maker/public/marker-shadow.png -------------------------------------------------------------------------------- /timetable-app/babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: [ 3 | '@vue/cli-plugin-babel/preset' 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /timetable-app/public/bus-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/timetable-app/public/bus-icon.png -------------------------------------------------------------------------------- /api/wrangler.toml: -------------------------------------------------------------------------------- 1 | name = "butter" 2 | main = "dist/worker.js" 3 | compatibility_date = "2023-04-27" 4 | type = "webpack" 5 | -------------------------------------------------------------------------------- /timetable-app/public/bus_stop_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/timetable-app/public/bus_stop_icon.png -------------------------------------------------------------------------------- /timetable-app/public/marker-icon-2x.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/timetable-app/public/marker-icon-2x.png -------------------------------------------------------------------------------- /timetable-app/public/marker-shadow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/timetable-app/public/marker-shadow.png -------------------------------------------------------------------------------- /storage/.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.json 3 | v0.0.0/* 4 | public/* 5 | !server.json 6 | *.tar/ 7 | old 8 | *.txt 9 | *.env 10 | public_v1/* 11 | -------------------------------------------------------------------------------- /timetable-app/public/icon/timetable-app-192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/timetable-app/public/icon/timetable-app-192.png -------------------------------------------------------------------------------- /timetable-app/public/icon/timetable-app-512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/takoyaki-3/butter/HEAD/timetable-app/public/icon/timetable-app-512.png -------------------------------------------------------------------------------- /lib/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | transform: { 3 | '^.+\\.jsx?$': 'babel-jest' // JavaScriptファイル用 4 | }, 5 | testEnvironment: 'jsdom' 6 | } 7 | -------------------------------------------------------------------------------- /about/src/plugins/vuetify.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue'; 2 | import Vuetify from 'vuetify/lib/framework'; 3 | 4 | Vue.use(Vuetify); 5 | 6 | export default new Vuetify({ 7 | }); 8 | -------------------------------------------------------------------------------- /cmd/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | converter: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile 7 | volumes: 8 | - ./:/app 9 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | transform: { 3 | '^.+\\.jsx?$': 'babel-jest', // JavaScriptファイル用 4 | }, 5 | testEnvironment: 'jsdom', 6 | // その他の設定... 7 | }; 8 | -------------------------------------------------------------------------------- /tag-maker/src/plugins/vuetify.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue'; 2 | import Vuetify from 'vuetify/lib/framework'; 3 | 4 | Vue.use(Vuetify); 5 | 6 | export default new Vuetify({ 7 | }); 8 | -------------------------------------------------------------------------------- /timetable-app/src/plugins/vuetify.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue'; 2 | import Vuetify from 'vuetify/lib/framework'; 3 | 4 | Vue.use(Vuetify); 5 | 6 | export default new Vuetify({ 7 | }); 8 | -------------------------------------------------------------------------------- /about/vue.config.js: -------------------------------------------------------------------------------- 1 | const { defineConfig } = require('@vue/cli-service') 2 | module.exports = defineConfig({ 3 | outputDir: '../dist/', 4 | transpileDependencies: [ 5 | 'vuetify' 6 | ] 7 | }) 8 | -------------------------------------------------------------------------------- /tag-maker/vue.config.js: -------------------------------------------------------------------------------- 1 | const { defineConfig } = require('@vue/cli-service') 2 | module.exports = defineConfig({ 3 | outputDir: '../dist/tag-maker', 4 | transpileDependencies: [ 5 | 'vuetify' 6 | ] 7 | }) 8 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | web: 4 | image: node:14 5 | volumes: 6 | - .:/usr/src/app 7 | working_dir: /usr/src/app 8 | command: ['sh', '-c', 'npm install && npm run build'] 9 | -------------------------------------------------------------------------------- /timetable-app/vue.config.js: -------------------------------------------------------------------------------- 1 | const { defineConfig } = require('@vue/cli-service') 2 | module.exports = defineConfig({ 3 | outputDir: '../dist/timetable', 4 | transpileDependencies: [ 5 | 'vuetify' 6 | ] 7 | }) 8 | -------------------------------------------------------------------------------- /about/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | vuetify-app: 4 | build: . 5 | volumes: 6 | - .:/app 7 | ports: 8 | - '8080:8080' 9 | command: sleep 3600 10 | # command: vue create . 11 | -------------------------------------------------------------------------------- /tag-maker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | vuetify-app: 4 | build: . 5 | volumes: 6 | - .:/app 7 | ports: 8 | - '8080:8080' 9 | command: sleep 3600 10 | # command: vue create . 11 | -------------------------------------------------------------------------------- /timetable-app/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | vuetify-app: 4 | build: . 5 | volumes: 6 | - .:/app 7 | ports: 8 | - '8080:8080' 9 | command: sleep 3600 10 | # command: vue create . 11 | -------------------------------------------------------------------------------- /cmd/.gitignore: -------------------------------------------------------------------------------- 1 | output.txt 2 | data_v0.json 3 | data_v1.json 4 | gtfs/* 5 | dir_out/* 6 | v0.0.0/* 7 | v1.0.0/* 8 | *.pem 9 | s3-conf.json 10 | *.exe 11 | *.txt 12 | *.csv 13 | archive.tar 14 | !requirements.txt 15 | *.tar 16 | dist 17 | conf.json 18 | feed/* 19 | feed_dir_out/* 20 | -------------------------------------------------------------------------------- /about/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:14 2 | 3 | WORKDIR /app 4 | 5 | COPY package*.json ./ 6 | 7 | RUN npm install 8 | 9 | COPY . . 10 | 11 | RUN npm install -g @vue/cli 12 | 13 | # RUN vue create --default --force . 14 | 15 | # RUN vue add vuetify 16 | 17 | # CMD [ "npm", "run", "serve" ] 18 | -------------------------------------------------------------------------------- /tag-maker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:14 2 | 3 | WORKDIR /app 4 | 5 | COPY package*.json ./ 6 | 7 | RUN npm install 8 | 9 | COPY . . 10 | 11 | RUN npm install -g @vue/cli 12 | 13 | # RUN vue create --default --force . 14 | 15 | # RUN vue add vuetify 16 | 17 | # CMD [ "npm", "run", "serve" ] 18 | -------------------------------------------------------------------------------- /timetable-app/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:14 2 | 3 | WORKDIR /app 4 | 5 | COPY package*.json ./ 6 | 7 | RUN npm install 8 | 9 | COPY . . 10 | 11 | RUN npm install -g @vue/cli 12 | 13 | # RUN vue create --default --force . 14 | 15 | # RUN vue add vuetify 16 | 17 | # CMD [ "npm", "run", "serve" ] 18 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "commonjs": true, 5 | "es2021": true 6 | }, 7 | "extends": "standard", 8 | "parserOptions": { 9 | "ecmaVersion": "latest" 10 | }, 11 | "rules": { 12 | "camelcase": "off" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /storage/go.mod: -------------------------------------------------------------------------------- 1 | module a.b/c 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/golang/snappy v0.0.4 // indirect 7 | github.com/hashicorp/errwrap v1.1.0 // indirect 8 | github.com/jlaffaye/ftp v0.1.0 // indirect 9 | github.com/joho/godotenv v1.5.1 // indirect 10 | github.com/takoyaki-3/go-json v0.0.2 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /about/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | 6 | # local env files 7 | .env.local 8 | .env.*.local 9 | 10 | # Log files 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | pnpm-debug.log* 15 | 16 | # Editor directories and files 17 | .idea 18 | .vscode 19 | *.suo 20 | *.ntvs* 21 | *.njsproj 22 | *.sln 23 | *.sw? 24 | -------------------------------------------------------------------------------- /tag-maker/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | 6 | # local env files 7 | .env.local 8 | .env.*.local 9 | 10 | # Log files 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | pnpm-debug.log* 15 | 16 | # Editor directories and files 17 | .idea 18 | .vscode 19 | *.suo 20 | *.ntvs* 21 | *.njsproj 22 | *.sln 23 | *.sw? 24 | -------------------------------------------------------------------------------- /timetable-app/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | node_modules 3 | /dist 4 | 5 | 6 | # local env files 7 | .env.local 8 | .env.*.local 9 | 10 | # Log files 11 | npm-debug.log* 12 | yarn-debug.log* 13 | yarn-error.log* 14 | pnpm-debug.log* 15 | 16 | # Editor directories and files 17 | .idea 18 | .vscode 19 | *.suo 20 | *.ntvs* 21 | *.njsproj 22 | *.sln 23 | *.sw? 24 | -------------------------------------------------------------------------------- /about/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "esnext", 5 | "baseUrl": "./", 6 | "moduleResolution": "node", 7 | "paths": { 8 | "@/*": [ 9 | "src/*" 10 | ] 11 | }, 12 | "lib": [ 13 | "esnext", 14 | "dom", 15 | "dom.iterable", 16 | "scripthost" 17 | ] 18 | } 19 | } -------------------------------------------------------------------------------- /lib/dist.js.LICENSE.txt: -------------------------------------------------------------------------------- 1 | /*! 2 | * The buffer module from node.js, for the browser. 3 | * 4 | * @author Feross Aboukhadijeh 5 | * @license MIT 6 | */ 7 | 8 | /*! ieee754. BSD-3-Clause License. Feross Aboukhadijeh */ 9 | 10 | /*! safe-buffer. MIT License. Feross Aboukhadijeh */ 11 | -------------------------------------------------------------------------------- /tag-maker/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "esnext", 5 | "baseUrl": "./", 6 | "moduleResolution": "node", 7 | "paths": { 8 | "@/*": [ 9 | "src/*" 10 | ] 11 | }, 12 | "lib": [ 13 | "esnext", 14 | "dom", 15 | "dom.iterable", 16 | "scripthost" 17 | ] 18 | } 19 | } -------------------------------------------------------------------------------- /timetable-app/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "module": "esnext", 5 | "baseUrl": "./", 6 | "moduleResolution": "node", 7 | "paths": { 8 | "@/*": [ 9 | "src/*" 10 | ] 11 | }, 12 | "lib": [ 13 | "esnext", 14 | "dom", 15 | "dom.iterable", 16 | "scripthost" 17 | ] 18 | } 19 | } -------------------------------------------------------------------------------- /tag/webpack.config.js: -------------------------------------------------------------------------------- 1 | // webpack.config.js 2 | 3 | const path = require('path'); 4 | 5 | module.exports = { 6 | entry: './src/index.js', 7 | output: { 8 | filename: 'dist.js', 9 | path: path.resolve(__dirname), 10 | library: { 11 | // name: 'Butter', 12 | type: 'module', 13 | }, 14 | }, 15 | experiments: { 16 | outputModule: true, 17 | }, 18 | mode: 'production' 19 | }; 20 | -------------------------------------------------------------------------------- /cmd/cmd.sh: -------------------------------------------------------------------------------- 1 | go run 0_rm.go # 2 | python 1_getDataList.py # 3 | python 2_line2obj.py # 4 | python 3_downloader.py # 5 | python 4_unziper.py # 6 | python subdirectory_fix.py # 7 | go run 5_split.go # 8 | go run 6_add_gtfs_id_info.go # 9 | go run 7_add_datalist.go # 10 | python 8_add_stopdata.py # 11 | python 8_add_stopdata_feed.py # 12 | go run 9_addSign.go # 13 | go run 999_uploadOriginData.go # 14 | go run 999_uploadOriginData_feed.go # 15 | # -------------------------------------------------------------------------------- /about/README.md: -------------------------------------------------------------------------------- 1 | # vuetify-dev 2 | 3 | ## Project setup 4 | ``` 5 | npm install 6 | ``` 7 | 8 | ### Compiles and hot-reloads for development 9 | ``` 10 | npm run serve 11 | ``` 12 | 13 | ### Compiles and minifies for production 14 | ``` 15 | npm run build 16 | ``` 17 | 18 | ### Lints and fixes files 19 | ``` 20 | npm run lint 21 | ``` 22 | 23 | ### Customize configuration 24 | See [Configuration Reference](https://cli.vuejs.org/config/). 25 | -------------------------------------------------------------------------------- /tag-maker/README.md: -------------------------------------------------------------------------------- 1 | # vuetify-dev 2 | 3 | ## Project setup 4 | ``` 5 | npm install 6 | ``` 7 | 8 | ### Compiles and hot-reloads for development 9 | ``` 10 | npm run serve 11 | ``` 12 | 13 | ### Compiles and minifies for production 14 | ``` 15 | npm run build 16 | ``` 17 | 18 | ### Lints and fixes files 19 | ``` 20 | npm run lint 21 | ``` 22 | 23 | ### Customize configuration 24 | See [Configuration Reference](https://cli.vuejs.org/config/). 25 | -------------------------------------------------------------------------------- /timetable-app/README.md: -------------------------------------------------------------------------------- 1 | # vuetify-dev 2 | 3 | ## Project setup 4 | ``` 5 | npm install 6 | ``` 7 | 8 | ### Compiles and hot-reloads for development 9 | ``` 10 | npm run serve 11 | ``` 12 | 13 | ### Compiles and minifies for production 14 | ``` 15 | npm run build 16 | ``` 17 | 18 | ### Lints and fixes files 19 | ``` 20 | npm run lint 21 | ``` 22 | 23 | ### Customize configuration 24 | See [Configuration Reference](https://cli.vuejs.org/config/). 25 | -------------------------------------------------------------------------------- /timetable-app/src/components/HelloWorld.vue: -------------------------------------------------------------------------------- 1 | 12 | 13 | 24 | -------------------------------------------------------------------------------- /timetable-app/public/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Bus TimeTable", 3 | "short_name": "BuTTER", 4 | "start_url": "/index.html", 5 | "display": "standalone", 6 | "background_color": "#ffffff", 7 | "theme_color": "#4A90E2", 8 | "icons": [ 9 | { 10 | "src": "icon/timetable-app-192.png", 11 | "sizes": "192x192", 12 | "type": "image/png" 13 | }, 14 | { 15 | "src": "icon/timetable-app-192.png", 16 | "sizes": "512x512", 17 | "type": "image/png" 18 | } 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /about/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import App from './App.vue' 3 | import vuetify from './plugins/vuetify' 4 | import hljs from 'highlight.js' 5 | import 'highlight.js/styles/github.css' 6 | 7 | Vue.config.productionTip = false 8 | 9 | new Vue({ 10 | vuetify, 11 | render: h => h(App), 12 | mounted() { 13 | this.$nextTick(() => { 14 | const blocks = document.querySelectorAll('pre code'); 15 | blocks.forEach((block) => { 16 | hljs.highlightBlock(block); 17 | }); 18 | }); 19 | }, 20 | }).$mount('#app') 21 | -------------------------------------------------------------------------------- /tag-maker/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import App from './App.vue' 3 | import vuetify from './plugins/vuetify' 4 | import hljs from 'highlight.js' 5 | import 'highlight.js/styles/github.css' 6 | 7 | Vue.config.productionTip = false 8 | 9 | new Vue({ 10 | vuetify, 11 | render: h => h(App), 12 | mounted() { 13 | this.$nextTick(() => { 14 | const blocks = document.querySelectorAll('pre code'); 15 | blocks.forEach((block) => { 16 | hljs.highlightBlock(block); 17 | }); 18 | }); 19 | }, 20 | }).$mount('#app') 21 | -------------------------------------------------------------------------------- /timetable-app/src/main.js: -------------------------------------------------------------------------------- 1 | import Vue from 'vue' 2 | import App from './App.vue' 3 | import vuetify from './plugins/vuetify' 4 | import hljs from 'highlight.js' 5 | import 'highlight.js/styles/github.css' 6 | 7 | Vue.config.productionTip = false 8 | 9 | new Vue({ 10 | vuetify, 11 | render: h => h(App), 12 | mounted() { 13 | this.$nextTick(() => { 14 | const blocks = document.querySelectorAll('pre code'); 15 | blocks.forEach((block) => { 16 | hljs.highlightBlock(block); 17 | }); 18 | }); 19 | }, 20 | }).$mount('#app') 21 | -------------------------------------------------------------------------------- /cmd/0_rm.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | dirPaths := []string{"./dir_out", "./gtfs", "./v0.0.0", "./data_v0.json", "./data_v1.json", "./feed_merged_csv_file.csv", "./merged_csv_file.csv", "./dataList.txt", "./feed", "./feed_dir_out", "./v1.0.0"} 10 | 11 | for _, dirPath := range dirPaths { 12 | err := os.RemoveAll(dirPath) 13 | if err != nil { 14 | fmt.Printf("Failed to delete directory: %v\n", err) 15 | } else { 16 | fmt.Println("Directory [" + dirPath + "] deleted successfully.", dirPath) 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /tag/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "butter-tag-v1", 3 | "version": "1.1.2", 4 | "description": "", 5 | "main": "webpack.config.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "build": "webpack" 9 | }, 10 | "keywords": [ 11 | "GTFS", 12 | "Bus", 13 | "Timetable" 14 | ], 15 | "author": "yutotom(https://github.com/yutotom),takoyaki3(https://github.com/takoyaki-3/)", 16 | "license": "MIT", 17 | "dependencies": { 18 | "butter-lib": "^1.1.0" 19 | }, 20 | "devDependencies": { 21 | "webpack-cli": "^5.1.4" 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /about/public/v0.0.0/root.json: -------------------------------------------------------------------------------- 1 | { 2 | "pub_keys": [ 3 | { 4 | "pubkey": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/public_key.pem" 5 | } 6 | ], 7 | "hosts": [ 8 | "https://storage.app.takoyaki3.com/", 9 | "https://butter.oozora283.com/" 10 | ], 11 | "original_data": { 12 | "host": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/v0.0.0/originalData/" 13 | }, 14 | "api_endpoints": [ 15 | "https://butter.hatano-yuuta7921.workers.dev/" 16 | ], 17 | "api_data_hosts": [ 18 | "https://storage.app.takoyaki3.com/" 19 | ], 20 | "last_update": "2023-05-06T19_02_00+09_00" 21 | } -------------------------------------------------------------------------------- /tag/test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Test 7 | 8 | 9 | 10 |

Test

11 | 12 |
13 |
14 | 15 |
16 | 17 | 18 | // idを地図上から探せる機能、名前から検索できる機能 19 | 20 | -------------------------------------------------------------------------------- /tag-maker/public/v0.0.0/root.json: -------------------------------------------------------------------------------- 1 | { 2 | "pub_keys": [ 3 | { 4 | "pubkey": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/public_key.pem" 5 | } 6 | ], 7 | "hosts": [ 8 | "https://storage.app.takoyaki3.com/", 9 | "https://butter.oozora283.com/" 10 | ], 11 | "original_data": { 12 | "host": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/v0.0.0/originalData/" 13 | }, 14 | "api_endpoints": [ 15 | "https://butter.hatano-yuuta7921.workers.dev/" 16 | ], 17 | "api_data_hosts": [ 18 | "https://storage.app.takoyaki3.com/" 19 | ], 20 | "last_update": "2023-05-06T19_02_00+09_00" 21 | } -------------------------------------------------------------------------------- /timetable-app/public/v0.0.0/root.json: -------------------------------------------------------------------------------- 1 | { 2 | "pub_keys": [ 3 | { 4 | "pubkey": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/public_key.pem" 5 | } 6 | ], 7 | "hosts": [ 8 | "https://storage.app.takoyaki3.com/", 9 | "https://butter.oozora283.com/" 10 | ], 11 | "original_data": { 12 | "host": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/v0.0.0/originalData/" 13 | }, 14 | "api_endpoints": [ 15 | "https://butter.hatano-yuuta7921.workers.dev/" 16 | ], 17 | "api_data_hosts": [ 18 | "https://storage.app.takoyaki3.com/" 19 | ], 20 | "last_update": "2023-05-06T19_02_00+09_00" 21 | } -------------------------------------------------------------------------------- /cmd/4_unziper.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import shutil 4 | import zipfile 5 | 6 | files = glob.glob("./feed/*") 7 | # os.mkdir("dir_out") 8 | for file in files: 9 | print(file) 10 | # zipの検査 11 | with zipfile.ZipFile(file, 'r')as zf: 12 | t = zf.testzip() 13 | print(t) # data/sample1.txt 14 | err = shutil.unpack_archive(file, 'feed_dir_out/'+os.path.basename(file)) 15 | print(err) 16 | 17 | files = glob.glob("./gtfs/*") 18 | # os.mkdir("dir_out") 19 | for file in files: 20 | print(file) 21 | # zipの検査 22 | with zipfile.ZipFile(file, 'r')as zf: 23 | t = zf.testzip() 24 | print(t) # data/sample1.txt 25 | err = shutil.unpack_archive(file, 'dir_out/'+os.path.basename(file)) 26 | print(err) 27 | -------------------------------------------------------------------------------- /lib/webpack.config.js: -------------------------------------------------------------------------------- 1 | // webpack.config.js 2 | 3 | const path = require('path') 4 | const webpack = require('webpack') 5 | 6 | module.exports = { 7 | entry: './src/fetch.js', 8 | output: { 9 | filename: 'dist.js', 10 | path: path.resolve(__dirname), 11 | library: { 12 | // name: 'Butter', 13 | type: 'module' 14 | } 15 | }, 16 | experiments: { 17 | outputModule: true 18 | }, 19 | resolve: { 20 | fallback: { 21 | crypto: require.resolve('crypto-browserify'), 22 | buffer: require.resolve('buffer/'), 23 | stream: require.resolve('stream-browserify') 24 | } 25 | }, 26 | plugins: [ 27 | new webpack.ProvidePlugin({ 28 | Buffer: ['buffer', 'Buffer'], 29 | process: 'process/browser' 30 | }) 31 | ], 32 | mode: 'production' 33 | } 34 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "scripts": { 3 | "build": "node build.js", 4 | "test": "jest", 5 | "lint": "eslint . --fix" 6 | }, 7 | "dependencies": { 8 | "babel-jest": "^29.7.0", 9 | "butter-lib": "^1.0.10", 10 | "eslint": "^8.56.0", 11 | "jest": "^29.7.0", 12 | "jest-environment-jsdom": "^29.7.0" 13 | }, 14 | "devDependencies": { 15 | "@babel/core": "^7.23.6", 16 | "@babel/preset-env": "^7.23.6", 17 | "babel-jest": "^29.7.0", 18 | "eslint": "^8.56.0", 19 | "eslint-config-standard": "^17.1.0", 20 | "eslint-plugin-import": "^2.29.1", 21 | "eslint-plugin-n": "^16.5.0", 22 | "eslint-plugin-promise": "^6.1.1", 23 | "jest": "^29.7.0", 24 | "jest-environment-jsdom": "^29.7.0", 25 | "webpack": "^5.82.1", 26 | "webpack-cli": "^5.1.1" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /timetable-app/src/App.vue: -------------------------------------------------------------------------------- 1 | 21 | 22 | 37 | -------------------------------------------------------------------------------- /api/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const webpack = require('webpack'); 3 | 4 | module.exports = { 5 | mode: 'production', 6 | entry: './src/index.js', 7 | output: { 8 | path: path.resolve(__dirname, 'dist'), 9 | filename: 'worker.js', 10 | }, 11 | module: { 12 | rules: [ 13 | { 14 | test: /\.js$/, 15 | exclude: /node_modules/, 16 | use: { 17 | loader: 'babel-loader', 18 | options: { 19 | presets: ['@babel/preset-env'], 20 | }, 21 | }, 22 | }, 23 | ], 24 | }, 25 | target: 'webworker', 26 | optimization: { 27 | minimize: false, // Cloudflare Workersでは、minificationはサポートされていません。 28 | }, 29 | resolve: { 30 | fallback: { 31 | "path": require.resolve("path-browserify") 32 | } 33 | } 34 | }; 35 | -------------------------------------------------------------------------------- /api/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "butter", 3 | "version": "0.0.0", 4 | "devDependencies": { 5 | "@babel/core": "^7.21.4", 6 | "@babel/preset-env": "^7.21.4", 7 | "babel-loader": "^9.1.2", 8 | "jest": "^29.5.0", 9 | "webpack": "^5.80.0", 10 | "webpack-cli": "^5.0.2", 11 | "wrangler": "2.16.0" 12 | }, 13 | "private": true, 14 | "scripts": { 15 | "start": "webpack && wrangler dev", 16 | "deploy": "wrangler publish", 17 | "test": "jest", 18 | "build": "webpack" 19 | }, 20 | "dependencies": { 21 | "bus-time-table-by-edge-runtime": "^1.0.1", 22 | "butter-lib": "^1.0.19", 23 | "h3-js": "^4.1.0", 24 | "haversine": "^1.1.1", 25 | "pako": "^2.1.0", 26 | "path-browserify": "^1.0.1", 27 | "redoc-cli": "^0.13.21" 28 | }, 29 | "devtool": "cheap-module-source-map", 30 | "main": "dist/index.js" 31 | } 32 | -------------------------------------------------------------------------------- /cmd/subdirectory_fix.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import os 3 | import shutil 4 | import zipfile 5 | 6 | 7 | def move_single_subdir_contents_to_parent_dir(directory_path): 8 | # 指定したディレクトリに含まれるファイル/フォルダの一覧を取得 9 | contents = os.listdir(directory_path) 10 | 11 | # 指定したディレクトリに含まれるフォルダの一覧を取得 12 | subdirs = [content for content in contents if os.path.isdir(os.path.join(directory_path, content))] 13 | 14 | print(contents) 15 | 16 | # 指定したディレクトリに含まれるフォルダの数が1つである場合 17 | if len(subdirs) == 1: 18 | subdir_path = os.path.join(directory_path, subdirs[0]) 19 | # フォルダ内の全てのファイル/フォルダを、親フォルダに移動 20 | for content in os.listdir(subdir_path): 21 | shutil.move(os.path.join(subdir_path, content), directory_path) 22 | # 空になったフォルダを削除 23 | os.rmdir(subdir_path) 24 | 25 | files = glob.glob("./dir_out/*") 26 | for file in files: 27 | if os.path.isdir(file): 28 | move_single_subdir_contents_to_parent_dir(file) 29 | -------------------------------------------------------------------------------- /cmd/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use the official Python image as base image 2 | FROM python:3.9-slim 3 | 4 | # Install Golang 5 | ENV GOLANG_VERSION 1.19 6 | RUN apt-get update && \ 7 | apt-get install -y --no-install-recommends wget gcc && \ 8 | wget https://golang.org/dl/go${GOLANG_VERSION}.linux-amd64.tar.gz && \ 9 | tar -C /usr/local -xzf go${GOLANG_VERSION}.linux-amd64.tar.gz && \ 10 | rm go${GOLANG_VERSION}.linux-amd64.tar.gz && \ 11 | apt-get remove -y wget && \ 12 | apt-get autoremove -y && \ 13 | rm -rf /var/lib/apt/lists/* 14 | ENV PATH $PATH:/usr/local/go/bin 15 | 16 | # Set the working directory 17 | WORKDIR /app 18 | 19 | RUN apt-get update && apt-get install -y build-essential 20 | 21 | # Copy requirements file 22 | COPY requirements.txt requirements.txt 23 | COPY helper /app/helper 24 | 25 | # Install required packages 26 | RUN pip install --trusted-host pypi.python.org -r requirements.txt 27 | 28 | # Run the script 29 | CMD while true; do bash ./cmd.sh; sleep 604800; done 30 | -------------------------------------------------------------------------------- /cmd/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/takoyaki-3/butter/cmd 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/aws/aws-sdk-go v1.44.61 // indirect 7 | github.com/cheggaaa/pb v1.0.29 // indirect 8 | github.com/golang/protobuf v1.5.2 // indirect 9 | github.com/jmespath/go-jmespath v0.4.0 // indirect 10 | github.com/mattn/go-runewidth v0.0.4 // indirect 11 | github.com/qedus/osmpbf v1.2.0 // indirect 12 | github.com/takoyaki-3/go-csv-tag/v3 v3.0.2 // indirect 13 | github.com/takoyaki-3/go-file-tool v0.0.1 // indirect 14 | github.com/takoyaki-3/go-geojson v0.0.1 // indirect 15 | github.com/takoyaki-3/go-gtfs/v2 v2.0.7 // indirect 16 | github.com/takoyaki-3/go-json v0.0.2 // indirect 17 | github.com/takoyaki-3/go-map/v2 v2.0.3 // indirect 18 | github.com/takoyaki-3/go-s3 v0.0.9 // indirect 19 | github.com/takoyaki-3/goc v0.0.1 // indirect 20 | github.com/uber/h3-go v3.0.1+incompatible // indirect 21 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect 22 | google.golang.org/protobuf v1.26.0 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2023 BuTTER Team 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /tag-maker/src/App.vue: -------------------------------------------------------------------------------- 1 | 25 | 26 | 41 | -------------------------------------------------------------------------------- /cmd/helper/file.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "crypto/sha256" 7 | "encoding/hex" 8 | filetool "github.com/takoyaki-3/go-file-tool" 9 | ) 10 | 11 | func CopyDir(srcDir, dstDir string) error { 12 | 13 | os.MkdirAll(dstDir, 0777) 14 | 15 | err, files := filetool.DirWalk(srcDir, filetool.DirWalkOption{}) 16 | if err != nil { 17 | return err 18 | } 19 | for _, file := range files { 20 | if file.IsDir { 21 | continue 22 | } 23 | err := Copy(file.Path, dstDir+"/"+file.Name) 24 | if err != nil { 25 | return err 26 | } 27 | } 28 | return nil 29 | } 30 | 31 | func Copy(srcPath, dstPath string) error { 32 | src, err := os.Open(srcPath) 33 | if err != nil { 34 | return err 35 | } 36 | defer src.Close() 37 | dst, err := os.Create(dstPath) 38 | if err != nil { 39 | return err 40 | } 41 | defer dst.Close() 42 | _, err = io.Copy(dst, src) 43 | if err != nil { 44 | return err 45 | } 46 | return err 47 | } 48 | 49 | func GetBinaryBySHA256(s string) string { 50 | r := sha256.Sum256([]byte(s)) 51 | return hex.EncodeToString(r[:]) 52 | } 53 | 54 | func FileName2IntegratedFileName(s string) string { 55 | return GetBinaryBySHA256(s) 56 | } 57 | -------------------------------------------------------------------------------- /tag/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: 'Roboto', sans-serif; 3 | background-color: #f5f5f5; 4 | margin: 0; 5 | padding: 0; 6 | } 7 | 8 | .butter-tag { 9 | margin: 20px; 10 | padding: 20px; 11 | background: #ffffff; 12 | box-shadow: 0px 2px 4px rgba(0, 0, 0, 0.1); 13 | border-radius: 4px; 14 | } 15 | 16 | .is_first_bus { 17 | margin-right: 20px; /* 右側のマージンで横間隔を設定 */ 18 | } 19 | 20 | .date { 21 | display: block; 22 | margin: 10px 0; 23 | padding: 10px; 24 | background: #f0f0f0; 25 | border: none; 26 | border-radius: 4px; 27 | font-size: 16px; 28 | } 29 | 30 | .info { 31 | padding: 10px; 32 | font-size: 14px; 33 | color: #666; 34 | } 35 | 36 | select { 37 | margin: 10px 0; 38 | padding: 10px; 39 | border: none; 40 | background: #f0f0f0; 41 | border-radius: 4px; 42 | font-size: 16px; 43 | } 44 | 45 | .card { 46 | background-color: #ffffff; 47 | border-radius: 4px; 48 | margin: 3px 0; 49 | padding: 3px; 50 | box-shadow: 0px 2px 4px rgba(0, 0, 0, 0.1); 51 | text-align: left; 52 | } 53 | 54 | .box { 55 | text-align: left; 56 | font-size: 16px; 57 | color: #333; 58 | } 59 | 60 | h1 { 61 | color: #333; 62 | text-align: center; 63 | padding: 20px; 64 | } 65 | -------------------------------------------------------------------------------- /api/readme.md: -------------------------------------------------------------------------------- 1 | # Cloudflare Worker Example with Webpack 2 | 3 | このプロジェクトは、Webpack を使用して Cloudflare Workers のコードをビルドし、デプロイする方法を示すサンプルです。この Worker は、HTTPリクエストを受け取り、簡単なメッセージを返します。 4 | 5 | ## 前提条件 6 | 7 | - [Node.js](https://nodejs.org/en) がインストールされていること 8 | - [Cloudflare アカウント](https://dash.cloudflare.com/sign-up) が作成されていること 9 | - [Wrangler CLI](https://developers.cloudflare.com/workers/wrangler/install-and-update/) がインストールおよび設定されていること 10 | 11 | ## セットアップ 12 | プロジェクトをクローンまたはダウンロードします。 13 | 14 | ```bash 15 | git clone https://github.com/your_username/cloudflare-worker-example.git 16 | cd cloudflare-worker-example 17 | ``` 18 | 19 | 依存関係をインストールします。 20 | 21 | ```bash 22 | npm install 23 | ``` 24 | 25 | ## 開発 26 | ローカルで開発を行い、変更をプレビューするには、以下のコマンドを使用します。 27 | 28 | ```bash 29 | npm run preview 30 | ``` 31 | 32 | これにより、Cloudflare Workers のプレビューエンドポイントが開きます。 33 | 34 | ## ビルド 35 | プロジェクトをビルドするには、以下のコマンドを実行します。 36 | 37 | ```bash 38 | npm run build 39 | ``` 40 | これにより、dist ディレクトリに worker.js が生成されます。 41 | 42 | ## デプロイ 43 | プロジェクトを Cloudflare Workers にデプロイするには、以下のコマンドを実行します。 44 | 45 | ```bash 46 | npm run publish 47 | ``` 48 | 49 | ``` 50 | redoc-cli bundle .\api.yml 51 | ``` 52 | 53 | これにより、指定したルート（この例では example.com/*）で動作するように Worker がデプロイされます。 54 | 55 | ## ライセンス 56 | このプロジェクトは、MITライセンス の下でリリースされています。 57 | -------------------------------------------------------------------------------- /about/src/App.vue: -------------------------------------------------------------------------------- 1 | 37 | 38 | 53 | -------------------------------------------------------------------------------- /timetable-app/public/sw.js: -------------------------------------------------------------------------------- 1 | const CACHE_NAME = 'butter-timetable-app-cache-v1'; 2 | const urlsToCache = []; 3 | 4 | // Listen for push events 5 | self.addEventListener('push', function(event) { 6 | let data = {}; 7 | if (event.data) { 8 | data = event.data.json(); 9 | } 10 | 11 | const options = { 12 | body: data.message, 13 | data: { url: data.url } 14 | }; 15 | 16 | event.waitUntil( 17 | self.registration.showNotification(data.title, options) 18 | ); 19 | }); 20 | 21 | self.addEventListener('install', event => { 22 | event.waitUntil( 23 | caches.open(CACHE_NAME) 24 | .then(cache => { 25 | console.log('Opened cache'); 26 | return cache.addAll(urlsToCache); 27 | }) 28 | ); 29 | }); 30 | 31 | self.addEventListener('fetch', event => { 32 | event.respondWith( 33 | caches.match(event.request) 34 | .then(response => { 35 | if (response) { 36 | return response; 37 | } 38 | return fetch(event.request); 39 | }) 40 | ); 41 | }); 42 | 43 | // Listen for notification click events 44 | self.addEventListener('notificationclick', function(event) { 45 | event.notification.close(); 46 | 47 | // Open the app and focus on the window 48 | if (event.notification.data.url) { 49 | event.waitUntil( 50 | clients.openWindow(event.notification.data.url) 51 | ); 52 | } 53 | }); 54 | -------------------------------------------------------------------------------- /storage/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | file-server-v0: 4 | image: golang:1.19 5 | volumes: 6 | - ./:/app 7 | working_dir: /app 8 | command: go run fileServer.go 9 | ports: 10 | - "${FILE_SERVER_V0_PORT:-8000}:8000" 11 | restart: unless-stopped 12 | 13 | file-server-v1: 14 | image: golang:1.19 15 | volumes: 16 | - ./:/app 17 | working_dir: /app 18 | command: go run fileServer-v1.go 19 | ports: 20 | - "${FILE_SERVER_V1_PORT:-8001}:8001" 21 | restart: unless-stopped 22 | 23 | data-updater-v0: 24 | image: golang:1.19 25 | volumes: 26 | - ./:/app 27 | working_dir: /app 28 | command: go run dataUpdater.go 29 | restart: unless-stopped 30 | 31 | data-updater-v1: 32 | image: golang:1.19 33 | volumes: 34 | - ./:/app 35 | working_dir: /app 36 | command: go run dataUpdater-v1.go 37 | restart: unless-stopped 38 | 39 | server-updater-v0-v1: 40 | image: golang:1.19 41 | volumes: 42 | - ./:/app 43 | working_dir: /app 44 | command: go run serverUpdater.go 45 | restart: unless-stopped 46 | 47 | # caddy: 48 | # image: caddy:2 49 | # volumes: 50 | # - ./Caddyfile:/etc/caddy/Caddyfile 51 | # - ./caddy_data:/data 52 | # - ./caddy_config:/config 53 | # ports: 54 | # - "80:80" 55 | # - "443:443" 56 | # restart: unless-stopped 57 | -------------------------------------------------------------------------------- /storage/go.md: -------------------------------------------------------------------------------- 1 | BuTTER
-------------------------------------------------------------------------------- /cmd/3_downloader.py: -------------------------------------------------------------------------------- 1 | import json 2 | import urllib.error 3 | import urllib.request 4 | import time 5 | import os 6 | 7 | json_open = open('data_v1.json', 'r', encoding='utf-8') 8 | datalist = json.load(json_open) 9 | 10 | for item in datalist: 11 | if 'gtfs_id' in item: 12 | print(item['gtfs_id']) 13 | gtfs_id = item['gtfs_id'] 14 | url = item['GTFS_url'] 15 | print(url) 16 | 17 | save_path = './feed/'+gtfs_id+'.zip' 18 | if not os.path.isdir('./feed'): 19 | os.makedirs('./feed') 20 | 21 | try: 22 | with urllib.request.urlopen(url) as download_file: 23 | data = download_file.read() 24 | with open(save_path, mode='wb') as save_file: 25 | save_file.write(data) 26 | except urllib.error.URLError as e: 27 | print(e) 28 | 29 | time.sleep(1) 30 | 31 | if not os.path.isdir('./gtfs'): 32 | os.makedirs('./gtfs') 33 | 34 | for item in datalist: 35 | if 'gtfs_id' not in item: 36 | continue 37 | print(item) 38 | gtfs_id = item['gtfs_id'] 39 | organization_id = item['organization_id'] 40 | url = item['GTFS_url'] 41 | 42 | origin_path = './feed/'+gtfs_id+'.zip' 43 | dest_path = './gtfs/'+organization_id+'.zip' 44 | 45 | # copy origin_path to dest_path 46 | with open(origin_path, 'rb') as origin_file: 47 | with open(dest_path, 'wb') as dest_file: 48 | dest_file.write(origin_file.read()) 49 | -------------------------------------------------------------------------------- /lib/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "butter-lib", 3 | "version": "1.1.0", 4 | "description": "", 5 | "main": "webpack.config.js", 6 | "scripts": { 7 | "test": "jest", 8 | "lint": "eslint . --fix", 9 | "build": "webpack" 10 | }, 11 | "keywords": [ 12 | "GTFS", 13 | "Bus", 14 | "Timetable" 15 | ], 16 | "author": "pfpfdev(https://github.com/pfpfdev),takoyaki3(https://github.com/takoyaki-3/)", 17 | "license": "MIT", 18 | "bugs": { 19 | "url": "https://github.com/takoyaki-3/butter/issues" 20 | }, 21 | "homepage": "https://butter.takoyaki3.com", 22 | "devDependencies": { 23 | "@babel/core": "^7.23.6", 24 | "@babel/preset-env": "^7.23.6", 25 | "babel-jest": "^29.7.0", 26 | "eslint": "^8.56.0", 27 | "eslint-config-standard": "^17.1.0", 28 | "jest": "^29.7.0", 29 | "jest-environment-jsdom": "^29.7.0", 30 | "jest-fetch-mock": "^3.0.3", 31 | "webpack": "^5.82.1", 32 | "webpack-cli": "^5.1.1" 33 | }, 34 | "dependencies": { 35 | "@peculiar/webcrypto": "^1.4.3", 36 | "axios": "^1.6.3", 37 | "buffer": "^6.0.3", 38 | "crypto": "^1.0.1", 39 | "crypto-browserify": "^3.12.0", 40 | "csv-parser": "^3.0.0", 41 | "google-protobuf": "^3.21.2", 42 | "h3-js": "^4.1.0", 43 | "haversine": "^1.1.1", 44 | "node-fetch": "^3.3.2", 45 | "pako": "^2.1.0", 46 | "process": "^0.11.10", 47 | "protobufjs": "^7.2.5", 48 | "stream-browserify": "^3.0.0" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /cmd/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | BuTTER ストレージ追加ツール 5 | 6 | 7 |

BuTTER ストレージ追加ツール

8 | 9 |
10 | 11 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /about/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | <%= htmlWebpackPlugin.options.title %> 11 | 12 | 13 | 14 | 15 | 16 | 17 | 22 | 23 | 24 | 25 | 32 | 33 | 34 | 38 |
39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /tag-maker/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | <%= htmlWebpackPlugin.options.title %> 11 | 12 | 13 | 14 | 15 | 16 | 17 | 22 | 23 | 24 | 25 | 32 | 33 | 34 | 38 |
39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /about/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BuTTER", 3 | "version": "1.0.0", 4 | "private": true, 5 | "description": "", 6 | "author": "", 7 | "scripts": { 8 | "serve": "vue-cli-service serve", 9 | "build": "vue-cli-service build", 10 | "lint": "vue-cli-service lint", 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "main": "index.js", 14 | "dependencies": { 15 | "butter-lib": "^1.1.0", 16 | "core-js": "^3.8.3", 17 | "highlight.js": "^11.8.0", 18 | "leaflet": "^1.9.4", 19 | "vue": "^2.6.14", 20 | "vue2-leaflet": "^2.7.1", 21 | "vuetify": "^2.6.0" 22 | }, 23 | "devDependencies": { 24 | "@babel/core": "^7.12.16", 25 | "@babel/eslint-parser": "^7.12.16", 26 | "@vue/cli-plugin-babel": "~5.0.0", 27 | "@vue/cli-plugin-eslint": "~5.0.0", 28 | "@vue/cli-service": "~5.0.0", 29 | "eslint": "^7.32.0", 30 | "eslint-plugin-vue": "^8.0.3", 31 | "sass": "~1.32.0", 32 | "sass-loader": "^10.0.0", 33 | "vue-cli-plugin-vuetify": "~2.5.8", 34 | "vue-template-compiler": "^2.6.14", 35 | "vuetify-loader": "^1.7.0" 36 | }, 37 | "eslintConfig": { 38 | "root": true, 39 | "env": { 40 | "node": true 41 | }, 42 | "extends": [ 43 | "plugin:vue/essential", 44 | "eslint:recommended" 45 | ], 46 | "parserOptions": { 47 | "parser": "@babel/eslint-parser" 48 | }, 49 | "rules": {} 50 | }, 51 | "browserslist": [ 52 | "> 1%", 53 | "last 2 versions", 54 | "not dead" 55 | ], 56 | "_id": "vuetify-dev@1.0.0", 57 | "keywords": [], 58 | "license": "ISC", 59 | "readme": "ERROR: No README data found!" 60 | } 61 | -------------------------------------------------------------------------------- /timetable-app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BuTTER", 3 | "version": "1.0.0", 4 | "private": true, 5 | "description": "", 6 | "author": "", 7 | "scripts": { 8 | "serve": "vue-cli-service serve", 9 | "build": "vue-cli-service build", 10 | "lint": "vue-cli-service lint", 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "main": "index.js", 14 | "dependencies": { 15 | "butter-lib": "^1.1.0", 16 | "core-js": "^3.8.3", 17 | "highlight.js": "^11.8.0", 18 | "leaflet": "^1.9.4", 19 | "vue": "^2.6.14", 20 | "vue2-leaflet": "^2.7.1", 21 | "vuetify": "^2.6.0" 22 | }, 23 | "devDependencies": { 24 | "@babel/core": "^7.12.16", 25 | "@babel/eslint-parser": "^7.12.16", 26 | "@vue/cli-plugin-babel": "~5.0.0", 27 | "@vue/cli-plugin-eslint": "~5.0.0", 28 | "@vue/cli-service": "~5.0.0", 29 | "eslint": "^7.32.0", 30 | "eslint-plugin-vue": "^8.0.3", 31 | "sass": "~1.32.0", 32 | "sass-loader": "^10.0.0", 33 | "vue-cli-plugin-vuetify": "~2.5.8", 34 | "vue-template-compiler": "^2.6.14", 35 | "vuetify-loader": "^1.7.0" 36 | }, 37 | "eslintConfig": { 38 | "root": true, 39 | "env": { 40 | "node": true 41 | }, 42 | "extends": [ 43 | "plugin:vue/essential", 44 | "eslint:recommended" 45 | ], 46 | "parserOptions": { 47 | "parser": "@babel/eslint-parser" 48 | }, 49 | "rules": {} 50 | }, 51 | "browserslist": [ 52 | "> 1%", 53 | "last 2 versions", 54 | "not dead" 55 | ], 56 | "_id": "vuetify-dev@1.0.0", 57 | "keywords": [], 58 | "license": "ISC", 59 | "readme": "ERROR: No README data found!" 60 | } 61 | -------------------------------------------------------------------------------- /timetable-app/public/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | <%= htmlWebpackPlugin.options.title %> 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 23 | 24 | 25 | 26 | 33 | 34 | 35 | 39 |
40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /timetable-app/src/components/SampleCode.vue: -------------------------------------------------------------------------------- 1 | 43 | 44 | 51 | -------------------------------------------------------------------------------- /tag-maker/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BuTTER", 3 | "version": "1.0.0", 4 | "private": true, 5 | "description": "", 6 | "author": "", 7 | "scripts": { 8 | "serve": "vue-cli-service serve", 9 | "build": "vue-cli-service build", 10 | "lint": "vue-cli-service lint", 11 | "test": "echo \"Error: no test specified\" && exit 1" 12 | }, 13 | "main": "index.js", 14 | "dependencies": { 15 | "@emotion/react": "^11.11.1", 16 | "@emotion/styled": "^11.11.0", 17 | "@mui/material": "^5.14.14", 18 | "butter-lib": "^1.1.0", 19 | "ci": "^2.3.0", 20 | "core-js": "^3.8.3", 21 | "highlight.js": "^11.8.0", 22 | "leaflet": "^1.9.4", 23 | "vue": "^2.6.14", 24 | "vue2-leaflet": "^2.7.1", 25 | "vuetify": "^2.6.0" 26 | }, 27 | "devDependencies": { 28 | "@babel/core": "^7.12.16", 29 | "@babel/eslint-parser": "^7.12.16", 30 | "@vue/cli-plugin-babel": "~5.0.0", 31 | "@vue/cli-plugin-eslint": "~5.0.0", 32 | "@vue/cli-service": "~5.0.0", 33 | "eslint": "^7.32.0", 34 | "eslint-plugin-vue": "^8.0.3", 35 | "sass": "~1.32.0", 36 | "sass-loader": "^10.0.0", 37 | "vue-cli-plugin-vuetify": "~2.5.8", 38 | "vue-template-compiler": "^2.6.14", 39 | "vuetify-loader": "^1.7.0" 40 | }, 41 | "eslintConfig": { 42 | "root": true, 43 | "env": { 44 | "node": true 45 | }, 46 | "extends": [ 47 | "plugin:vue/essential", 48 | "eslint:recommended" 49 | ], 50 | "parserOptions": { 51 | "parser": "@babel/eslint-parser" 52 | }, 53 | "rules": {} 54 | }, 55 | "browserslist": [ 56 | "> 1%", 57 | "last 2 versions", 58 | "not dead" 59 | ], 60 | "_id": "vuetify-dev@1.0.0", 61 | "keywords": [], 62 | "license": "ISC", 63 | "readme": "ERROR: No README data found!" 64 | } 65 | -------------------------------------------------------------------------------- /cmd/helper/sign.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "io/ioutil" 5 | "crypto" 6 | "crypto/rand" 7 | "crypto/rsa" 8 | "crypto/sha256" 9 | "crypto/x509" 10 | "encoding/pem" 11 | 12 | filetool "github.com/takoyaki-3/go-file-tool" 13 | ) 14 | 15 | func Sing(dataBytes, privateKeyBytes []byte) ([]byte, error) { 16 | privateKeyBlock, _ := pem.Decode(privateKeyBytes) 17 | 18 | privateKey, err := x509.ParsePKCS1PrivateKey(privateKeyBlock.Bytes) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | // ファイルのハッシュ値を計算する 24 | hash := sha256.Sum256(dataBytes) 25 | 26 | // ハッシュ値に署名する 27 | signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, crypto.SHA256, hash[:]) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | return signature, nil 33 | } 34 | 35 | func AddSing(path string, privateKeyBytes []byte) error { 36 | // ファイルを読み込む 37 | file, err := ioutil.ReadFile(path) 38 | if err != nil { 39 | return err 40 | } 41 | signature, err := Sing(file, privateKeyBytes) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | // 署名をファイルに書き込む 47 | err = ioutil.WriteFile(path+".sig", signature, 0644) 48 | return err 49 | } 50 | 51 | func AddDirfileSing(dirPath string, privateKeyBytes []byte) error { 52 | err, files := filetool.DirWalk(dirPath, filetool.DirWalkOption{}) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | for _, item := range files { 58 | if item.IsDir { 59 | continue 60 | } 61 | // ファイルを読み込む 62 | file, err := ioutil.ReadFile(item.Path) 63 | if err != nil { 64 | return err 65 | } 66 | signature, err := Sing(file, privateKeyBytes) 67 | if err != nil { 68 | return err 69 | } 70 | 71 | // 署名をファイルに書き込む 72 | err = ioutil.WriteFile(item.Path+".sig", signature, 0644) 73 | if err != nil { 74 | return err 75 | } 76 | } 77 | 78 | return err 79 | } 80 | -------------------------------------------------------------------------------- /cmd/6_add_gtfs_id_info.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "path/filepath" 7 | 8 | json "github.com/takoyaki-3/go-json" 9 | ) 10 | 11 | type InfoType struct { 12 | VersionID string `json:"version_id"` 13 | ByStopHashValueSize int `json:"by_stop_hash_value_size"` 14 | ByTripHashValueSize int `json:"by_trip_hash_value_size"` 15 | } 16 | 17 | func main() { 18 | 19 | // Set the root directory path 20 | moveInfoFile("v0.0.0") 21 | moveInfoFile("v1.0.0") 22 | } 23 | 24 | func moveInfoFile(root string) { 25 | // Get the list of directories in the root directory 26 | dirs, err := ioutil.ReadDir(root) 27 | if err != nil { 28 | fmt.Println(err) 29 | return 30 | } 31 | 32 | // Loop through the directories in the root directory 33 | for _, dir := range dirs { 34 | // Check if the item is a directory 35 | if dir.IsDir() { 36 | gtfsID := dir.Name() 37 | 38 | // Get the list of subdirectories in the directory 39 | subdirs, err := ioutil.ReadDir(filepath.Join(root, dir.Name())) 40 | if err != nil { 41 | fmt.Println(err) 42 | continue 43 | } 44 | 45 | versionInfo := []InfoType{} 46 | 47 | // Loop through the subdirectories in the directory 48 | for _, subdir := range subdirs { 49 | // Check if the item is a directory 50 | if subdir.IsDir() { 51 | // Print the subdirectory name and full path 52 | dirPath := filepath.Join(root, dir.Name(), subdir.Name()) 53 | fmt.Printf(" Found subdirectory: %s (%s) %s\n", subdir.Name(), gtfsID, dirPath) 54 | 55 | // JSON Load 56 | info := InfoType{} 57 | json.LoadFromPath(dirPath+"/info.json", &info) 58 | info.VersionID = subdir.Name() 59 | versionInfo = append(versionInfo, info) 60 | fmt.Println(info) 61 | } 62 | } 63 | 64 | json.DumpToFile(versionInfo, "./"+root+"/"+dir.Name()+"/info.json") 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /lib/test2.html: -------------------------------------------------------------------------------- 1 | 2 | 63 | -------------------------------------------------------------------------------- /about/src/components/SampleCode.vue: -------------------------------------------------------------------------------- 1 | 43 | 44 | 51 | -------------------------------------------------------------------------------- /cmd/9_addSign.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "sync" 10 | 11 | "github.com/takoyaki-3/butter/cmd/helper" 12 | ) 13 | 14 | // Worker function to sign files 15 | func signWorker(files <-chan string, privateKeyData []byte, wg *sync.WaitGroup) { 16 | defer wg.Done() 17 | for path := range files { 18 | err := helper.AddSing(path, privateKeyData) 19 | if err != nil { 20 | log.Printf("Error signing file %s: %v", path, err) 21 | } 22 | } 23 | } 24 | 25 | func SignAllFilesInDir(directory string, privateKeyPath string, workerCount int) error { 26 | privateKeyData, err := ioutil.ReadFile(privateKeyPath) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | files := make(chan string, 100) // Buffer size of 100 for file paths 32 | var wg sync.WaitGroup 33 | 34 | // Start worker goroutines 35 | for i := 0; i < workerCount; i++ { 36 | wg.Add(1) 37 | go signWorker(files, privateKeyData, &wg) 38 | } 39 | 40 | err = filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { 41 | if err != nil { 42 | return err 43 | } 44 | 45 | // Skip directories 46 | if info.IsDir() { 47 | return nil 48 | } 49 | 50 | // Skip already signed files 51 | if filepath.Ext(path) == ".sig" { 52 | return nil 53 | } 54 | 55 | // Send file path to workers 56 | files <- path 57 | return nil 58 | }) 59 | 60 | if err != nil { 61 | close(files) 62 | wg.Wait() 63 | return err 64 | } 65 | 66 | // Close the channel and wait for all workers to finish 67 | close(files) 68 | wg.Wait() 69 | 70 | return nil 71 | } 72 | 73 | func main() { 74 | fmt.Println("start sign") 75 | workerCount := 10 // Number of parallel workers 76 | err := SignAllFilesInDir("./v0.0.0/", "key.pem", workerCount) 77 | if err != nil { 78 | log.Fatalln(err) 79 | } 80 | err = SignAllFilesInDir("./v1.0.0/", "key.pem", workerCount) 81 | if err != nil { 82 | log.Fatalln(err) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /lib/ds.txt: -------------------------------------------------------------------------------- 1 | ├── 2023-04-08T20_51_45+09_00 2 | │ ├── GTFS 3 | │ │ ├── agency.txt 4 | │ │ ├── agency.txt.sig 5 | │ │ ├── calendar.txt 6 | │ │ ├── calendar.txt.sig 7 | │ │ ├── calendar_dates.txt 8 | │ │ ├── calendar_dates.txt.sig 9 | │ │ ├── fare_attributes.txt 10 | │ │ ├── fare_attributes.txt.sig 11 | │ │ ├── fare_rules.txt 12 | │ │ ├── fare_rules.txt.sig 13 | │ │ ├── feed_info.txt 14 | │ │ ├── feed_info.txt.sig 15 | │ │ ├── office_jp.txt 16 | │ │ ├── office_jp.txt.sig 17 | │ │ ├── routes.txt 18 | │ │ ├── routes.txt.sig 19 | │ │ ├── shapes.txt 20 | │ │ ├── shapes.txt.sig 21 | │ │ ├── stop_times.txt 22 | │ │ ├── stop_times.txt.sig 23 | │ │ ├── stops.txt 24 | │ │ ├── stops.txt.sig 25 | │ │ ├── transfers.txt 26 | │ │ ├── transfers.txt.sig 27 | │ │ ├── translations.txt 28 | │ │ ├── translations.txt.sig 29 | │ │ ├── trips.txt 30 | │ │ └── trips.txt.sig 31 | │ ├── byStops 32 | │ │ ├── 0.tar.gz 33 | │ │ ├── 1.tar.gz 34 | │ │ ├── 2.tar.gz 35 | │ │ ├── 3.tar.gz 36 | │ │ ├── 4.tar.gz 37 | │ │ ├── 5.tar.gz 38 | │ │ ├── 6.tar.gz 39 | │ │ ├── 7.tar.gz 40 | │ │ ├── 8.tar.gz 41 | │ │ ├── 9.tar.gz 42 | │ │ ├── a.tar.gz 43 | │ │ ├── b.tar.gz 44 | │ │ ├── c.tar.gz 45 | │ │ ├── d.tar.gz 46 | │ │ ├── e.tar.gz 47 | │ │ └── f.tar.gz 48 | │ ├── byTrips 49 | │ │ ├── 0.tar.gz 50 | │ │ ├── 1.tar.gz 51 | │ │ ├── 2.tar.gz 52 | │ │ ├── 3.tar.gz 53 | │ │ ├── 4.tar.gz 54 | │ │ ├── 5.tar.gz 55 | │ │ ├── 6.tar.gz 56 | │ │ ├── 7.tar.gz 57 | │ │ ├── 8.tar.gz 58 | │ │ ├── 9.tar.gz 59 | │ │ ├── a.tar.gz 60 | │ │ ├── b.tar.gz 61 | │ │ ├── c.tar.gz 62 | │ │ ├── d.tar.gz 63 | │ │ ├── e.tar.gz 64 | │ │ └── f.tar.gz 65 | │ ├── info.json 66 | │ ├── info.json.sig 67 | │ ├── stops.txt 68 | │ └── stops.txt.sig 69 | ├── info.json 70 | └── info.json.sig -------------------------------------------------------------------------------- /storage/go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 4 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 5 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 6 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 7 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 8 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 9 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 10 | github.com/jlaffaye/ftp v0.1.0 h1:DLGExl5nBoSFoNshAUHwXAezXwXBvFdx7/qwhucWNSE= 11 | github.com/jlaffaye/ftp v0.1.0/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE= 12 | github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= 13 | github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= 14 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 15 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 16 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 17 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 18 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 19 | github.com/takoyaki-3/go-json v0.0.2 h1:IRG20KZUFt0ZnYikI6ob+/8cSjTA41rTaaA5cVAKKLo= 20 | github.com/takoyaki-3/go-json v0.0.2/go.mod h1:39fxS11XWIvygjbgeXg4bpyvAxLZ2lr1Cv+9DbbQ9Ew= 21 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 22 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 23 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 24 | -------------------------------------------------------------------------------- /api/src/index.js: -------------------------------------------------------------------------------- 1 | import Butter from '../../lib/dist' 2 | import { functionParams, functionParamIsJSON } from './functionParams.js'; 3 | 4 | addEventListener('fetch', (event) => { 5 | event.respondWith(handleRequest(event.request)); 6 | }); 7 | 8 | async function handleRequest(request) { 9 | if (request.method !== 'GET') { 10 | return new Response('Only GET requests are allowed', { status: 405 }); 11 | } 12 | 13 | // https://api.butter.takoyaki3.com/getShapes?gtfs_id=ToeiBus 14 | 15 | try { 16 | const url = new URL(request.url); 17 | const functionName = url.pathname.split('/').pop(); 18 | // console.log('functionName', url.searchParams) 19 | const paramsObj = Object.fromEntries(url.searchParams); 20 | // console.log('paramsObj', JSON.stringify(paramsObj)) 21 | // console.log('functionParams', JSON.stringify(functionParams[functionName])) 22 | const params = functionParams[functionName].map((param,index) => (param && functionParamIsJSON[functionName][index]) ? JSON.parse(paramsObj[param]) : paramsObj[param]); 23 | console.log({ functionName, params }); 24 | console.log( JSON.stringify(params)); 25 | 26 | // BuTTERライブラリの初期化 27 | await Butter.init('https://butter.takoyaki3.com/v0.0.0/root.json', { useFetch: true }); 28 | 29 | // 指定されたBuTTER関数の実行 30 | if (functionName in Butter) { 31 | const result = await Butter[functionName](...params); 32 | console.log({ result }); 33 | return new Response(JSON.stringify(result), { 34 | headers: { 'Content-Type': 'application/json' }, 35 | }); 36 | } else { 37 | return new Response(JSON.stringify({ error: 'Function not found' }), { 38 | status: 400, 39 | headers: { 'Content-Type': 'application/json' }, 40 | }); 41 | } 42 | } catch (error) { 43 | console.log(error); 44 | return new Response(JSON.stringify({ error: error.message }), { 45 | status: 500, 46 | headers: { 47 | 'content-type': 'application/json', 48 | 'Access-Control-Allow-Origin': '*', 49 | 'Access-Control-Allow-Methods': 'GET, POST, PUT, DELETE, OPTIONS', 50 | 'Access-Control-Allow-Headers': 'Content-Type', 51 | }, 52 | }); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /lib/readme.md: -------------------------------------------------------------------------------- 1 | ## BuTTER Library 2 | 3 | BuTTER Library は、ストレージ上に細分化した状態で保存されているGTFSを基にした時刻表情報を集め、ブラウザ内で必要な情報に加工するライブラリです。DBを使わずにデータ処理をブラウザ内とする 4 | 5 | ## 関数と利用方法 6 | 7 | 1. `Butter.init()` 8 | この関数は初期化に使用されます。例えば： 9 | ```javascript 10 | import Butter from './dist.js'; 11 | Butter.init() 12 | ``` 13 | 14 | 2. `Butter.getHostDataList()` 15 | この関数はホストのデータリストを取得するために使用されます。例： 16 | ```javascript 17 | const hostData = await Butter.getHostDataList() 18 | console.log(hostData) 19 | ``` 20 | 21 | 3. `Butter.getAgencyInfo(gtfs_id)` 22 | 特定のGTFS IDに関連する機関情報を取得します。例： 23 | ```javascript 24 | const agencyInfo = await Butter.getAgencyInfo("your_gtfs_id") 25 | console.log(agencyInfo) 26 | ``` 27 | 28 | 4. `Butter.getBusStops(gtfs_id, version_id)` 29 | 特定のGTFS IDとバージョンIDに関連するバス停を取得します。例： 30 | ```javascript 31 | const stops = await Butter.getBusStops("your_gtfs_id", "your_version_id") 32 | console.log(stops) 33 | ``` 34 | 35 | 5. `Butter.getTrips(gtfs_id, version_id)` 36 | 特定のGTFS IDとバージョンIDに関連するバスの旅行情報を取得します。例： 37 | ```javascript 38 | const trips = await Butter.getTrips("your_gtfs_id", "your_version_id") 39 | console.log(trips) 40 | ``` 41 | 42 | 6. `Butter.getStopsBySubstring(substring)` 43 | 特定の文字列を含むバス停を取得します。例： 44 | ```javascript 45 | const stops = await Butter.getStopsBySubstring("substring") 46 | console.log(stops) 47 | ``` 48 | 49 | 7. `Butter.getStopsWithinRadius(lat, lon, radius)` 50 | 特定の緯度、経度、半径の範囲内にあるバス停を取得します。例： 51 | ```javascript 52 | const aroundStops = await Butter.getStopsWithinRadius(35.693906, 139.701504, 500) 53 | console.log(aroundStops) 54 | ``` 55 | 56 | 8. `Butter.getBusInfo(lat, lon)` 57 | 特定の緯度、経度に関連するバスの情報を取得します。例： 58 | ```javascript 59 | const busInfo = await Butter.getBusInfo(35.693906, 139.701504) 60 | console.log(busInfo) 61 | ``` 62 | 63 | 9. `Butter.fetchTimeTableV1(gtfs_id, options)` 64 | 特定のGTFS IDとオプションに基づいて時刻表情報を取得します。例： 65 | ```javascript 66 | let tt = await Butter.fetchTimeTableV1("your_gtfs_id", { 67 | date: "20230513", 68 | stop_ids: ["your_stop_id"] 69 | }) 70 | console.log(tt) 71 | ``` 72 | 73 | 10. `Butter.getComsumedOp()` 74 | これまでに消費されたオペレーションの数を取得します。例： 75 | ```javascript 76 | console.log("COMSUMED OPERATIONS ARE", Butter.getComsumedOp()) 77 | ``` 78 | 注意：上記の例では、`"your_gtfs_id"`, `"your_version_id"`, `"substring"`, `"your_stop_id"`などは実際の値に置き換える必要があります。 79 | 80 | ## webpackのbuild方法 81 | 82 | ### 依存モジュールのインストール 83 | 84 | ```sh 85 | npm install 86 | ``` 87 | 88 | ### build 89 | 90 | ```sh 91 | npm run build 92 | ``` 93 | 94 | -------------------------------------------------------------------------------- /cmd/helper/tar.go: -------------------------------------------------------------------------------- 1 | package helper 2 | 3 | import ( 4 | "archive/tar" 5 | "compress/gzip" 6 | "os" 7 | "time" 8 | "io/ioutil" 9 | "strings" 10 | "path/filepath" 11 | ) 12 | 13 | type TarGzWriter struct { 14 | tw *tar.Writer 15 | gw *gzip.Writer 16 | f *os.File 17 | } 18 | 19 | func NewTarGzWriter(filename string) (*TarGzWriter, error) { 20 | f, err := os.Create(filename) 21 | if err != nil { 22 | return nil, err 23 | } 24 | gw := gzip.NewWriter(f) 25 | tw := tar.NewWriter(gw) 26 | return &TarGzWriter{tw: tw, gw: gw, f: f}, nil 27 | } 28 | 29 | func (tgz *TarGzWriter) Close() { 30 | tgz.tw.Close() 31 | tgz.gw.Close() 32 | tgz.f.Close() 33 | } 34 | 35 | func (tgz *TarGzWriter) AddData(path string, data []byte) error { 36 | // Write header 37 | err := tgz.tw.WriteHeader(&tar.Header{ 38 | Name: path, 39 | Mode: int64(777), 40 | ModTime: time.Now().Truncate(24 * time.Hour), 41 | Size: int64(len(data)), 42 | }) 43 | if err != nil { 44 | return err 45 | } 46 | _, err = tgz.tw.Write(data) 47 | if err != nil { 48 | return err 49 | } 50 | return nil 51 | } 52 | 53 | func (tgz *TarGzWriter) AddDataWithSign(path string, data []byte, privateKeyBytes []byte) error { 54 | tgz.AddData(path, data) 55 | 56 | // add signature 57 | signBytes, err := Sing(data, privateKeyBytes) 58 | if err != nil { 59 | return err 60 | } 61 | return tgz.AddData(path + ".sig", signBytes) 62 | } 63 | 64 | // 指定したディレクトリ内のファイルを.tar形式にまとめる 65 | func CreateTarArchive(sourceDir, targetFile string) error { 66 | file, err := os.Create(targetFile) 67 | if err != nil { 68 | return err 69 | } 70 | defer file.Close() 71 | 72 | tarWriter := tar.NewWriter(file) 73 | defer tarWriter.Close() 74 | 75 | return filepath.Walk(sourceDir, func(path string, info os.FileInfo, err error) error { 76 | if err != nil { 77 | return err 78 | } 79 | path = strings.ReplaceAll(path,"\\","/") 80 | 81 | header, err := tar.FileInfoHeader(info, info.Name()) 82 | if err != nil { 83 | return err 84 | } 85 | 86 | header.Name = path 87 | 88 | if err := tarWriter.WriteHeader(header); err != nil { 89 | return err 90 | } 91 | 92 | if info.IsDir() { 93 | return nil 94 | } 95 | 96 | data, err := ioutil.ReadFile(path) 97 | if err != nil { 98 | return err 99 | } 100 | 101 | _, err = tarWriter.Write(data) 102 | return err 103 | }) 104 | } 105 | -------------------------------------------------------------------------------- /storage/fileServer.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "time" 7 | "bytes" 8 | "io/ioutil" 9 | ) 10 | 11 | func monitorFile(filePath, timeFilePath string, done chan bool) { 12 | ticker := time.NewTicker(1 * time.Minute) 13 | var lastFileContent []byte 14 | lastFileContent, _ = ioutil.ReadFile(filePath) 15 | 16 | for { 17 | select { 18 | case <-ticker.C: 19 | fileContent, err := ioutil.ReadFile(filePath) 20 | if err != nil { 21 | log.Printf("Error reading file: %v", err) 22 | continue 23 | } 24 | 25 | if !bytes.Equal(fileContent, lastFileContent) { 26 | log.Printf("File %s has been modified", filePath) 27 | lastFileContent = fileContent 28 | time.AfterFunc(10*time.Second, func() { 29 | done <- true 30 | }) 31 | } 32 | 33 | // 現在の時刻を取得し、ファイルに書き込む 34 | currentTime := time.Now().Format(time.RFC3339) 35 | if err := ioutil.WriteFile(timeFilePath, []byte(currentTime), 0644); err != nil { 36 | log.Printf("Error writing last checked time: %v", err) 37 | } 38 | } 39 | } 40 | } 41 | 42 | func main() { 43 | dir := "./public/v0.0.0" 44 | port := "8000" 45 | 46 | fs := http.FileServer(http.Dir(dir)) 47 | http.HandleFunc("/server", func(w http.ResponseWriter, r *http.Request) { 48 | responseMessage := "{\"version\":\"1.3\"}" 49 | w.Write([]byte(responseMessage)) 50 | }) 51 | http.Handle("/", fs) 52 | 53 | // CORS設定 54 | headers := make(http.Header) 55 | headers.Set("Access-Control-Allow-Origin", "*") 56 | headers.Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") 57 | headers.Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization") 58 | corsHandler := func(h http.Handler) http.Handler { 59 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 60 | for key, values := range headers { 61 | for _, value := range values { 62 | w.Header().Add(key, value) 63 | } 64 | } 65 | if r.Method == "OPTIONS" { 66 | w.WriteHeader(http.StatusOK) 67 | return 68 | } 69 | h.ServeHTTP(w, r) 70 | }) 71 | } 72 | 73 | done := make(chan bool) 74 | go monitorFile("./fileServer.go", "./fileServerLastCheckedTime.txt", done) 75 | 76 | go func() { 77 | log.Printf("Serving %s on HTTP port: %s\n", dir, port) 78 | log.Fatal(http.ListenAndServe(":"+port, corsHandler(http.DefaultServeMux))) 79 | }() 80 | 81 | <-done 82 | log.Printf("File has been modified. Shutting down in 10 seconds.") 83 | time.Sleep(10 * time.Second) 84 | log.Printf("Shutting down server.") 85 | } -------------------------------------------------------------------------------- /storage/fileServer-v1.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "net/http" 6 | "time" 7 | "bytes" 8 | "io/ioutil" 9 | ) 10 | 11 | func monitorFile(filePath, timeFilePath string, done chan bool) { 12 | ticker := time.NewTicker(1 * time.Minute) 13 | var lastFileContent []byte 14 | lastFileContent, _ = ioutil.ReadFile(filePath) 15 | 16 | for { 17 | select { 18 | case <-ticker.C: 19 | fileContent, err := ioutil.ReadFile(filePath) 20 | if err != nil { 21 | log.Printf("Error reading file: %v", err) 22 | continue 23 | } 24 | 25 | if !bytes.Equal(fileContent, lastFileContent) { 26 | log.Printf("File %s has been modified", filePath) 27 | lastFileContent = fileContent 28 | time.AfterFunc(10*time.Second, func() { 29 | done <- true 30 | }) 31 | } 32 | 33 | // 現在の時刻を取得し、ファイルに書き込む 34 | currentTime := time.Now().Format(time.RFC3339) 35 | if err := ioutil.WriteFile(timeFilePath, []byte(currentTime), 0644); err != nil { 36 | log.Printf("Error writing last checked time: %v", err) 37 | } 38 | } 39 | } 40 | } 41 | 42 | func main() { 43 | dir := "./public_v1/v1.0.0" 44 | port := "8001" 45 | 46 | fs := http.FileServer(http.Dir(dir)) 47 | http.HandleFunc("/server", func(w http.ResponseWriter, r *http.Request) { 48 | responseMessage := "{\"version\":\"1.4.1\"}" 49 | w.Write([]byte(responseMessage)) 50 | }) 51 | http.Handle("/", fs) 52 | 53 | // CORS設定 54 | headers := make(http.Header) 55 | headers.Set("Access-Control-Allow-Origin", "*") 56 | headers.Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") 57 | headers.Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, Authorization") 58 | corsHandler := func(h http.Handler) http.Handler { 59 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 60 | for key, values := range headers { 61 | for _, value := range values { 62 | w.Header().Add(key, value) 63 | } 64 | } 65 | if r.Method == "OPTIONS" { 66 | w.WriteHeader(http.StatusOK) 67 | return 68 | } 69 | h.ServeHTTP(w, r) 70 | }) 71 | } 72 | 73 | done := make(chan bool) 74 | go monitorFile("./fileServer.go", "./fileServerLastCheckedTime.txt", done) 75 | 76 | go func() { 77 | log.Printf("Serving %s on HTTP port: %s\n", dir, port) 78 | log.Fatal(http.ListenAndServe(":"+port, corsHandler(http.DefaultServeMux))) 79 | }() 80 | 81 | <-done 82 | log.Printf("File has been modified. Shutting down in 10 seconds.") 83 | time.Sleep(10 * time.Second) 84 | log.Printf("Shutting down server.") 85 | } -------------------------------------------------------------------------------- /about/src/components/HelloWorld.vue: -------------------------------------------------------------------------------- 1 | 69 | 70 | 100 | -------------------------------------------------------------------------------- /storage/serverUpdater.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "bytes" 6 | "log" 7 | "net/http" 8 | "time" 9 | ) 10 | 11 | func monitorURL(url string, filePath string) { 12 | ticker := time.NewTicker(1 * time.Minute) 13 | for { 14 | select { 15 | case <-ticker.C: 16 | resp, err := http.Get(url) 17 | if err != nil { 18 | log.Printf("Error fetching URL: %v", err) 19 | continue 20 | } 21 | defer resp.Body.Close() 22 | 23 | content, err := ioutil.ReadAll(resp.Body) 24 | if err != nil { 25 | log.Printf("Error reading response body: %v", err) 26 | continue 27 | } 28 | 29 | // 文字数が100字以下の場合、処理をスキップ 30 | if len(content) <= 100 { 31 | log.Printf("Content is less than or equal to 100 characters, skipping update.") 32 | continue 33 | } 34 | 35 | // ファイルから前回の内容を読み込む 36 | lastContent, err := ioutil.ReadFile(filePath) 37 | if err != nil { 38 | log.Printf("Error reading from file: %v, assuming it's a new content", err) 39 | lastContent = nil 40 | } 41 | 42 | if lastContent == nil || !bytes.Equal(lastContent, content) { 43 | log.Printf("Content has been modified, saving to file: %s", filePath) 44 | err = ioutil.WriteFile(filePath, content, 0644) 45 | if err != nil { 46 | log.Printf("Error writing to file: %v", err) 47 | } 48 | } 49 | } 50 | } 51 | } 52 | 53 | func monitorFile(filePath, timeFilePath string, done chan bool) { 54 | ticker := time.NewTicker(1 * time.Minute) 55 | var lastFileContent []byte 56 | lastFileContent, _ = ioutil.ReadFile(filePath) 57 | 58 | for { 59 | select { 60 | case <-ticker.C: 61 | fileContent, err := ioutil.ReadFile(filePath) 62 | if err != nil { 63 | log.Printf("Error reading file: %v", err) 64 | continue 65 | } 66 | 67 | if !bytes.Equal(fileContent, lastFileContent) { 68 | log.Printf("File %s has been modified", filePath) 69 | lastFileContent = fileContent 70 | time.AfterFunc(10*time.Second, func() { 71 | done <- true 72 | }) 73 | } 74 | 75 | // 現在の時刻を取得し、ファイルに書き込む 76 | currentTime := time.Now().Format(time.RFC3339) 77 | if err := ioutil.WriteFile(timeFilePath, []byte(currentTime), 0644); err != nil { 78 | log.Printf("Error writing last checked time: %v", err) 79 | } 80 | } 81 | } 82 | } 83 | 84 | func main() { 85 | 86 | go monitorURL("https://butter.takoyaki3.com/code/serverUpdater.go", "./serverUpdater.go") 87 | go monitorURL("https://butter.takoyaki3.com/code/dataUpdater.go", "./dataUpdater.go") 88 | go monitorURL("https://butter.takoyaki3.com/code/fileServer.go", "./fileServer.go") 89 | go monitorURL("https://butter.takoyaki3.com/code/dataUpdater-v1.go", "./dataUpdater-v1.go") 90 | go monitorURL("https://butter.takoyaki3.com/code/fileServer-v1.go", "./fileServer-v1.go") 91 | go monitorURL("https://butter.takoyaki3.com/code/go.mod", "./go.mod") 92 | go monitorURL("https://butter.takoyaki3.com/code/go.sum", "./go.sum") 93 | 94 | done := make(chan bool) 95 | go monitorFile("./serverUpdater.go", "./serverUpdaterLastCheckedTime.txt", done) 96 | 97 | <-done 98 | log.Printf("File has been modified. Shutting down in 10 seconds.") 99 | time.Sleep(10 * time.Second) 100 | log.Printf("Shutting down server.") 101 | } -------------------------------------------------------------------------------- /lib/realtime-bus-locations.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | バスの位置情報 5 | 6 | 7 | 8 | 13 | 14 | 15 |
16 | 17 | 18 | 19 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /cmd/check.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/tar" 5 | "fmt" 6 | "os" 7 | "io" 8 | "log" 9 | 10 | json "github.com/takoyaki-3/go-json" 11 | ) 12 | 13 | type dataListStr []struct { 14 | NAMING_FAILED string `json:"事業者名,omitempty"` 15 | URL string `json:"事業者名_url,omitempty"` 16 | NAMING_FAILED0 string `json:"都道府県,omitempty"` 17 | GTFS string `json:"GTFSフィード名,omitempty"` 18 | NAMING_FAILED1 string `json:"ライセンス,omitempty"` 19 | URL0 string `json:"ライセンス_url,omitempty"` 20 | URLs string `json:"URLs,omitempty"` 21 | GTFSURL string `json:"GTFS_url,omitempty"` 22 | GTFS0 string `json:"最新GTFS開始日,omitempty"` 23 | GTFS1 string `json:"最新GTFS終了日,omitempty"` 24 | NAMING_FAILED2 string `json:"最終更新日,omitempty"` 25 | NAMING_FAILED3 string `json:"詳細,omitempty"` 26 | GTFS_ID string `json:"gtfs_id,omitempty"` 27 | VehiclePositionURL string `json:"VehiclePosition_url,omitempty"` 28 | } 29 | 30 | type OrganizationInfo []struct { 31 | VersionID string `json:"version_id"` 32 | ByStopHashValueSize int `json:"by_stop_hash_value_size"` 33 | ByTripHashValueSize int `json:"by_trip_hash_value_size"` 34 | } 35 | 36 | func main(){ 37 | fmt.Println("start") 38 | 39 | dataList := dataListStr{} 40 | if err := json.LoadFromPath("./data.json", &dataList);err!=nil{ 41 | log.Fatalln(err) 42 | } 43 | 44 | // GTFS_IDごとのデータ存在確認 45 | for _,v:=range dataList{ 46 | if v.GTFS_ID==""{ 47 | continue 48 | } 49 | err := checkCrganization(v.GTFS_ID) 50 | if err != nil { 51 | fmt.Println(err) 52 | } 53 | } 54 | 55 | // n-gramフォルダが存在するか 56 | 57 | // 58 | } 59 | 60 | func checkCrganization(GTFS_ID string)error{ 61 | // 62 | info := OrganizationInfo{} 63 | err := json.LoadFromPath("v0.0.0/"+GTFS_ID+"/info.json", &info) 64 | if err != nil{ 65 | return err 66 | } 67 | 68 | // 各ファイルがあるか判定 69 | dir := "v0.0.0/"+GTFS_ID+"/"+info[0].VersionID 70 | 71 | // ByTrips 72 | for i:=0;i<16< { 71 | const results = [] 72 | for(const functionName in functionParams) { 73 | const params = functionParams[functionName].filter(param => sampleParams[param]!==undefined); 74 | const url = `${API_ENDPOINT}/${functionName}?${params.map(param => `${param}=${(typeof(sampleParams[param]) === 'object' || typeof(sampleParams[param]) === 'array') ? JSON.stringify(sampleParams[param]) : sampleParams[param]}`).join('&')}` 75 | try { 76 | const response = await axios.get(url) 77 | // results.push({ functionName, params, response: response.data }) 78 | console.log(url) 79 | console.log(response.data) 80 | } catch (error) { 81 | console.log(error) 82 | results.push({ functionName, error, url }) 83 | } 84 | } 85 | results.forEach(result => console.log(result)) 86 | } 87 | 88 | main(); -------------------------------------------------------------------------------- /lib/test/helper.test.js: -------------------------------------------------------------------------------- 1 | const { helper } = require('../src/helper') 2 | const { pemToUint8Array } = require('../src/helper') 3 | 4 | describe('pemToUint8Array', () => { 5 | it('converts a PEM string to a Uint8Array', () => { 6 | // この部分は適切なBase64エンコードされた文字列に置き換えてください 7 | const base64EncodedString = 'VGhpcyBpcyBhIHRlc3Qgc3RyaW5nLg==' // "This is a test string."のBase64エンコード 8 | const pem = `-----BEGIN TEST-----\n${base64EncodedString}\n-----END TEST-----` 9 | 10 | const result = pemToUint8Array(pem) 11 | 12 | expect(result).toBeInstanceOf(Uint8Array) 13 | // デコードされた文字列の長さを検証 14 | expect(result.length).toBe(atob(base64EncodedString).length) 15 | // デコードされた文字列の各文字のcharCodeAtとUint8Arrayの値を比較 16 | const decodedString = atob(base64EncodedString) 17 | for (let i = 0; i < decodedString.length; i++) { 18 | expect(result[i]).toBe(decodedString.charCodeAt(i)) 19 | } 20 | }) 21 | }) 22 | 23 | describe('csvToObject', () => { 24 | it('should convert CSV string to an array of objects', () => { 25 | const csv = 'header1,header2\nvalue1,value2\nvalue3,value4' 26 | const result = helper.csvToObject(csv) 27 | expect(result).toEqual([ 28 | { header1: 'value1', header2: 'value2' }, 29 | { header1: 'value3', header2: 'value4' } 30 | ]) 31 | }) 32 | }) 33 | 34 | it('should parse a date string to a Date object', () => { 35 | const dateStr = '20230101' 36 | const result = helper.parseDate(dateStr) 37 | expect(result).toEqual(new Date('2023-01-01T09:00:00')) 38 | }) 39 | 40 | describe('getDayOfWeek', () => { 41 | it('should return the day of the week for a given date string', () => { 42 | const dateStr = '20230101' // 2023年1月1日は日曜日 43 | const result = helper.getDayOfWeek(dateStr) 44 | expect(result).toBe('日') 45 | }) 46 | }) 47 | 48 | describe('helper.storeCache', () => { 49 | it('stores a value in the cache', () => { 50 | const cache = {} 51 | helper.storeCache('key', 'value', cache) 52 | expect(cache.key).toBe('value') 53 | }) 54 | 55 | it('respects the cache size limit', () => { 56 | const cache = {} 57 | const cacheSize = 2 58 | helper.storeCache('key1', 'value1', cache, cacheSize) 59 | helper.storeCache('key2', 'value2', cache, cacheSize) 60 | // TODO キャッシュサイズ以上になった場合の対応 61 | // helper.storeCache('key3', 'value3', cache, cacheSize); 62 | expect(Object.keys(cache).length).toBeLessThanOrEqual(cacheSize) 63 | }) 64 | }) 65 | 66 | describe('helper.loadCache', () => { 67 | it('retrieves a value from the cache', () => { 68 | const cache = { key: 'value' } 69 | const result = helper.loadCache('key', cache) 70 | expect(result).toBe('value') 71 | }) 72 | 73 | it('returns undefined for missing keys', () => { 74 | const cache = { key: 'value' } 75 | const result = helper.loadCache('nonexistent', cache) 76 | expect(result).toBeUndefined() 77 | }) 78 | }) 79 | 80 | describe('helper.csvToObject', () => { 81 | it('converts CSV to an array of objects', () => { 82 | const csv = 'name,age\nAlice,30\nBob,35' 83 | const result = helper.csvToObject(csv) 84 | expect(result).toEqual([ 85 | { name: 'Alice', age: '30' }, 86 | { name: 'Bob', age: '35' } 87 | ]) 88 | }) 89 | }) 90 | 91 | describe('helper.setIntersection', () => { 92 | it('returns the intersection of two sets', () => { 93 | const setA = new Set([1, 2, 3]) 94 | const setB = new Set([2, 3, 4]) 95 | const result = helper.setIntersection(setA, setB) 96 | expect(result).toEqual(new Set([2, 3])) 97 | }) 98 | }) 99 | -------------------------------------------------------------------------------- /storage/s3.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "encoding/json" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | 13 | "github.com/aws/aws-sdk-go/aws" 14 | "github.com/aws/aws-sdk-go/aws/credentials" 15 | "github.com/aws/aws-sdk-go/aws/session" 16 | "github.com/aws/aws-sdk-go/service/s3" 17 | ) 18 | 19 | // { 20 | // "s3_endpoint": "https://s3-compatible-storage.example.com", 21 | // "s3_access_key": "your_access_key", 22 | // "s3_secret_key": "your_secret_key", 23 | // "s3_bucket": "your_bucket", 24 | // "tar_file_url": "https://example.com/path/to/your/file.tar" 25 | // } 26 | 27 | type Config struct { 28 | S3Endpoint string `json:"s3_endpoint"` 29 | S3AccessKey string `json:"s3_access_key"` 30 | S3SecretKey string `json:"s3_secret_key"` 31 | S3Bucket string `json:"s3_bucket"` 32 | TarFileURL string `json:"tar_file_url"` 33 | } 34 | 35 | func main() { 36 | // Load configuration from JSON file 37 | config, err := loadConfig("config.json") 38 | if err != nil { 39 | log.Fatal(err) 40 | } 41 | 42 | // Download .tar file from the URL 43 | tarData, err := downloadTarFile(config.TarFileURL) 44 | if err != nil { 45 | log.Fatal(err) 46 | } 47 | 48 | err = uploadTarFilesToS3(config, tarData) 49 | if err != nil { 50 | log.Fatal(err) 51 | } 52 | } 53 | 54 | func loadConfig(file string) (*Config, error) { 55 | data, err := ioutil.ReadFile(file) 56 | if err != nil { 57 | return nil, err 58 | } 59 | 60 | var config Config 61 | err = json.Unmarshal(data, &config) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | return &config, nil 67 | } 68 | 69 | func downloadTarFile(url string) ([]byte, error) { 70 | resp, err := http.Get(url) 71 | if err != nil { 72 | return nil, err 73 | } 74 | defer resp.Body.Close() 75 | 76 | if resp.StatusCode != http.StatusOK { 77 | return nil, fmt.Errorf("failed to download .tar file, status code: %d", resp.StatusCode) 78 | } 79 | 80 | data, err := ioutil.ReadAll(resp.Body) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | return data, nil 86 | } 87 | 88 | func uploadTarFilesToS3(config *Config, tarData []byte) error { 89 | // Initialize the S3 client 90 | sess, err := session.NewSession(&aws.Config{ 91 | Region: aws.String("us-east-1"), 92 | Endpoint: aws.String(config.S3Endpoint), 93 | Credentials: credentials.NewStaticCredentials(config.S3AccessKey, config.S3SecretKey, ""), 94 | }) 95 | if err != nil { 96 | return err 97 | } 98 | 99 | s3Client := s3.New(sess) 100 | 101 | // Create a buffer with the .tar data and read it 102 | tarReader := tar.NewReader(bytes.NewReader(tarData)) 103 | 104 | counter := 0 105 | 106 | for { 107 | header, err := tarReader.Next() 108 | 109 | if err == io.EOF { 110 | break 111 | } else if err != nil { 112 | return err 113 | } 114 | 115 | if header.Typeflag == tar.TypeReg { 116 | var buf bytes.Buffer 117 | _, err = io.Copy(&buf, tarReader) 118 | if err != nil { 119 | return err 120 | } 121 | 122 | // Upload file to S3-compatible storage 123 | _, err = s3Client.PutObject(&s3.PutObjectInput{ 124 | Bucket: aws.String(config.S3Bucket), 125 | Key: aws.String(header.Name), 126 | Body: bytes.NewReader(buf.Bytes()), 127 | }) 128 | if err != nil { 129 | return err 130 | } 131 | 132 | fmt.Println(counter, "Uploaded:", header.Name) 133 | counter++ 134 | } 135 | } 136 | 137 | return nil 138 | } 139 | 140 | -------------------------------------------------------------------------------- /cmd/sender.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto" 5 | "crypto/rand" 6 | "crypto/rsa" 7 | "crypto/sha256" 8 | "crypto/x509" 9 | "encoding/pem" 10 | "fmt" 11 | . "github.com/takoyaki-3/butter/cmd/helper" 12 | "io" 13 | "net" 14 | "os" 15 | ) 16 | 17 | func main() { 18 | sourceDir := "./v0.0.0/" 19 | targetFile := "archive.tar" 20 | serverAddr := "butter.takoyaki3.com:8001" 21 | 22 | // 1. 指定したディレクトリ内のファイルを.tar形式にまとめる 23 | err := CreateTarArchive(sourceDir, targetFile) 24 | if err != nil { 25 | fmt.Printf("Error creating tar archive: %v\n", err) 26 | return 27 | } 28 | 29 | // 2. ソケット通信で.tarファイルを送信する 30 | conn, err := net.Dial("tcp", serverAddr) 31 | if err != nil { 32 | fmt.Printf("Error connecting to server: %v\n", err) 33 | return 34 | } 35 | defer conn.Close() 36 | 37 | file, err := os.Open(targetFile) 38 | if err != nil { 39 | fmt.Printf("Error opening tar file: %v\n", err) 40 | return 41 | } 42 | defer file.Close() 43 | 44 | fileInfo, err := file.Stat() 45 | if err != nil { 46 | fmt.Printf("Error getting file info: %v\n", err) 47 | return 48 | } 49 | 50 | // 電子署名用のキーペアを生成 51 | privKey, err := rsa.GenerateKey(rand.Reader, 2048) 52 | if err != nil { 53 | fmt.Printf("Error generating private key: %v\n", err) 54 | return 55 | } 56 | 57 | pubKeyBytes, err := x509.MarshalPKIXPublicKey(&privKey.PublicKey) 58 | if err != nil { 59 | fmt.Printf("Error marshaling public key: %v\n", err) 60 | return 61 | } 62 | 63 | pubKeyPem := pem.EncodeToMemory(&pem.Block{ 64 | Type: "PUBLIC KEY", 65 | Bytes: pubKeyBytes, 66 | }) 67 | 68 | // 公開鍵のサイズを送信 69 | pubKeySize := len(pubKeyPem) 70 | pubKeySizeStr := fmt.Sprintf("%20d", pubKeySize) // パディングを追加 71 | _, err = conn.Write([]byte(pubKeySizeStr)) 72 | if err != nil { 73 | fmt.Printf("Error sending public key size: %v\n", err) 74 | return 75 | } 76 | 77 | // 確認メッセージを受信 78 | confirmation := make([]byte, 7) 79 | _, err = conn.Read(confirmation) 80 | if err != nil || string(confirmation) != "confirm" { 81 | fmt.Printf("Error receiving confirmation: %v\n", err) 82 | return 83 | } 84 | 85 | // 公開鍵を送信 86 | _, err = conn.Write(pubKeyPem) 87 | if err != nil { 88 | fmt.Printf("Error sending public key: %v\n", err) 89 | return 90 | } 91 | fmt.Println(string(pubKeyPem)) 92 | 93 | // ファイルサイズを送信 94 | fileSize := fmt.Sprintf("%20d", fileInfo.Size()) 95 | conn.Write([]byte(fileSize)) 96 | 97 | buffer := make([]byte, 1024) 98 | for { 99 | _, err = file.Read(buffer) 100 | if err == io.EOF { 101 | break 102 | } 103 | 104 | // データを送信 105 | _, err = conn.Write(buffer) 106 | if err != nil { 107 | fmt.Printf("Error sending file data: %v\n", err) 108 | return 109 | } 110 | } 111 | 112 | // 電子署名を生成 113 | signature, err := createSignature(targetFile, privKey) 114 | if err != nil { 115 | fmt.Printf("Error creating signature: %v\n", err) 116 | return 117 | } 118 | 119 | // 電子署名を送信 120 | _, err = conn.Write(signature) 121 | if err != nil { 122 | fmt.Printf("Error sending signature: %v\n", err) 123 | return 124 | } 125 | 126 | fmt.Println("File and signature sent successfully.") 127 | } 128 | 129 | // 電子署名を生成する 130 | func createSignature(filePath string, privKey *rsa.PrivateKey) ([]byte, error) { 131 | file, err := os.Open(filePath) 132 | if err != nil { 133 | return nil, err 134 | } 135 | defer file.Close() 136 | 137 | hash := sha256.New() 138 | if _, err := io.Copy(hash, file); err != nil { 139 | return nil, err 140 | } 141 | 142 | hashed := hash.Sum(nil) 143 | signature, err := rsa.SignPKCS1v15(rand.Reader, privKey, crypto.SHA256, hashed) 144 | if err != nil { 145 | return nil, err 146 | } 147 | 148 | return signature, nil 149 | } 150 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## BuTTER 2 | 3 |

4 | BuTTER | Website | Tag Maker | Timetable 5 |

6 |

7 | 8 |
9 | 10 |
11 | 12 | ## 概要 13 | 14 | BuTTER（Bus Time-Table by Edge-Rumtime）は、数行のBuTTERタグをWebページに貼り付けることで、最新の時刻表をWebページに埋め込むことができるツールです。それに付随したライブラリ、データ変換ツール、WebAPIも提供しています。 15 | 16 | ## 特徴 17 | 18 | - **簡単な実装**: BuTTERタグをWebページに貼り付けるだけで、時刻表を表示できます。 19 | - **自動更新**: 時刻表は自動的に更新されるため、ダイヤ改正にも対応できます。 20 | - **様々な機能**: バス停検索、周辺バス停表示、リアルタイム位置情報表示などの機能を提供します。 21 | - **開発者向け**: 開発者はBuTTERライブラリを使用して、独自のアプリケーションに時刻表機能を統合できます。 22 | 23 | ## インストール 24 | 25 | BuTTERは、以下の方法で利用できます。 26 | 27 | - **BuTTERタグ**: [https://tag-maker.butter.takoyaki3.com/](https://tag-maker.butter.takoyaki3.com/) でタグを生成し、Webページに貼り付けます。 28 | - **BuTTERライブラリ**: npmで公開されています。`npm install butter-lib` でインストールできます。 29 | - **WebAPI**: [https://api.butter.takoyaki3.com/v1](https://api.butter.takoyaki3.com/v1) からアクセスできます。詳細なAPI仕様は、[https://butter.takoyaki3.com/api.html](https://butter.takoyaki3.com/api.html) を参照してください。 30 | 31 | ## 使い方 32 | 33 | ### BuTTERタグ 34 | 35 | 1. [https://tag-maker.butter.takoyaki3.com/](https://tag-maker.butter.takoyaki3.com/) にアクセスします。 36 | 2. 表示したい事業者とバス停を選択します。 37 | 3. 生成されたタグをコピーし、Webページに貼り付けます。 38 | 39 | ### BuTTERライブラリ 40 | 41 | ```javascript 42 | import Butter from 'butter-lib'; 43 | 44 | // BuTTERの初期化 45 | await Butter.init('https://butter.takoyaki3.com/v1.0.0/root.json'); 46 | 47 | // データリストの取得 48 | const dataList = await Butter.getHostDataList(); 49 | 50 | // 特定の事業者の時刻表の取得 51 | const timetable = await Butter.fetchTimeTableV1('ToeiBus', { 52 | date: '20240715', 53 | stop_ids: ['0605-07'], 54 | }); 55 | ``` 56 | 57 | ### WebAPI 58 | 59 | WebAPIのエンドポイントとパラメータについては、[https://butter.takoyaki3.com/api.html](https://butter.takoyaki3.com/api.html) を参照してください。 60 | 61 | ### 環境変数 62 | 63 | BuTTERライブラリを使用する場合、以下の環境変数を設定できます。 64 | 65 | - `BUTTER_ROOT`: BuTTERのルートURL。デフォルトは `https://butter.takoyaki3.com/v1.0.0/root.json` です。 66 | 67 | ### APIエンドポイント 68 | 69 | BuTTERのWebAPIのエンドポイントは、 `https://api.butter.takoyaki3.com/v1` です。 70 | 71 | ### 設定ファイル 72 | 73 | BuTTERは設定ファイルを使用しません。 74 | 75 | ### コマンド実行例 76 | 77 | BuTTERのデータ変換ツールは、以下のコマンドで実行できます。 78 | 79 | ```bash 80 | ./cmd.sh 81 | ``` 82 | 83 | ## プロジェクト構成 84 | 85 | ``` 86 | ├── about 87 | │ ├── public 88 | │ │ └── v0.0.0 89 | │ │ └── root.json 90 | │ └── src 91 | │ └── components 92 | │ └── DemoSpace.vue 93 | ├── api 94 | │ └── src 95 | │ └── functionParams.js 96 | ├── cmd 97 | │ └── helper 98 | │ └── sign.go 99 | ├── lib 100 | │ └── src 101 | │ └── internal.js 102 | ├── storage 103 | │ └── public_v1 104 | │ └── v1.0.0 105 | │ └── n-gram 106 | │ └── テ.csv 107 | ├── tag-maker 108 | │ └── src 109 | │ └── components 110 | │ └── TagInput.vue 111 | └── timetable-app 112 | └── public 113 | └── v0.0.0 114 | └── root.json 115 | ``` 116 | 117 | ### ディレクトリ 118 | 119 | - `about`: BuTTERのプロジェクト概要とデモページのコードを含みます。 120 | - `api`: BuTTERのWebAPIのコードを含みます。 121 | - `cmd`: GTFSデータをBuTTER形式に変換するスクリプトを含みます。 122 | - `lib`: BuTTERライブラリのコードを含みます。 123 | - `storage`: BuTTERストレージサーバーのコードを含みます。 124 | - `tag-maker`: BuTTERタグを生成するWebアプリケーションのコードを含みます。 125 | - `timetable-app`: BuTTERを活用した時刻表アプリのコードを含みます。 126 | 127 | ### ファイル 128 | 129 | - `root.json`: BuTTERのルート設定ファイル。 130 | - `datalist.json`: データリスト。 131 | - `info.json`: バージョン情報。 132 | - `*.tar.gz`: 分割された時刻表データ。 133 | - `*.csv`: GTFSデータ。 134 | 135 | ## ライセンス 136 | 137 | MIT License 138 | 139 | レポジトリをクローンして独自のサーバで運営しても構いません。万が一、このプロジェクトオーナーがサーバ運営をやめた場合でも独自ホストにより引き続き表示することができます。 140 | 141 | ## 貢献 142 | 143 | バグレポートやフィーチャーリクエストは、GitHubのIssuesにて受け付けています。また、Pull Requestも歓迎します。 -------------------------------------------------------------------------------- /cmd/8_add_stopdata.py: -------------------------------------------------------------------------------- 1 | import h3 2 | import pandas as pd 3 | import os 4 | 5 | # ディレクトリのパスを指定する 6 | directory = './dir_out' 7 | 8 | # ディレクトリ内のディレクトリ一覧を取得する 9 | subdirectories = [os.path.join(directory, d) for d in os.listdir(directory) if os.path.isdir(os.path.join(directory, d))] 10 | 11 | # ディレクトリ一覧を出力する 12 | print(subdirectories) 13 | 14 | csv_files = [] 15 | 16 | for dir in subdirectories: 17 | dir = dir.replace('\\','/',-1) 18 | gtfsID = dir.split('/')[2][:-4] 19 | print(gtfsID) 20 | 21 | # GTFS stop_times.txtファイルを読み込む 22 | stop_times = pd.read_csv('./dir_out/'+gtfsID+'.zip/stops.txt') 23 | stop_times['gtfs_id'] = gtfsID 24 | 25 | # インデックスを含むstop_times.txtファイルを書き出す 26 | filename = 'stop_times_with_h3index_' + gtfsID + '.txt' 27 | stop_times.to_csv(filename, index=False) 28 | csv_files.append(filename) 29 | 30 | # CSVファイルを読み込んでDataFrameに追加する 31 | df_list = [] 32 | for file in csv_files: 33 | df = pd.read_csv(file) 34 | df_list.append(df) 35 | os.remove(file) 36 | 37 | # DataFrameを結合する 38 | merged_df = pd.concat(df_list) 39 | 40 | # # 結合したDataFrameをCSVファイルに書き出す 41 | merged_df.to_csv('merged_csv_file.csv', index=False) 42 | 43 | import pandas as pd 44 | import h3 45 | 46 | # H3 level to use for indexing 47 | H3_LEVEL = 7 48 | 49 | # Load stops.txt file into a pandas DataFrame 50 | df_stops = merged_df 51 | 52 | # Add H3 index column to DataFrame 53 | df_stops["h3index"] = df_stops.apply(lambda row: h3.geo_to_h3(row["stop_lat"], row["stop_lon"], H3_LEVEL), axis=1) 54 | 55 | # Get all unique H3 indexes in the DataFrame 56 | unique_h3indexes = df_stops["h3index"].unique() 57 | 58 | # Create a dictionary to store the stops for each H3 index 59 | h3index_to_stops_dict = {h3index: [] for h3index in unique_h3indexes} 60 | 61 | # Iterate through each stop and add it to the appropriate H3 index 62 | for index, row in df_stops.iterrows(): 63 | h3index = row["h3index"] 64 | h3indexes_to_add = h3.k_ring(h3index, 1) # Get all H3 indexes within 1 ring of the current H3 index 65 | for index_to_add in h3indexes_to_add: 66 | if index_to_add in h3index_to_stops_dict: 67 | h3index_to_stops_dict[index_to_add].append(row) 68 | 69 | # Save each H3 index's stops to a separate file 70 | if not os.path.isdir('v0.0.0/byH3index'): 71 | os.mkdir('v0.0.0/byH3index') 72 | 73 | for h3index in unique_h3indexes: 74 | df_h3index_stops = pd.DataFrame(h3index_to_stops_dict[h3index]) 75 | filename = f"v0.0.0/byH3index/{h3index}_stops.csv" 76 | df_h3index_stops.to_csv(filename, index=False) 77 | 78 | # n-gram 79 | import pandas as pd 80 | from nltk.util import ngrams 81 | 82 | # N-gram length 83 | for N in range (1,2): 84 | 85 | # Load stops.txt file into a pandas DataFrame 86 | df_stops = pd.read_csv("merged_csv_file.csv") 87 | 88 | # Create n-grams for each stop_name 89 | ngram_lists = df_stops["stop_name"].apply(lambda x: [''.join(ngram) for ngram in ngrams(x, N)]) 90 | 91 | # Flatten n-gram lists 92 | ngrams_flat = [ngram for ngram_list in ngram_lists for ngram in ngram_list] 93 | 94 | # Get all unique n-grams 95 | unique_ngrams = list(set(ngrams_flat)) 96 | 97 | # Create a dictionary to store the stops for each n-gram 98 | ngram_to_stops_dict = {} 99 | 100 | # Iterate through each stop and add it to the appropriate n-gram 101 | for index, row in df_stops.iterrows(): 102 | stop_name = row["stop_name"] 103 | ngrams_to_add = list(ngrams(stop_name, N)) 104 | for ngram_to_add in ngrams_to_add: 105 | ngram_str = ''.join(ngram_to_add) 106 | if ngram_str not in ngram_to_stops_dict: 107 | ngram_to_stops_dict[ngram_str] = [] 108 | ngram_to_stops_dict[ngram_str].append(row) 109 | 110 | # Save each n-gram's stops to a separate file 111 | 112 | if not os.path.isdir('v0.0.0/n-gram'): 113 | os.mkdir('v0.0.0/n-gram') 114 | for ngram in unique_ngrams: 115 | df_ngram_stops = pd.DataFrame(ngram_to_stops_dict[ngram]) 116 | filename = f"v0.0.0/n-gram/{ngram}.csv" 117 | df_ngram_stops.to_csv(filename, index=False) 118 | -------------------------------------------------------------------------------- /cmd/8_add_stopdata_feed.py: -------------------------------------------------------------------------------- 1 | import h3 2 | import pandas as pd 3 | import os 4 | 5 | # ディレクトリのパスを指定する 6 | directory = './feed_dir_out' 7 | 8 | # ディレクトリ内のディレクトリ一覧を取得する 9 | subdirectories = [os.path.join(directory, d) for d in os.listdir(directory) if os.path.isdir(os.path.join(directory, d))] 10 | 11 | # ディレクトリ一覧を出力する 12 | print(subdirectories) 13 | 14 | csv_files = [] 15 | 16 | for dir in subdirectories: 17 | dir = dir.replace('\\','/',-1) 18 | gtfsID = dir.split('/')[2][:-4] 19 | feedID = dir.split('_FEEDID_')[1].replace('.zip','') 20 | print(gtfsID, feedID) 21 | 22 | # GTFS stop_times.txtファイルを読み込む 23 | stop_times = pd.read_csv(directory+'/'+gtfsID+'.zip/stops.txt') 24 | stop_times['gtfs_id'] = gtfsID 25 | stop_times['feed_id'] = feedID 26 | 27 | # インデックスを含むstop_times.txtファイルを書き出す 28 | filename = 'stop_times_with_h3index_' + gtfsID + '.txt' 29 | stop_times.to_csv(filename, index=False) 30 | csv_files.append(filename) 31 | 32 | # CSVファイルを読み込んでDataFrameに追加する 33 | df_list = [] 34 | for file in csv_files: 35 | df = pd.read_csv(file) 36 | df_list.append(df) 37 | os.remove(file) 38 | 39 | # DataFrameを結合する 40 | merged_df = pd.concat(df_list) 41 | 42 | # # 結合したDataFrameをCSVファイルに書き出す 43 | merged_df.to_csv('feed_merged_csv_file.csv', index=False) 44 | 45 | import pandas as pd 46 | import h3 47 | 48 | # H3 level to use for indexing 49 | H3_LEVEL = 7 50 | 51 | # Load stops.txt file into a pandas DataFrame 52 | df_stops = merged_df 53 | 54 | # Add H3 index column to DataFrame 55 | df_stops["h3index"] = df_stops.apply(lambda row: h3.geo_to_h3(row["stop_lat"], row["stop_lon"], H3_LEVEL), axis=1) 56 | 57 | # Get all unique H3 indexes in the DataFrame 58 | unique_h3indexes = df_stops["h3index"].unique() 59 | 60 | # Create a dictionary to store the stops for each H3 index 61 | h3index_to_stops_dict = {h3index: [] for h3index in unique_h3indexes} 62 | 63 | # Iterate through each stop and add it to the appropriate H3 index 64 | for index, row in df_stops.iterrows(): 65 | h3index = row["h3index"] 66 | h3indexes_to_add = h3.k_ring(h3index, 1) # Get all H3 indexes within 1 ring of the current H3 index 67 | for index_to_add in h3indexes_to_add: 68 | if index_to_add in h3index_to_stops_dict: 69 | h3index_to_stops_dict[index_to_add].append(row) 70 | 71 | # Save each H3 index's stops to a separate file 72 | if not os.path.isdir('v1.0.0/byH3index'): 73 | os.mkdir('v1.0.0/byH3index') 74 | 75 | for h3index in unique_h3indexes: 76 | df_h3index_stops = pd.DataFrame(h3index_to_stops_dict[h3index]) 77 | filename = f"v1.0.0/byH3index/{h3index}_stops.csv" 78 | df_h3index_stops.to_csv(filename, index=False) 79 | 80 | # n-gram 81 | import pandas as pd 82 | from nltk.util import ngrams 83 | 84 | # N-gram length 85 | for N in range (1,2): 86 | 87 | # Load stops.txt file into a pandas DataFrame 88 | df_stops = pd.read_csv("feed_merged_csv_file.csv") 89 | 90 | # Create n-grams for each stop_name 91 | ngram_lists = df_stops["stop_name"].apply(lambda x: [''.join(ngram) for ngram in ngrams(x, N)]) 92 | 93 | # Flatten n-gram lists 94 | ngrams_flat = [ngram for ngram_list in ngram_lists for ngram in ngram_list] 95 | 96 | # Get all unique n-grams 97 | unique_ngrams = list(set(ngrams_flat)) 98 | 99 | # Create a dictionary to store the stops for each n-gram 100 | ngram_to_stops_dict = {} 101 | 102 | # Iterate through each stop and add it to the appropriate n-gram 103 | for index, row in df_stops.iterrows(): 104 | stop_name = row["stop_name"] 105 | ngrams_to_add = list(ngrams(stop_name, N)) 106 | for ngram_to_add in ngrams_to_add: 107 | ngram_str = ''.join(ngram_to_add) 108 | if ngram_str not in ngram_to_stops_dict: 109 | ngram_to_stops_dict[ngram_str] = [] 110 | ngram_to_stops_dict[ngram_str].append(row) 111 | 112 | # Save each n-gram's stops to a separate file 113 | 114 | if not os.path.isdir('v1.0.0/n-gram'): 115 | os.mkdir('v1.0.0/n-gram') 116 | for ngram in unique_ngrams: 117 | df_ngram_stops = pd.DataFrame(ngram_to_stops_dict[ngram]) 118 | filename = f"v1.0.0/n-gram/{ngram}.csv" 119 | df_ngram_stops.to_csv(filename, index=False) 120 | -------------------------------------------------------------------------------- /lib/src/internal.js: -------------------------------------------------------------------------------- 1 | const helperLib = require('./helper.js') 2 | const helper = helperLib.helper 3 | 4 | export class ButterInternal { 5 | constructor (rootCA) { 6 | this.CONFIG = {} 7 | this.RUNTIME = { 8 | isReady: false, 9 | readyFlags: { 10 | ca: false 11 | }, 12 | cache: {}, 13 | pub_key: '' 14 | } 15 | this.setCA(rootCA) 16 | } 17 | 18 | /** 19 | * CA（証明書発行局）の設定を行います。 20 | * @param {string} rootCA - CAのルートURL。 21 | * @returns {Promise} - 処理が完了したら解決されるPromise。 22 | */ 23 | async setCA (rootCA) { 24 | console.log('setCA') 25 | // 末尾をスラッシュなしに統一する 26 | if (!rootCA.endsWith('root.json')) { 27 | return // TODO ファイルがなかった場合のエラー処理 28 | } 29 | this.CONFIG.rootCA = rootCA 30 | 31 | let req 32 | try { 33 | console.log('fetchAndParseJSON this.CONFIG.rootCA', this.CONFIG.rootCA) 34 | req = await helper.fetchAndParseJSON(this.CONFIG.rootCA) 35 | console.log('req', req) 36 | } catch (err) { 37 | console.error({ err }) 38 | throw new Error('Failed to fetch rootCA') 39 | } 40 | 41 | this.RUNTIME.CA = req 42 | this.RUNTIME.pub_key = req.pub_keys[0].pubkey 43 | 44 | console.log('set') 45 | 46 | const hostList = [] 47 | const oneMouthAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000) // 30日前の日時を設定します 48 | 49 | for (let i = 0; i < this.RUNTIME.CA.hosts.length; i++) { 50 | const host = this.RUNTIME.CA.hosts[i] 51 | 52 | try { // ホストからのデータ取得を試みる 53 | const data = await helper.fetchJSON(`${host}/datalist.json`, this.RUNTIME.pub_key) 54 | if (data.updated === null) continue 55 | 56 | // updatedをISO 8601形式に変換します 57 | const updatedISO = data.updated.replace(/_/g, ':') 58 | const updatedDate = new Date(updatedISO) // Dateオブジェクトを生成します 59 | if (updatedDate > oneMouthAgo) { // 30日以内に更新されていれば、hostListに追加します 60 | hostList.push(host) 61 | } 62 | } catch (e) { 63 | console.log(`Failed to fetch data from ${host}:`, e.message) // エラーメッセージをログに出力 64 | continue // このホストはスキップし、次のホストへ移動 65 | } 66 | } 67 | 68 | console.log('hostList', hostList) 69 | // ランダムにホストを選びます 70 | if (hostList.length > 0) { 71 | const randomIndex = Math.floor(Math.random() * hostList.length) 72 | this.RUNTIME.host = hostList[randomIndex] 73 | console.log(`host: ${this.RUNTIME.host}`) 74 | } else { 75 | console.log('there are no host which updated last 30 days.') 76 | const randomIndex = Math.floor(Math.random() * this.RUNTIME.CA.hosts.length) 77 | this.RUNTIME.host = this.RUNTIME.CA.hosts[randomIndex] 78 | console.log(`host: ${this.RUNTIME.host}`) 79 | } 80 | 81 | this.RUNTIME.readyFlags = {} 82 | this.RUNTIME.readyFlags.ca = true 83 | } 84 | 85 | /** 86 | * CAの準備が完了するまで待機します。 87 | * @returns {Promise} - CAの準備が完了したら解決されるPromise。 88 | */ 89 | async waitReady () { 90 | while (!this.RUNTIME.isReady) { 91 | await helper.sleep(100) 92 | if (Object.values(this.RUNTIME.readyFlags).every(e => e)) { 93 | this.RUNTIME.isReady = true 94 | } 95 | } 96 | } 97 | 98 | async waitCAReady () { 99 | while (!this.RUNTIME.readyFlags.ca) { 100 | console.log('waiting CA ready') 101 | await helper.sleep(100) 102 | } 103 | } 104 | 105 | /** 106 | * 緯度と経度から距離を計算します。 107 | * @param {number} lat1 - 最初の緯度。 108 | * @param {number} lon1 - 最初の経度。 109 | * @param {number} lat2 - 2番目の緯度。 110 | * @param {number} lon2 - 2番目の経度。 111 | * @returns {number} - 計算された距離（km単位）。 112 | */ 113 | async distance (lat1, lon1, lat2, lon2) { 114 | const R = 6371 // 地球の半径（km） 115 | const dLat = (lat2 - lat1) * (Math.PI / 180) 116 | const dLon = (lon2 - lon1) * (Math.PI / 180) 117 | const a = 118 | Math.sin(dLat / 2) * Math.sin(dLat / 2) + 119 | Math.cos(lat1 * (Math.PI / 180)) * 120 | Math.cos(lat2 * (Math.PI / 180)) * 121 | Math.sin(dLon / 2) * 122 | Math.sin(dLon / 2) 123 | const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)) 124 | return R * c 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /lib/test.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Test 6 | 7 | 8 |

Test

9 | 10 | 105 | 106 | -------------------------------------------------------------------------------- /storage/readme.md: -------------------------------------------------------------------------------- 1 | # BuTTERストレージホスト方法 2 | 3 | ## docker-composeによるホスト（SSL化環境が既に存在する場合） 4 | 5 | ### 前提条件 6 | - DockerとDocker Composeがインストールされている。 7 | - 使用するドメイン名が決定しており、DNS設定でこのサーバーを指している。 8 | - 起動したコンテナに対する通信をSSL化できるリバースプロキシ等が存在する。 9 | 10 | ### ホスト手順 11 | 12 | #### 1. サービスのデプロイ 13 | 14 | 1. ターミナルを開き、`storage`ディレクトリに移動します。 15 | 16 | 2. 次のコマンドにより、すべてのサービスを起動します。 17 | 18 | ```sh 19 | docker-compose up -d 20 | ``` 21 | 22 | 3. 起動したコンテナのポート（初期設定は8000）に対してHTTPSアクセスできるよう設定する 23 | 24 | Caddy、NgineやApacheなどを間に挟み、HTTPSによりアクセスできるようにします。 25 | 26 | 4. 公開したURLにアクセスし、ストレージとして登録ボタンを押す 27 | 28 | ![](addStoragePage.png) 29 | 30 | 下のようなメッセージが表示されれば追加成功です。 31 | 32 | ![](addSuccess.png) 33 | 34 | 5. 5分ほど待機する 35 | 36 | 5分ほど待機した後に[https://butter.takoyaki3.com/v0.0.0/root.json](https://butter.takoyaki3.com/v0.0.0/root.json)にアクセスしてROOTファイルを確認すると、ストレージが追加されたか確認できます。 37 | 38 | ```json 39 | { 40 | "pub_keys": [ 41 | { 42 | "pubkey": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/public_key.pem" 43 | } 44 | ], 45 | "hosts": [ 46 | "https://storage.app.takoyaki3.com", 47 | "https://butter.oozora283.com" 48 | ], 49 | "original_data": { 50 | "host": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/v0.0.0/originalData/" 51 | }, 52 | "api_endpoints": [ 53 | "https://butter.hatano-yuuta7921.workers.dev/" 54 | ], 55 | "api_data_hosts": [ 56 | "https://storage.app.takoyaki3.com/" 57 | ], 58 | "last_update": "2023-05-06T19_02_00+09_00" 59 | } 60 | ``` 61 | 62 | ## docker-composeによるホスト（SSL化環境がない場合） 63 | 64 | ### 前提条件 65 | - DockerとDocker Composeがインストールされている。 66 | - 使用するドメイン名が決定しており、DNS設定でこのサーバーを指している。 67 | 68 | ### ホスト手順 69 | 70 | #### 1. Caddyファイルの作成 71 | 72 | `storage`ディレクトリに含まれる`Caddyfile`を以下の内容に書き換えます。your-domain.comは使用するドメインに置き換えてください。 73 | 74 | ``` 75 | your-domain.com { 76 | reverse_proxy file-server:8000 77 | } 78 | ``` 79 | 80 | #### 2. docker-compose.ymlの修正 81 | 82 | 次のように`docker-compose.yml`のCaddyに関するコメントアウトを解除します。 83 | 84 | ```yml 85 | version: '3.8' 86 | services: 87 | file-server: 88 | image: golang:1.19 89 | volumes: 90 | - ./:/app 91 | working_dir: /app 92 | command: go run fileServer.go 93 | ports: 94 | - "8000:8000" 95 | restart: unless-stopped 96 | 97 | data-updater: 98 | image: golang:1.19 99 | volumes: 100 | - ./:/app 101 | working_dir: /app 102 | command: go run dataUpdater.go 103 | restart: unless-stopped 104 | 105 | server-updater: 106 | image: golang:1.19 107 | volumes: 108 | - ./:/app 109 | working_dir: /app 110 | command: go run serverUpdater.go 111 | restart: unless-stopped 112 | 113 | caddy: 114 | image: caddy:2 115 | volumes: 116 | - ./Caddyfile:/etc/caddy/Caddyfile 117 | - caddy_data:/data 118 | - caddy_config:/config 119 | ports: 120 | - "80:80" 121 | - "443:443" 122 | restart: unless-stopped 123 | volumes: 124 | caddy_data: 125 | caddy_config: 126 | ``` 127 | 128 | #### 3. サービスのデプロイ 129 | 130 | 1. ターミナルを開き、`storage`ディレクトリに移動します。 131 | 132 | 2. 次のコマンドにより、すべてのサービスを起動します。 133 | 134 | ```sh 135 | docker-compose up -d 136 | ``` 137 | 138 | このコマンドは、背後でDockerコンテナをビルドし、起動します。Caddyは自動的にLet's EncryptからSSL証明書を取得し、設定したドメインでHTTPS接続を提供します。 139 | 140 | 3. 公開したURLにアクセスし、ストレージとして登録ボタンを押す 141 | 142 | ![](addStoragePage.png) 143 | 144 | 4. 5分ほど待機する 145 | 146 | 5分ほど待機した後に[https://butter.takoyaki3.com/v0.0.0/root.json](https://butter.takoyaki3.com/v0.0.0/root.json)にアクセスしてROOTファイルを確認すると、ストレージが追加されたか確認できます。 147 | 148 | ```json 149 | { 150 | "pub_keys": [ 151 | { 152 | "pubkey": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/public_key.pem" 153 | } 154 | ], 155 | "hosts": [ 156 | "https://storage.app.takoyaki3.com", 157 | "https://butter.oozora283.com" 158 | ], 159 | "original_data": { 160 | "host": "https://pub-ad1f4a48b8ef46779b720e734b0c2e1d.r2.dev/v0.0.0/originalData/" 161 | }, 162 | "api_endpoints": [ 163 | "https://butter.hatano-yuuta7921.workers.dev/" 164 | ], 165 | "api_data_hosts": [ 166 | "https://storage.app.takoyaki3.com/" 167 | ], 168 | "last_update": "2023-05-06T19_02_00+09_00" 169 | } 170 | ``` 171 | 172 | ## 注意事項 173 | - SSL証明書の取得には、インターネットからのアクセスが必要です。 174 | - DNS設定が正しく行われていることを確認してください。 175 | - この設定を使用することで、提供されたサービスにHTTPS経由で安全にアクセスできるようになります。 176 | 177 | -------------------------------------------------------------------------------- /lib/test_v1.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Test 6 | 7 | 8 |

Test

9 | 10 | 114 | 115 | -------------------------------------------------------------------------------- /cmd/999_uploadOriginData.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "crypto" 7 | "crypto/rsa" 8 | "crypto/sha256" 9 | "crypto/x509" 10 | "encoding/pem" 11 | "io/ioutil" 12 | "log" 13 | "time" 14 | 15 | . "github.com/takoyaki-3/butter/cmd/helper" 16 | json "github.com/takoyaki-3/go-json" 17 | gos3 "github.com/takoyaki-3/go-s3" 18 | 19 | "os" 20 | "path/filepath" 21 | "strings" 22 | ) 23 | 24 | type OriginalData struct { 25 | DataList []OriginalDataItem `json:"data_list"` 26 | } 27 | type OriginalDataItem struct { 28 | Key string `json:"key"` 29 | } 30 | 31 | 32 | 33 | func VerifySignature(originalFilePath, signatureFilePath, publicKeyPath string) bool { 34 | // 公開鍵の読み込み 35 | publicKeyBytes, err := ioutil.ReadFile(publicKeyPath) 36 | if err != nil { 37 | log.Fatalf("Failed to load public key: %v", err) 38 | return false 39 | } 40 | 41 | // PEMデコード 42 | block, _ := pem.Decode(publicKeyBytes) 43 | if block == nil || block.Type != "PUBLIC KEY" { 44 | log.Fatalf("Failed to decode PEM block containing public key") 45 | return false 46 | } 47 | 48 | pub, err := x509.ParsePKIXPublicKey(block.Bytes) 49 | if err != nil { 50 | log.Fatalf("Failed to parse public key: %v", err) 51 | return false 52 | } 53 | 54 | rsaPub, ok := pub.(*rsa.PublicKey) 55 | if !ok { 56 | log.Fatalf("Not an RSA public key") 57 | return false 58 | } 59 | 60 | // 元のファイルのハッシュの計算 61 | originalData, err := ioutil.ReadFile(originalFilePath) 62 | if err != nil { 63 | log.Fatalf("Failed to read original file: %v", err) 64 | return false 65 | } 66 | hashed := sha256.Sum256(originalData) 67 | 68 | // 署名の読み込み 69 | signature, err := ioutil.ReadFile(signatureFilePath) 70 | if err != nil { 71 | log.Fatalf("Failed to read signature file: %v", err) 72 | return false 73 | } 74 | 75 | // 署名の検証 76 | err = rsa.VerifyPKCS1v15(rsaPub, crypto.SHA256, hashed[:], signature) 77 | if err != nil { 78 | log.Printf("Signature verification failed: %v", err) 79 | return false 80 | } 81 | 82 | return true 83 | } 84 | 85 | func CheckAllSignaturesInDir(directory, publicKeyPath string) error { 86 | return filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { 87 | if err != nil { 88 | return err 89 | } 90 | 91 | // ディレクトリと.sig拡張子のファイルはスキップ 92 | if info.IsDir() || strings.HasSuffix(path, ".sig") { 93 | return nil 94 | } 95 | 96 | signaturePath := path + ".sig" 97 | if _, err := os.Stat(signaturePath); os.IsNotExist(err) { 98 | return fmt.Errorf("signature file not found for %s", path) 99 | } 100 | 101 | // 署名を検証 102 | if !VerifySignature(path, signaturePath, publicKeyPath) { 103 | return fmt.Errorf("signature verification failed for %s", path) 104 | } 105 | 106 | return nil 107 | }) 108 | } 109 | 110 | func checkData()error{ 111 | directory := "./v0.0.0" 112 | publicKeyPath := "./public_Key.pem" // 公開鍵ファイルのパス 113 | 114 | if err := CheckAllSignaturesInDir(directory, publicKeyPath); err != nil { 115 | fmt.Println("Error:", err) 116 | } 117 | return nil 118 | } 119 | 120 | func main() { 121 | 122 | fmt.Println("start upload origin data") 123 | 124 | if err:=checkData();err!=nil{ 125 | log.Fatalln(err) 126 | return 127 | } 128 | 129 | sourceDir := "./v0.0.0/" 130 | now := time.Now().Format("20060102-150405") 131 | targetFile := now + ".tar" 132 | 133 | s3, err := gos3.NewSession("s3-conf.json") 134 | if err != nil { 135 | log.Fatalln(err) 136 | } 137 | // 0. index.htmlのコピー 138 | newFile, err := os.Create("v0.0.0/index.html") 139 | if err != nil { 140 | fmt.Println(err) 141 | } 142 | oldFile, err := os.Open("index.html") 143 | if err != nil { 144 | fmt.Println(err) 145 | } 146 | 147 | io.Copy(newFile, oldFile) 148 | 149 | // 1. 指定したディレクトリ内のファイルを.tar形式にまとめる 150 | err = CreateTarArchive(sourceDir, targetFile) 151 | if err != nil { 152 | fmt.Printf("Error creating tar archive: %v\n", err) 153 | return 154 | } 155 | 156 | // 2. 現在のバージョン情報を取得する 157 | var originalData OriginalData 158 | err = s3.DownloadToReaderFunc("v0.0.0/originalData/info.json", func(r io.Reader) error { 159 | err := json.LoadFromReader(r, &originalData) 160 | if err != nil { 161 | return err 162 | } 163 | return nil 164 | }) 165 | 166 | // 3. 新しいデータをアップロードする 167 | key := "v0.0.0/originalData/" + targetFile 168 | err = s3.UploadFromPath(targetFile, key) 169 | if err != nil { 170 | log.Fatalln(err) 171 | } 172 | 173 | // 4. ファイルとオリジナルデータバージョン情報をアップロードする 174 | originalData.DataList = append(originalData.DataList, OriginalDataItem{Key: targetFile}) 175 | key = "v0.0.0/originalData/info.json" 176 | str, _ := json.DumpToString(originalData) 177 | err = s3.UploadFromRaw([]byte(str), key) 178 | if err != nil { 179 | log.Fatalln(err) 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /cmd/999_uploadOriginData_feed.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "crypto" 7 | "crypto/rsa" 8 | "crypto/sha256" 9 | "crypto/x509" 10 | "encoding/pem" 11 | "io/ioutil" 12 | "log" 13 | "time" 14 | 15 | . "github.com/takoyaki-3/butter/cmd/helper" 16 | json "github.com/takoyaki-3/go-json" 17 | gos3 "github.com/takoyaki-3/go-s3" 18 | 19 | "os" 20 | "path/filepath" 21 | "strings" 22 | ) 23 | 24 | type OriginalData struct { 25 | DataList []OriginalDataItem `json:"data_list"` 26 | } 27 | type OriginalDataItem struct { 28 | Key string `json:"key"` 29 | } 30 | 31 | 32 | 33 | func VerifySignature(originalFilePath, signatureFilePath, publicKeyPath string) bool { 34 | // 公開鍵の読み込み 35 | publicKeyBytes, err := ioutil.ReadFile(publicKeyPath) 36 | if err != nil { 37 | log.Fatalf("Failed to load public key: %v", err) 38 | return false 39 | } 40 | 41 | // PEMデコード 42 | block, _ := pem.Decode(publicKeyBytes) 43 | if block == nil || block.Type != "PUBLIC KEY" { 44 | log.Fatalf("Failed to decode PEM block containing public key") 45 | return false 46 | } 47 | 48 | pub, err := x509.ParsePKIXPublicKey(block.Bytes) 49 | if err != nil { 50 | log.Fatalf("Failed to parse public key: %v", err) 51 | return false 52 | } 53 | 54 | rsaPub, ok := pub.(*rsa.PublicKey) 55 | if !ok { 56 | log.Fatalf("Not an RSA public key") 57 | return false 58 | } 59 | 60 | // 元のファイルのハッシュの計算 61 | originalData, err := ioutil.ReadFile(originalFilePath) 62 | if err != nil { 63 | log.Fatalf("Failed to read original file: %v", err) 64 | return false 65 | } 66 | hashed := sha256.Sum256(originalData) 67 | 68 | // 署名の読み込み 69 | signature, err := ioutil.ReadFile(signatureFilePath) 70 | if err != nil { 71 | log.Fatalf("Failed to read signature file: %v", err) 72 | return false 73 | } 74 | 75 | // 署名の検証 76 | err = rsa.VerifyPKCS1v15(rsaPub, crypto.SHA256, hashed[:], signature) 77 | if err != nil { 78 | log.Printf("Signature verification failed: %v", err) 79 | return false 80 | } 81 | 82 | return true 83 | } 84 | 85 | func CheckAllSignaturesInDir(directory, publicKeyPath string) error { 86 | return filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { 87 | if err != nil { 88 | return err 89 | } 90 | 91 | // ディレクトリと.sig拡張子のファイルはスキップ 92 | if info.IsDir() || strings.HasSuffix(path, ".sig") { 93 | return nil 94 | } 95 | 96 | signaturePath := path + ".sig" 97 | if _, err := os.Stat(signaturePath); os.IsNotExist(err) { 98 | return fmt.Errorf("signature file not found for %s", path) 99 | } 100 | 101 | // 署名を検証 102 | if !VerifySignature(path, signaturePath, publicKeyPath) { 103 | return fmt.Errorf("signature verification failed for %s", path) 104 | } 105 | 106 | return nil 107 | }) 108 | } 109 | 110 | func checkData()error{ 111 | directory := "./v1.0.0" 112 | publicKeyPath := "./public_Key.pem" // 公開鍵ファイルのパス 113 | 114 | if err := CheckAllSignaturesInDir(directory, publicKeyPath); err != nil { 115 | fmt.Println("Error:", err) 116 | } 117 | return nil 118 | } 119 | 120 | func main() { 121 | 122 | fmt.Println("start upload origin data feed") 123 | 124 | if err:=checkData();err!=nil{ 125 | log.Fatalln(err) 126 | return 127 | } 128 | 129 | sourceDir := "./v1.0.0/" 130 | now := time.Now().Format("20060102-150405") 131 | targetFile := now + ".tar" 132 | 133 | s3, err := gos3.NewSession("s3-conf.json") 134 | if err != nil { 135 | log.Fatalln(err) 136 | } 137 | // 0. index.htmlのコピー 138 | newFile, err := os.Create("v1.0.0/index.html") 139 | if err != nil { 140 | fmt.Println(err) 141 | } 142 | oldFile, err := os.Open("index.html") 143 | if err != nil { 144 | fmt.Println(err) 145 | } 146 | 147 | io.Copy(newFile, oldFile) 148 | 149 | // 1. 指定したディレクトリ内のファイルを.tar形式にまとめる 150 | err = CreateTarArchive(sourceDir, targetFile) 151 | if err != nil { 152 | fmt.Printf("Error creating tar archive: %v\n", err) 153 | return 154 | } 155 | 156 | // 2. 現在のバージョン情報を取得する 157 | var originalData OriginalData 158 | err = s3.DownloadToReaderFunc("v1.0.0/originalData/info.json", func(r io.Reader) error { 159 | err := json.LoadFromReader(r, &originalData) 160 | if err != nil { 161 | return err 162 | } 163 | return nil 164 | }) 165 | 166 | // 3. 新しいデータをアップロードする 167 | key := "v1.0.0/originalData/" + targetFile 168 | err = s3.UploadFromPath(targetFile, key) 169 | if err != nil { 170 | log.Fatalln(err) 171 | } 172 | 173 | // 4. ファイルとオリジナルデータバージョン情報をアップロードする 174 | originalData.DataList = append(originalData.DataList, OriginalDataItem{Key: targetFile}) 175 | key = "v1.0.0/originalData/info.json" 176 | str, _ := json.DumpToString(originalData) 177 | err = s3.UploadFromRaw([]byte(str), key) 178 | if err != nil { 179 | log.Fatalln(err) 180 | } 181 | } 182 | -------------------------------------------------------------------------------- /cmd/receiver.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "crypto" 7 | "crypto/rsa" 8 | "crypto/sha256" 9 | "crypto/x509" 10 | "encoding/pem" 11 | "fmt" 12 | "io" 13 | "net" 14 | "os" 15 | "path/filepath" 16 | "strconv" 17 | "strings" 18 | ) 19 | 20 | func main() { 21 | listenAddr := "0.0.0.0:8001" 22 | destDir := "v0.0.0" 23 | 24 | // サーバーを起動して待機 25 | listener, err := net.Listen("tcp", listenAddr) 26 | if err != nil { 27 | fmt.Printf("Error listening on %s: %v\n", listenAddr, err) 28 | return 29 | } 30 | defer listener.Close() 31 | 32 | fmt.Printf("Listening on %s\n", listenAddr) 33 | 34 | conn, err := listener.Accept() 35 | if err != nil { 36 | fmt.Printf("Error accepting connection: %v\n", err) 37 | return 38 | } 39 | defer conn.Close() 40 | 41 | fmt.Println("Client connected.") 42 | 43 | // 公開鍵のサイズを受信 44 | pubKeySizeBuffer := make([]byte, 20) 45 | _, err = conn.Read(pubKeySizeBuffer) 46 | if err != nil { 47 | fmt.Printf("Error reading public key size: %v\n", err) 48 | return 49 | } 50 | 51 | pubKeySizeStr := strings.TrimSpace(string(pubKeySizeBuffer)) // 余計な空白文字を取り除く 52 | pubKeySize, err := strconv.Atoi(pubKeySizeStr) 53 | if err != nil { 54 | fmt.Printf("Error parsing public key size: %v\n", err) 55 | return 56 | } 57 | 58 | // 確認メッセージを送信 59 | _, err = conn.Write([]byte("confirm")) 60 | if err != nil { 61 | fmt.Printf("Error sending confirmation: %v\n", err) 62 | return 63 | } 64 | 65 | // 公開鍵を受信 66 | pubKeyPem := make([]byte, pubKeySize) 67 | _, err = conn.Read(pubKeyPem) 68 | if err != nil { 69 | fmt.Printf("Error reading public key: %v\n", err) 70 | return 71 | } 72 | 73 | block, _ := pem.Decode(pubKeyPem) 74 | pubKey, err := x509.ParsePKIXPublicKey(block.Bytes) 75 | if err != nil { 76 | fmt.Printf("Error parsing public key: %v\n", err) 77 | return 78 | } 79 | 80 | rsaPubKey, ok := pubKey.(*rsa.PublicKey) 81 | if !ok { 82 | fmt.Println("Invalid public key") 83 | return 84 | } 85 | fmt.Println(string(pubKeyPem)) 86 | 87 | // ファイルサイズを受信 88 | fileSizeBuffer := make([]byte, 20) 89 | _, err = conn.Read(fileSizeBuffer) 90 | if err != nil { 91 | fmt.Printf("Error reading file size: %v\n", err) 92 | return 93 | } 94 | 95 | fileSize, err := strconv.ParseInt(strings.TrimSpace(string(fileSizeBuffer)), 10, 64) 96 | if err != nil { 97 | fmt.Printf("Error parsing file size: %v\n", err) 98 | return 99 | } 100 | 101 | // .tarファイルを受信 102 | tarBuffer := bytes.NewBuffer(make([]byte, 0, fileSize)) 103 | _, err = io.CopyN(tarBuffer, conn, fileSize) 104 | if err != nil { 105 | fmt.Printf("Error receiving tar file: %v\n", err) 106 | return 107 | } 108 | 109 | // 電子署名を受信 110 | signature := make([]byte, 256) 111 | _, err = conn.Read(signature) 112 | if err != nil { 113 | fmt.Printf("Error receiving signature: %v\n", err) 114 | return 115 | } 116 | 117 | // 3. 電子署名を検証する 118 | isValid, err := verifySignature(tarBuffer.Bytes(), signature, rsaPubKey) 119 | if err != nil { 120 | fmt.Printf("Error verifying signature: %v\n", err) 121 | return 122 | } 123 | 124 | if !isValid { 125 | fmt.Println("Invalid signature. Aborting.") 126 | return 127 | } 128 | 129 | fmt.Println("Signature verified.") 130 | 131 | // 4. 電子署名が正しい場合、サーバーが.tarファイルを特定のディレクトリに展開する 132 | err = extractTarArchive(tarBuffer, destDir) 133 | if err != nil { 134 | fmt.Printf("Error extracting tar archive: %v\n", err) 135 | return 136 | } 137 | 138 | fmt.Println("File received and extracted successfully.") 139 | } 140 | 141 | // 電子署名を検証する 142 | func verifySignature(data, signature []byte, pubKey *rsa.PublicKey) (bool, error) { 143 | hash := sha256.Sum256(data) 144 | 145 | err := rsa.VerifyPKCS1v15(pubKey, crypto.SHA256, hash[:], signature) 146 | if err != nil { 147 | return false, err 148 | } 149 | 150 | return true, nil 151 | } 152 | 153 | // .tarファイルを特定のディレクトリに展開する 154 | func extractTarArchive(tarBuffer *bytes.Buffer, destDir string) error { 155 | tarReader := tar.NewReader(tarBuffer) 156 | 157 | for { 158 | header, err := tarReader.Next() 159 | if err == io.EOF { 160 | break 161 | } 162 | 163 | if err != nil { 164 | return err 165 | } 166 | 167 | target := filepath.Join(destDir, header.Name) 168 | 169 | switch header.Typeflag { 170 | case tar.TypeDir: 171 | if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil { 172 | return err 173 | } 174 | case tar.TypeReg: 175 | file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) 176 | if err != nil { 177 | return err 178 | } 179 | 180 | _, err = io.Copy(file, tarReader) 181 | file.Close() 182 | 183 | if err != nil { 184 | return err 185 | } 186 | default: 187 | return fmt.Errorf("unsupported type: %v", header.Typeflag) 188 | } 189 | } 190 | 191 | return nil 192 | } 193 | -------------------------------------------------------------------------------- /cmd/7_add_datalist.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | "net/http" 7 | "strings" 8 | 9 | json "github.com/takoyaki-3/go-json" 10 | ) 11 | 12 | type DataItem struct { 13 | GtfsID string `json:"gtfs_id"` 14 | FeedID string `json:"feed_id"` 15 | AgencyID string `json:"agency_id"` 16 | Name string `json:"name"` 17 | License string `json:"license"` 18 | LicenseURL string `json:"licenseUrl"` 19 | ProviderURL string `json:"providerUrl"` 20 | ProviderName string `json:"providerName"` 21 | ProviderAgencyName string `json:"providerAgencyName"` 22 | Memo string `json:"memo"` 23 | UpdatedAt string `json:"updatedAt"` 24 | AlertURL string `json:"Alert_url,omitempty"` 25 | TripUpdateURL string `json:"TripUpdate_url,omitempty"` 26 | VehiclePositionURL string `json:"VehiclePosition_url,omitempty"` 27 | } 28 | 29 | type DataList struct { 30 | Updated string `json:"updated"` 31 | Data []DataItem `json:"data_list"` 32 | } 33 | 34 | type Data []struct { 35 | AgentName string `json:"事業者名,omitempty"` 36 | URL string `json:"事業者名_url,omitempty"` 37 | Prefecture string `json:"都道府県,omitempty"` 38 | GTFS string `json:"GTFSフィード名,omitempty"` 39 | License string `json:"ライセンス,omitempty"` 40 | LicenseURL string `json:"ライセンス_url,omitempty"` 41 | URLs string `json:"URLs,omitempty"` 42 | GTFSURL string `json:"GTFS_url,omitempty"` 43 | StartDate string `json:"最新GTFS開始日,omitempty"` 44 | EndDate string `json:"最新GTFS終了日,omitempty"` 45 | UpdateDate string `json:"最終更新日,omitempty"` 46 | Detail string `json:"詳細,omitempty"` 47 | GtfsID string `json:"gtfs_id,omitempty"` 48 | FeedID string `json:"feed_id,omitempty"` 49 | AlertURL string `json:"Alert_url,omitempty"` 50 | TripUpdateURL string `json:"TripUpdate_url,omitempty"` 51 | VehiclePositionURL string `json:"VehiclePosition_url,omitempty"` 52 | } 53 | 54 | func main() { 55 | checkCORSandAddList("v0.0.0", "data_v0.json") 56 | checkCORSandAddList("v1.0.0", "data_v1.json") 57 | } 58 | 59 | func checkCORSandAddList(root string, filename string) { 60 | data := Data{} 61 | json.LoadFromPath(filename, &data) 62 | 63 | t := time.Now() 64 | datalist := DataList{ 65 | Updated: t.Format("2006-01-02T15_04_05+09_00"), 66 | } 67 | 68 | for _, v := range data { 69 | time.Sleep(time.Second) 70 | t, err := time.Parse("2006-01-02", v.UpdateDate) 71 | if err != nil { 72 | continue 73 | } 74 | 75 | if v.VehiclePositionURL != "" { 76 | // ODPT API Keyが含まれるか確認 77 | if strings.Contains(v.VehiclePositionURL,"acl:consumerKey=") { 78 | // ODPT API Keyを削除 79 | v.GTFSURL = strings.Split(v.GTFSURL,"acl:consumerKey=")[0] 80 | v.AlertURL = strings.Split(v.AlertURL,"acl:consumerKey=")[0] 81 | v.TripUpdateURL = strings.Split(v.TripUpdateURL,"acl:consumerKey=")[0] 82 | v.VehiclePositionURL = strings.Split(v.VehiclePositionURL,"acl:consumerKey=")[0] 83 | } else { 84 | // CROSが許可されているか確認 85 | 86 | // HTTPクライアントの設定 87 | client := &http.Client{ 88 | Timeout: time.Second * 10, 89 | } 90 | 91 | url := v.VehiclePositionURL 92 | 93 | // GETリクエストの作成 94 | req, err := http.NewRequest("GET", url, nil) 95 | if err != nil { 96 | fmt.Printf("Failed to create request for %s: %v\n", url, err) 97 | continue 98 | } 99 | 100 | // リクエストの実行 101 | resp, err := client.Do(req) 102 | if err != nil { 103 | fmt.Printf("Failed to perform request for %s: %v\n", url, err) 104 | continue 105 | } 106 | defer resp.Body.Close() 107 | 108 | // CORS設定をチェック 109 | corsHeader := resp.Header.Get("Access-Control-Allow-Origin") 110 | if corsHeader == "*" { 111 | // CROS OK 112 | fmt.Printf("The URL %s has CORS settings set to '*'.\n", url) 113 | } else { 114 | fmt.Printf("The URL %s does not have CORS settings set to '*'. It is set to: %s\n", url, corsHeader) 115 | v.VehiclePositionURL = "https://cros-proxy.butter.takoyaki3.com/" + v.VehiclePositionURL; 116 | } 117 | } 118 | } 119 | 120 | datalist.Data = append(datalist.Data, DataItem{ 121 | GtfsID: v.GtfsID, 122 | FeedID: v.FeedID, 123 | // AgencyID: v., 124 | Name: v.AgentName, 125 | License: v.License, 126 | LicenseURL: v.LicenseURL, 127 | ProviderURL: v.GTFSURL, 128 | ProviderName: "", 129 | ProviderAgencyName: "", 130 | UpdatedAt: t.Format("2006-01-02T15_04_05+09_00"), 131 | AlertURL: v.AlertURL, 132 | TripUpdateURL: v.TripUpdateURL, 133 | VehiclePositionURL: v.VehiclePositionURL, 134 | }) 135 | } 136 | 137 | err := json.DumpToFile(datalist, root+"/datalist.json") 138 | fmt.Println(err) 139 | } 140 | -------------------------------------------------------------------------------- /storage/ftps.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/tar" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "log" 9 | "path/filepath" 10 | "strings" 11 | "net/http" 12 | "io/ioutil" 13 | 14 | json "github.com/takoyaki-3/go-json" 15 | "github.com/jlaffaye/ftp" 16 | ) 17 | 18 | type Config struct { 19 | FTPHost string `json:"ftp_host"` 20 | FTPUser string `json:"ftp_user"` 21 | FTPPass string `json:"ftp_pass"` 22 | FTPPort string `json:"ftp_port"` 23 | Prefix string `json:"prefix"` 24 | } 25 | 26 | type PublicKey struct { 27 | Pubkey string `json:"pubkey"` 28 | } 29 | type OriginalData struct { 30 | Host string `json:"host"` 31 | } 32 | type Root struct { 33 | PubKeys []PublicKey `json:"pub_keys"` 34 | Hosts []string `json:"hosts"` 35 | OriginalData OriginalData `json:"original_data"` 36 | LastUpdate string `json:"last_update"` 37 | } 38 | 39 | type Info struct { 40 | DataList []OriginalDataItem `json:"data_list"` 41 | } 42 | type OriginalDataItem struct { 43 | Key string `json:"key"` 44 | } 45 | 46 | func main() { 47 | var config Config 48 | err := json.LoadFromPath("./config.json", &config) 49 | if err != nil { 50 | log.Fatalln(err) 51 | } 52 | 53 | var root Root 54 | rootData, err := downloadFile("https://butter.takoyaki3.com/v0.0.0/root.json") 55 | if err != nil { 56 | log.Fatalln(err) 57 | } 58 | err = json.LoadFromString(string(rootData), &root) 59 | if err != nil { 60 | log.Fatalln(err) 61 | } 62 | 63 | var info Info 64 | infoData, err := downloadFile(root.OriginalData.Host + "info.json") 65 | if err != nil { 66 | log.Fatalln(err) 67 | } 68 | err = json.LoadFromString(string(infoData), &info) 69 | if err != nil { 70 | log.Fatalln(err) 71 | } 72 | 73 | infoPath := "info.json" 74 | var oldInfo Info 75 | err = json.LoadFromPath(infoPath, &oldInfo) 76 | if err == nil && info.DataList[len(info.DataList)-1].Key == oldInfo.DataList[len(oldInfo.DataList)-1].Key { 77 | log.Println("No updates found. Exiting.") 78 | return 79 | } 80 | 81 | err = ioutil.WriteFile(infoPath, infoData, 0644) 82 | if err != nil { 83 | log.Fatalln(err) 84 | } 85 | 86 | tarData, err := downloadFile(root.OriginalData.Host + info.DataList[len(info.DataList)-1].Key) 87 | if err != nil { 88 | log.Fatal(err) 89 | } 90 | 91 | err = uploadTarFilesToFTP(config.FTPHost, config.FTPPort, config.FTPUser, config.FTPPass, config.Prefix, tarData) 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | } 96 | func uploadTarFilesToFTP(host, port, user, pass, prefix string, tarData []byte) error { 97 | // Connect to FTP server 98 | conn, err := ftp.Dial(fmt.Sprintf("%s:%s", host, port)) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | defer conn.Quit() 104 | 105 | // Login to FTP server 106 | err = conn.Login(user, pass) 107 | if err != nil { 108 | return err 109 | } 110 | 111 | counter := 0 112 | 113 | // Create a buffer with the .tar data and read it 114 | tarReader := tar.NewReader(bytes.NewReader(tarData)) 115 | 116 | for { 117 | header, err := tarReader.Next() 118 | 119 | if err == io.EOF { 120 | break 121 | } else if err != nil { 122 | return err 123 | } 124 | 125 | if header.Typeflag == tar.TypeReg { 126 | // Check and create the directory if it doesn't exist 127 | dir := filepath.Dir(prefix + header.Name) 128 | err := createDirIfNotExists(conn, dir) 129 | if err != nil { 130 | return err 131 | } 132 | 133 | var buf bytes.Buffer 134 | _, err = io.Copy(&buf, tarReader) 135 | if err != nil { 136 | return err 137 | } 138 | 139 | // Upload file to FTP server 140 | err = conn.Stor(prefix + header.Name, &buf) 141 | if err != nil { 142 | return err 143 | } 144 | 145 | fmt.Println(counter,"Uploaded:", header.Name) 146 | counter++ 147 | } 148 | } 149 | 150 | return nil 151 | } 152 | 153 | var cash map[string]bool 154 | 155 | func createDirIfNotExists(conn *ftp.ServerConn, dir string) error { 156 | if cash == nil { 157 | cash = map[string]bool{} 158 | } 159 | if _, ok := cash[dir]; ok { 160 | return nil 161 | } 162 | 163 | parts := strings.Split(dir, "/") 164 | currentPath := "" 165 | for _, part := range parts { 166 | currentPath = filepath.Join(currentPath, part) 167 | _, err := conn.List(currentPath) 168 | if err != nil { 169 | err := conn.MakeDir(currentPath) 170 | if err != nil { 171 | // Check if the error message contains "550" and "File exists" or "Directory exists" 172 | if strings.Contains(err.Error(), "550") && (strings.Contains(err.Error(), "File exists") || strings.Contains(err.Error(), "Directory exists")) { 173 | // If the error is related to the file or directory already existing, skip this iteration 174 | continue 175 | } else { 176 | return err 177 | } 178 | } 179 | } 180 | } 181 | cash[dir] = true 182 | return nil 183 | } 184 | 185 | func downloadFile(url string) ([]byte, error) { 186 | fmt.Println(url) 187 | resp, err := http.Get(url) 188 | if err != nil { 189 | return nil, err 190 | } 191 | defer resp.Body.Close() 192 | 193 | if resp.StatusCode != http.StatusOK { 194 | return nil, fmt.Errorf("failed to download file, status code: %d", resp.StatusCode) 195 | } 196 | 197 | data, err := ioutil.ReadAll(resp.Body) 198 | if err != nil { 199 | return nil, err 200 | } 201 | 202 | return data, nil 203 | } 204 | -------------------------------------------------------------------------------- /cmd/2_line2obj.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | data_v0 = [] 4 | data_v1 = [] 5 | 6 | with open('dataList.txt', 'r', encoding='utf-8') as f: 7 | lines = f.readlines() # ファイルのすべての行を読み込む 8 | line_count = len(lines) 9 | i = 0 10 | while i < line_count: 11 | if lines[i].strip() == '-------------': # 区切り線の場合 12 | d_v0 = {} # 1つの事業者の情報を格納する辞書 13 | d_v1 = {} # 1つの事業者の情報を格納する辞書 14 | i += 1 15 | while i < line_count and lines[i].strip() != '-------------': 16 | parts = lines[i].strip().split(':', 1) # キーと値に分割する 17 | if len(parts) == 2: 18 | d_v0[parts[0]] = parts[1] # 辞書に追加する 19 | d_v1[parts[0]] = parts[1] # 辞書に追加する 20 | i += 1 21 | 22 | # IDの設定 23 | # 都営バス・GTFSデータレポジトリ・ODPT一般公開・ODPT限定公開ごとにURLからgtfs_idを生成する 24 | if 'GTFS_url' in d_v1: 25 | print(d_v1) 26 | # 都営バス 27 | if d_v1['GTFS_url'] == 'https://api-public.odpt.org/api/v4/files/Toei/data/ToeiBus-GTFS.zip': 28 | d_v1['gtfs_id'] = d_v1['feed_id'] 29 | d_v1['feed_id'] = d_v1['feed_id'].split('_FEEDID_')[1] 30 | d_v1['organization_id'] = d_v1['feed_id'].split('_FEEDID_')[0] 31 | print('gtfs_id:',d_v1['gtfs_id']) 32 | print('feed_id',d_v1['feed_id']) 33 | d_v0['gtfs_id'] = d_v1['gtfs_id'].split('_FEEDID_')[0] 34 | d_v0['feed_id'] = d_v1['feed_id'] 35 | d_v0['organization_id'] = d_v1['organization_id'] 36 | # GTFSデータレポジトリ 37 | elif 'https://api.gtfs-data.jp/v2/organizations/' in d_v1['GTFS_url']: 38 | d_v1['gtfs_id'] = d_v1['feed_id'] 39 | d_v1['feed_id'] = d_v1['feed_id'].split('_FEEDID_')[0] 40 | d_v1['organization_id'] = d_v1['feed_id'].split('_FEEDID_')[0] 41 | print('gtfs_id:',d_v1['gtfs_id']) 42 | print('feed_id',d_v1['feed_id']) 43 | d_v0['gtfs_id'] = d_v1['gtfs_id'].split('_FEEDID_')[0] 44 | d_v0['feed_id'] = d_v1['feed_id'] 45 | d_v0['organization_id'] = d_v1['organization_id'] 46 | # ODPT一般公開 47 | elif 'https://api-public.odpt.org/api/v4/files/odpt/' in d_v1['GTFS_url']: 48 | gtfs_id = d_v1['GTFS_url'].split('https://api-public.odpt.org/api/v4/files/odpt/')[1] 49 | feed_id = gtfs_id.split('/')[1].split('.zip')[0] 50 | gtfs_id = gtfs_id.split('/')[0] 51 | d_v1['feed_id'] = feed_id 52 | d_v1['gtfs_id'] = gtfs_id + "_FEEDID_" + feed_id 53 | d_v1['organization_id'] = d_v1['feed_id'].split('_FEEDID_')[0] 54 | print('gtfs_id:',d_v1['gtfs_id']) 55 | print('feed_id',d_v1['feed_id']) 56 | d_v0['gtfs_id'] = d_v1['gtfs_id'].split('_FEEDID_')[0] 57 | d_v0['feed_id'] = d_v1['feed_id'] 58 | d_v0['organization_id'] = d_v1['organization_id'] 59 | # ODPT限定公開 60 | elif 'https://api.odpt.org/api/v4/files/odpt/' in d_v1['GTFS_url']: 61 | gtfs_id = d_v1['GTFS_url'].split('https://api.odpt.org/api/v4/files/odpt/')[1] 62 | feed_id = gtfs_id.split('/')[1].split('.zip')[0] 63 | gtfs_id = gtfs_id.split('/')[0] 64 | d_v1['gtfs_id'] = gtfs_id + "_FEEDID_" + feed_id 65 | d_v1['feed_id'] = d_v1['gtfs_id'].split('_FEEDID_')[1] 66 | d_v1['organization_id'] = d_v1['feed_id'].split('_FEEDID_')[0] 67 | print('gtfs_id:',d_v1['gtfs_id']) 68 | print('feed_id',d_v1['feed_id']) 69 | d_v0['gtfs_id'] = d_v1['gtfs_id'].split('_FEEDID_')[0] 70 | d_v0['feed_id'] = d_v1['feed_id'] 71 | d_v0['organization_id'] = d_v1['organization_id'] 72 | elif 'https://api.odpt.org/api/v4/files/' in d_v1['GTFS_url']: 73 | gtfs_id = d_v1['GTFS_url'].split('https://api.odpt.org/api/v4/files/')[1] 74 | feed_id = gtfs_id.split('/')[1].split('.zip')[0] 75 | gtfs_id = gtfs_id.split('/')[0] 76 | d_v1['gtfs_id'] = gtfs_id + "_FEEDID_" + feed_id 77 | d_v1['feed_id'] = d_v1['gtfs_id'].split('_FEEDID_')[1] 78 | d_v1['organization_id'] = d_v1['feed_id'].split('_FEEDID_')[0] 79 | print('gtfs_id:',d_v1['gtfs_id']) 80 | print('feed_id',d_v1['feed_id']) 81 | d_v0['gtfs_id'] = d_v1['gtfs_id'].split('_FEEDID_')[0] 82 | d_v0['feed_id'] = d_v1['feed_id'] 83 | d_v0['organization_id'] = d_v1['organization_id'] 84 | print('d:',d_v1) 85 | if 'gtfs_id' in d_v1: 86 | data_v1.append(d_v1) # リストに追加する 87 | if 'gtfs_id' in d_v0: 88 | data_v0.append(d_v0) 89 | 90 | # オブジェクトをJSON形式に変換してファイルに書き込む 91 | with open('data_v1.json', 'w', encoding='utf-8') as f_v1: 92 | json.dump(data_v1, f_v1, ensure_ascii=False) 93 | 94 | # オブジェクトをJSON形式に変換してファイルに書き込む 95 | with open('data_v0.json', 'w', encoding='utf-8') as f_v0: 96 | json.dump(data_v0, f_v0, ensure_ascii=False) 97 | 98 | -------------------------------------------------------------------------------- /cmd/go.sum: -------------------------------------------------------------------------------- 1 | github.com/aws/aws-sdk-go v1.44.61 h1:NcpLSS3Z0MiVQIYugx4I40vSIEEAXT0baO684ExNRco= 2 | github.com/aws/aws-sdk-go v1.44.61/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= 3 | github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= 4 | github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= 7 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 8 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 9 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 10 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 11 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 12 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 13 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 14 | github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= 15 | github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 16 | github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= 17 | github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y= 18 | github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 19 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 20 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 21 | github.com/qedus/osmpbf v1.2.0 h1:yRm5ECkiUsN9sA+UN9yNnm64AVW2OYhOCb+gBa1FYCU= 22 | github.com/qedus/osmpbf v1.2.0/go.mod h1:Cfv6JyqTZ72BjoW9FyFBQOC2DYJbL78yw+DLhBvSH+M= 23 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 24 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 25 | github.com/takoyaki-3/go-csv-tag/v3 v3.0.2 h1:pQab6q6qolQMtP3aoGKPIQXu5AYFgLuIMGMsIWKd0HE= 26 | github.com/takoyaki-3/go-csv-tag/v3 v3.0.2/go.mod h1:9ZK/Ry0iZm5T+ihPuvYrWCJo5TkKnIbUx82RLSb1HoM= 27 | github.com/takoyaki-3/go-file-tool v0.0.1 h1:PvWRm/XPrBKsQXW6KhdwLVvNT14YYwIiQRVA6M8vZXk= 28 | github.com/takoyaki-3/go-file-tool v0.0.1/go.mod h1:JsjDaZogx/qIQNlX4Dz+EL+bdYCEDLTZwU+WR+P+mhM= 29 | github.com/takoyaki-3/go-geojson v0.0.1 h1:uRo5mszCqPUYaQ3CpJk4q86WA7ZjaFzJ+3it3caHoNM= 30 | github.com/takoyaki-3/go-geojson v0.0.1/go.mod h1:0Yd9ZVbTx+59M//yuOl7FrTTcN8dgx3JSXevt4Nxatg= 31 | github.com/takoyaki-3/go-gtfs/v2 v2.0.7 h1:4k0Djggf/C+qxPoRsXaYQfbiG552fNijtnRp1tfqkJk= 32 | github.com/takoyaki-3/go-gtfs/v2 v2.0.7/go.mod h1:ydOx7BrOXP3erPoGh0Jc+gyvuyw3iDTj+BNFjk6/uL0= 33 | github.com/takoyaki-3/go-json v0.0.0-20211221023225-18c8e7ac7ccb h1:+ho64ruBJraFsn8q8UL6jPhPRQRHrLMikXmcxQ4vszQ= 34 | github.com/takoyaki-3/go-json v0.0.0-20211221023225-18c8e7ac7ccb/go.mod h1:39fxS11XWIvygjbgeXg4bpyvAxLZ2lr1Cv+9DbbQ9Ew= 35 | github.com/takoyaki-3/go-json v0.0.2 h1:IRG20KZUFt0ZnYikI6ob+/8cSjTA41rTaaA5cVAKKLo= 36 | github.com/takoyaki-3/go-json v0.0.2/go.mod h1:39fxS11XWIvygjbgeXg4bpyvAxLZ2lr1Cv+9DbbQ9Ew= 37 | github.com/takoyaki-3/go-map/v2 v2.0.3 h1:ovQFSlqs7msTsC+3NvHncqF09+Jep8yDvku/TqLnHrc= 38 | github.com/takoyaki-3/go-map/v2 v2.0.3/go.mod h1:2Se8GjYwKasvDwrK2p3Ei7lkpAGULTWoy56Hnk0VK/w= 39 | github.com/takoyaki-3/go-s3 v0.0.9 h1:o2Dx3X2WlUIWewbi8uRpDXj3xZFJqkhi3K4XolM2QrQ= 40 | github.com/takoyaki-3/go-s3 v0.0.9/go.mod h1:rp+Zx5p5vMScu9sbO2ZcNdCdghV9AaVnY5Lomg5QgxQ= 41 | github.com/takoyaki-3/goc v0.0.1 h1:A8+ffyd/BGRz0vqfO9l4r0XOY4Vn7X0QBj9zXhKFink= 42 | github.com/takoyaki-3/goc v0.0.1/go.mod h1:1G2vCfYcBkWXFVcgXf88O1JNcldnoBg6hX6lGhoNLAc= 43 | github.com/uber/h3-go v3.0.1+incompatible h1:RVpBm8qd7mM94YuIhNQfCXpVj6mPY6gNsVstDg1FvjY= 44 | github.com/uber/h3-go v3.0.1+incompatible/go.mod h1:66a2M4rQlf+dtkTWj3bHoLFgDT/Rt4kLT8dMuEQVQvw= 45 | golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 46 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 47 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= 48 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 49 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 50 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= 51 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 52 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 53 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 54 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 55 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 56 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 57 | google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= 58 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 59 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 60 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 61 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 62 | -------------------------------------------------------------------------------- /storage/dataUpdater.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/tar" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | "time" 13 | "bytes" 14 | 15 | "github.com/joho/godotenv" 16 | json "github.com/takoyaki-3/go-json" 17 | ) 18 | 19 | type PublicKey struct { 20 | Pubkey string `json:"pubkey"` 21 | } 22 | type OriginalData struct { 23 | Host string `json:"host"` 24 | } 25 | type Root struct { 26 | PubKeys []PublicKey `json:"pub_keys"` 27 | Hosts []string `json:"hosts"` 28 | OriginalData OriginalData `json:"original_data"` 29 | LastUpdate string `json:"last_update"` 30 | } 31 | 32 | type Info struct { 33 | DataList []OriginalDataItem `json:"data_list"` 34 | } 35 | type OriginalDataItem struct { 36 | Key string `json:"key"` 37 | } 38 | 39 | func monitorFile(filePath, timeFilePath string, done chan bool) { 40 | ticker := time.NewTicker(1 * time.Minute) 41 | var lastFileContent []byte 42 | lastFileContent, _ = ioutil.ReadFile(filePath) 43 | 44 | for { 45 | select { 46 | case <-ticker.C: 47 | fileContent, err := ioutil.ReadFile(filePath) 48 | if err != nil { 49 | log.Printf("Error reading file: %v", err) 50 | continue 51 | } 52 | 53 | if !bytes.Equal(fileContent, lastFileContent) { 54 | log.Printf("File %s has been modified", filePath) 55 | lastFileContent = fileContent 56 | time.AfterFunc(10*time.Second, func() { 57 | done <- true 58 | }) 59 | } 60 | 61 | // 現在の時刻を取得し、ファイルに書き込む 62 | currentTime := time.Now().Format(time.RFC3339) 63 | if err := ioutil.WriteFile(timeFilePath, []byte(currentTime), 0644); err != nil { 64 | log.Printf("Error writing last checked time: %v", err) 65 | } 66 | } 67 | } 68 | } 69 | 70 | func main() { 71 | 72 | BUTTER_ROOT_URL := "https://butter.takoyaki3.com/v0.0.0/root.json" 73 | 74 | err := godotenv.Load() 75 | if err == nil { 76 | BUTTER_ROOT_URL = os.Getenv("BUTTER_ROOT_URL_V0") 77 | } else { 78 | fmt.Println(".env file not found") 79 | } 80 | fmt.Println(BUTTER_ROOT_URL) 81 | 82 | done := make(chan bool) 83 | go monitorFile("./dataUpdater.go","./dataUpdaterLastCheckedTime.txt", done) 84 | 85 | go func() { 86 | 87 | isFirst := true 88 | for { 89 | var root Root 90 | rootData, err := downloadFile(BUTTER_ROOT_URL) 91 | if err != nil { 92 | log.Fatalln(err) 93 | } 94 | err = json.LoadFromString(string(rootData), &root) 95 | if err != nil { 96 | log.Fatalln(err) 97 | } 98 | 99 | var info Info 100 | infoData, err := downloadFile(root.OriginalData.Host + "info.json") 101 | if err != nil { 102 | log.Fatalln(err) 103 | } 104 | err = json.LoadFromString(string(infoData), &info) 105 | if err != nil { 106 | log.Fatalln(err) 107 | } 108 | 109 | infoPath := "info.json" 110 | var oldInfo Info 111 | err = json.LoadFromPath(infoPath, &oldInfo) 112 | fmt.Println("info.json:",oldInfo) 113 | if !isFirst { 114 | if err == nil { 115 | if info.DataList[len(info.DataList)-1].Key == oldInfo.DataList[len(oldInfo.DataList)-1].Key { 116 | log.Println("No updates found. Exiting.") 117 | time.Sleep(time.Minute*5) 118 | continue 119 | } 120 | } 121 | } 122 | isFirst = false 123 | fmt.Println("start download. ["+root.OriginalData.Host + info.DataList[len(info.DataList)-1].Key+"]") 124 | 125 | err = ioutil.WriteFile(infoPath, infoData, 0644) 126 | if err != nil { 127 | log.Fatalln(err) 128 | } 129 | 130 | err = DownloadAndExtractTar(root.OriginalData.Host + info.DataList[len(info.DataList)-1].Key, "./" + info.DataList[len(info.DataList)-1].Key) 131 | if err != nil { 132 | log.Fatal(err) 133 | } 134 | 135 | fmt.Println("dev") 136 | 137 | err = os.Rename("./public","./old") 138 | fmt.Println(err) 139 | err = os.Rename("./" + info.DataList[len(info.DataList)-1].Key, "public") 140 | fmt.Println(err) 141 | if err != nil { 142 | continue 143 | } 144 | err = os.RemoveAll("old") 145 | fmt.Println(err) 146 | } 147 | }() 148 | 149 | <-done 150 | log.Printf("File has been modified. Shutting down in 10 seconds.") 151 | time.Sleep(10 * time.Second) 152 | log.Printf("Shutting down server.") 153 | } 154 | func DownloadAndExtractTar(url, dest string) error { 155 | // Check if output directory exists, create it if not 156 | if _, err := os.Stat(dest); os.IsNotExist(err) { 157 | err = os.MkdirAll(dest, 0755) 158 | if err != nil { 159 | return fmt.Errorf("failed to create destination directory: %v", err) 160 | } 161 | } 162 | 163 | resp, err := http.Get(url) 164 | if err != nil { 165 | return err 166 | } 167 | defer resp.Body.Close() 168 | 169 | if resp.StatusCode != http.StatusOK { 170 | return fmt.Errorf("failed to download %s: %d %s", url, resp.StatusCode, resp.Status) 171 | } 172 | 173 | tr := tar.NewReader(resp.Body) 174 | 175 | for { 176 | header, err := tr.Next() 177 | if err == io.EOF { 178 | break 179 | } 180 | if err != nil { 181 | return err 182 | } 183 | 184 | target := filepath.Join(dest, header.Name) 185 | 186 | switch header.Typeflag { 187 | case tar.TypeDir: 188 | if _, err := os.Stat(target); err != nil { 189 | if err := os.MkdirAll(target, 0755); err != nil { 190 | return err 191 | } 192 | } 193 | case tar.TypeReg: 194 | file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) 195 | if err != nil { 196 | return err 197 | } 198 | defer file.Close() 199 | 200 | if _, err := io.Copy(file, tr); err != nil { 201 | return err 202 | } 203 | default: 204 | return fmt.Errorf("unknown type: %c in %s", header.Typeflag, header.Name) 205 | } 206 | } 207 | 208 | return nil 209 | } 210 | 211 | func downloadFile(url string) ([]byte, error) { 212 | fmt.Println(url) 213 | resp, err := http.Get(url) 214 | if err != nil { 215 | return nil, err 216 | } 217 | defer resp.Body.Close() 218 | 219 | if resp.StatusCode != http.StatusOK { 220 | return nil, fmt.Errorf("failed to download file, status code: %d", resp.StatusCode) 221 | } 222 | 223 | data, err := ioutil.ReadAll(resp.Body) 224 | if err != nil { 225 | return nil, err 226 | } 227 | 228 | return data, nil 229 | } -------------------------------------------------------------------------------- /storage/dataUpdater-v1.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "archive/tar" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "log" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | "time" 13 | "bytes" 14 | 15 | "github.com/joho/godotenv" 16 | json "github.com/takoyaki-3/go-json" 17 | ) 18 | 19 | type PublicKey struct { 20 | Pubkey string `json:"pubkey"` 21 | } 22 | type OriginalData struct { 23 | Host string `json:"host"` 24 | } 25 | type Root struct { 26 | PubKeys []PublicKey `json:"pub_keys"` 27 | Hosts []string `json:"hosts"` 28 | OriginalData OriginalData `json:"original_data"` 29 | LastUpdate string `json:"last_update"` 30 | } 31 | 32 | type Info struct { 33 | DataList []OriginalDataItem `json:"data_list"` 34 | } 35 | type OriginalDataItem struct { 36 | Key string `json:"key"` 37 | } 38 | 39 | func monitorFile(filePath, timeFilePath string, done chan bool) { 40 | ticker := time.NewTicker(1 * time.Minute) 41 | var lastFileContent []byte 42 | lastFileContent, _ = ioutil.ReadFile(filePath) 43 | 44 | for { 45 | select { 46 | case <-ticker.C: 47 | fileContent, err := ioutil.ReadFile(filePath) 48 | if err != nil { 49 | log.Printf("Error reading file: %v", err) 50 | continue 51 | } 52 | 53 | if !bytes.Equal(fileContent, lastFileContent) { 54 | log.Printf("File %s has been modified", filePath) 55 | lastFileContent = fileContent 56 | time.AfterFunc(10*time.Second, func() { 57 | done <- true 58 | }) 59 | } 60 | 61 | // 現在の時刻を取得し、ファイルに書き込む 62 | currentTime := time.Now().Format(time.RFC3339) 63 | if err := ioutil.WriteFile(timeFilePath, []byte(currentTime), 0644); err != nil { 64 | log.Printf("Error writing last checked time: %v", err) 65 | } 66 | } 67 | } 68 | } 69 | 70 | func main() { 71 | 72 | BUTTER_ROOT_URL := "https://butter.takoyaki3.com/v1.0.0/root.json" 73 | 74 | err := godotenv.Load() 75 | if err == nil { 76 | BUTTER_ROOT_URL = os.Getenv("BUTTER_ROOT_URL_V1") 77 | } else { 78 | fmt.Println(".env file not found") 79 | } 80 | fmt.Println(BUTTER_ROOT_URL) 81 | 82 | done := make(chan bool) 83 | go monitorFile("./dataUpdater.go","./dataUpdaterLastCheckedTime.txt", done) 84 | 85 | go func() { 86 | 87 | isFirst := true 88 | for { 89 | var root Root 90 | rootData, err := downloadFile(BUTTER_ROOT_URL) 91 | if err != nil { 92 | log.Fatalln(err) 93 | } 94 | err = json.LoadFromString(string(rootData), &root) 95 | if err != nil { 96 | log.Fatalln(err) 97 | } 98 | 99 | var info Info 100 | infoData, err := downloadFile(root.OriginalData.Host + "info.json") 101 | if err != nil { 102 | log.Fatalln(err) 103 | } 104 | err = json.LoadFromString(string(infoData), &info) 105 | if err != nil { 106 | log.Fatalln(err) 107 | } 108 | 109 | infoPath := "info_v1.json" 110 | var oldInfo Info 111 | err = json.LoadFromPath(infoPath, &oldInfo) 112 | fmt.Println("info.json:",oldInfo) 113 | if !isFirst { 114 | if err == nil { 115 | if info.DataList[len(info.DataList)-1].Key == oldInfo.DataList[len(oldInfo.DataList)-1].Key { 116 | log.Println("No updates found. Exiting.") 117 | time.Sleep(time.Minute*5) 118 | continue 119 | } 120 | } 121 | } 122 | isFirst = false 123 | fmt.Println("start download. ["+root.OriginalData.Host + info.DataList[len(info.DataList)-1].Key+"]") 124 | 125 | err = ioutil.WriteFile(infoPath, infoData, 0644) 126 | if err != nil { 127 | log.Fatalln(err) 128 | } 129 | 130 | err = DownloadAndExtractTar(root.OriginalData.Host + info.DataList[len(info.DataList)-1].Key, "./" + info.DataList[len(info.DataList)-1].Key) 131 | if err != nil { 132 | log.Fatal(err) 133 | } 134 | 135 | fmt.Println("dev") 136 | 137 | err = os.Rename("./public_v1","./old") 138 | fmt.Println(err) 139 | err = os.Rename("./" + info.DataList[len(info.DataList)-1].Key, "public_v1") 140 | fmt.Println(err) 141 | if err != nil { 142 | continue 143 | } 144 | err = os.RemoveAll("old") 145 | fmt.Println(err) 146 | } 147 | }() 148 | 149 | <-done 150 | log.Printf("File has been modified. Shutting down in 10 seconds.") 151 | time.Sleep(10 * time.Second) 152 | log.Printf("Shutting down server.") 153 | } 154 | func DownloadAndExtractTar(url, dest string) error { 155 | // Check if output directory exists, create it if not 156 | if _, err := os.Stat(dest); os.IsNotExist(err) { 157 | err = os.MkdirAll(dest, 0755) 158 | if err != nil { 159 | return fmt.Errorf("failed to create destination directory: %v", err) 160 | } 161 | } 162 | 163 | resp, err := http.Get(url) 164 | if err != nil { 165 | return err 166 | } 167 | defer resp.Body.Close() 168 | 169 | if resp.StatusCode != http.StatusOK { 170 | return fmt.Errorf("failed to download %s: %d %s", url, resp.StatusCode, resp.Status) 171 | } 172 | 173 | tr := tar.NewReader(resp.Body) 174 | 175 | for { 176 | header, err := tr.Next() 177 | if err == io.EOF { 178 | break 179 | } 180 | if err != nil { 181 | return err 182 | } 183 | 184 | target := filepath.Join(dest, header.Name) 185 | 186 | switch header.Typeflag { 187 | case tar.TypeDir: 188 | if _, err := os.Stat(target); err != nil { 189 | if err := os.MkdirAll(target, 0755); err != nil { 190 | return err 191 | } 192 | } 193 | case tar.TypeReg: 194 | file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) 195 | if err != nil { 196 | return err 197 | } 198 | defer file.Close() 199 | 200 | if _, err := io.Copy(file, tr); err != nil { 201 | return err 202 | } 203 | default: 204 | return fmt.Errorf("unknown type: %c in %s", header.Typeflag, header.Name) 205 | } 206 | } 207 | 208 | return nil 209 | } 210 | 211 | func downloadFile(url string) ([]byte, error) { 212 | fmt.Println(url) 213 | resp, err := http.Get(url) 214 | if err != nil { 215 | return nil, err 216 | } 217 | defer resp.Body.Close() 218 | 219 | if resp.StatusCode != http.StatusOK { 220 | return nil, fmt.Errorf("failed to download file, status code: %d", resp.StatusCode) 221 | } 222 | 223 | data, err := ioutil.ReadAll(resp.Body) 224 | if err != nil { 225 | return nil, err 226 | } 227 | 228 | return data, nil 229 | } -------------------------------------------------------------------------------- /cmd/1_getDataList.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import os 4 | from datetime import datetime 5 | 6 | # JSONファイルのURL 7 | files_url = 'https://api.gtfs-data.jp/v2/files' 8 | feeds_url = 'https://api.gtfs-data.jp/v2/feeds' 9 | odpt_url = 'https://members-portal.odpt.org/api/v1/resources' 10 | 11 | # URLからJSONを取得する関数 12 | def get_json_from_url(url): 13 | response = requests.get(url) 14 | response.raise_for_status() 15 | return response.json() 16 | 17 | # ファイルの読み込み 18 | files_content = get_json_from_url(files_url)['body'] 19 | feeds_content = get_json_from_url(feeds_url)['body'] 20 | odpt_content = get_json_from_url(odpt_url) 21 | 22 | # feeds_contentからorganization_idをキーとした辞書を作成 23 | feeds_mapping = {feed['organization_id']: feed for feed in feeds_content} 24 | 25 | # 出力テキストを作成する関数 26 | def create_output_text_with_vehicle_position(files, feeds): 27 | output_text = "-------------\n" 28 | for file in files: 29 | organization_id = file['organization_id'] 30 | feed_id = organization_id + '_FEEDID_' + file['feed_id'] 31 | feed = feeds.get(organization_id, {}) 32 | vehicle_position_url = feed.get('real_time', {}).get('vehicle_position_url', '') 33 | output_text += f"事業者名:{file['organization_name']}\n" 34 | output_text += f"事業者名_url:{file['organization_web_url']}\n" 35 | output_text += f"都道府県:{file['feed_pref_id']}番\n" 36 | output_text += f"GTFSフィード名:{file['feed_name']}\n" 37 | output_text += f"ライセンス:{file['feed_license_id']}公開元: {file['organization_name']}\n" 38 | output_text += f"ライセンス_url:{file['feed_license_url']}\n" 39 | output_text += f"organization_id:{organization_id}\n" 40 | output_text += f"feed_id:{feed_id}\n" 41 | if vehicle_position_url: # VehiclePositionのURLが存在する場合のみ追加 42 | output_text += f"URLs:GTFS, VehiclePosition\n" 43 | else: 44 | output_text += f"URLs:GTFS\n" 45 | output_text += f"GTFS_url:{file['file_url']}\n" 46 | if vehicle_position_url: # VehiclePositionのURLが存在する場合のみ追加 47 | output_text += f"VehiclePosition_url:{vehicle_position_url}\n" 48 | output_text += f"最新GTFS開始日:{file['file_from_date']}\n" 49 | output_text += f"最新GTFS終了日:{file['file_to_date']}\n" 50 | output_text += f"最終更新日:{feed['last_updated_at']}\n" 51 | output_text += f"詳細:詳細\n-------------\n" 52 | return output_text 53 | 54 | def generate_odpt(entry): 55 | name_ja = entry['name_ja'] 56 | url_ja = entry['url_ja'] 57 | datasets = entry['datasets'] 58 | output = "" 59 | 60 | # Read the JSON file 61 | conf = None 62 | if os.path.isfile('conf.json'): 63 | with open('conf.json', 'r', encoding='utf-8') as file: 64 | conf = json.load(file) 65 | 66 | for dataset in datasets: 67 | if dataset.get('vehicle_position'): 68 | gtfs_url = next((res['url'] for res in dataset.get('dataresource', []) if dataset.get('format_type') in ('GTFS', 'GTFS/GTFS-JP')), '') 69 | vehicle_position_url = dataset['vehicle_position']['url'] 70 | license_type = dataset['license_type'] 71 | license_url = "https://creativecommons.org/licenses/by/4.0/deed.ja" if "CC BY 4.0" in license_type else "" 72 | today_date = datetime.today().strftime('%Y-%m-%d') 73 | 74 | # API アクセスキーの置き換え 75 | if '[アクセストークン/YOUR_ACCESS_TOKEN]' in gtfs_url: 76 | if conf == None: 77 | continue 78 | gtfs_url = gtfs_url.replace('[アクセストークン/YOUR_ACCESS_TOKEN]', conf['odptAPIKey']) 79 | vehicle_position_url = vehicle_position_url.replace('[アクセストークン/YOUR_ACCESS_TOKEN]', conf['odptAPIKey']) 80 | 81 | output += f"事業者名:{name_ja}\n" 82 | output += f"事業者名_url:{url_ja}\n" 83 | output += "都道府県:\n" 84 | output += f"GTFSフィード名:{name_ja}\n" 85 | output += f"ライセンス:{license_type}\n" 86 | output += f"ライセンス_url:{license_url}\n" 87 | output += "URLs:GTFS, VehiclePosition\n" 88 | output += f"GTFS_url:{gtfs_url}\n" 89 | output += f"VehiclePosition_url:{vehicle_position_url}\n" 90 | output += f"最新GTFS開始日:{today_date}\n" 91 | output += f"最新GTFS終了日:{today_date}\n" 92 | output += f"最終更新日:{today_date}\n" 93 | output += "詳細:詳細\n-------------\n" 94 | return output 95 | 96 | # 出力テキストを作成 97 | output_text = create_output_text_with_vehicle_position(files_content, feeds_mapping) 98 | output_text += "".join([generate_odpt(entry) for entry in odpt_content]) 99 | 100 | now = datetime.now() 101 | # 日付を指定された形式にフォーマット 102 | dateStr = now.strftime("%Y-%m-%d") 103 | 104 | # 東京都交通局を後から追加 105 | output_text += '''事業者名:東京都交通局 106 | 事業者名_url:https://www.kotsu.metro.tokyo.jp 107 | 都道府県:東京都 108 | GTFSフィード名:東京都交通局 109 | ライセンス:CC BY 4.0公開元:東京都交通局・公共交通オープンデータ協議会 110 | ライセンス_url:https://creativecommons.org/licenses/by/4.0/ 111 | URLs:GTFS, VehiclePosition 112 | GTFS_url:https://api-public.odpt.org/api/v4/files/Toei/data/ToeiBus-GTFS.zip 113 | VehiclePosition_url:https://api-public.odpt.org/api/v4/gtfs/realtime/ToeiBus 114 | 詳細:詳細 115 | 最新GTFS開始日:'''+dateStr+''' 116 | 最新GTFS終了日:'''+dateStr+''' 117 | 最終更新日:'''+dateStr+''' 118 | organization_id:ToeiBus 119 | feed_id:ToeiBus_FEEDID_ToeiBus 120 | ------------- 121 | ''' 122 | 123 | conf = None 124 | if os.path.isfile('conf.json'): 125 | with open('conf.json', 'r', encoding='utf-8') as file: 126 | conf = json.load(file) 127 | 128 | # 横浜市営バスを後から追加 129 | output_text += '''事業者名:横浜市交通局 130 | 事業者名_url:https://www.city.yokohama.lg.jp/kotsu/bus 131 | 都道府県:神奈川県 132 | GTFSフィード名:横浜市営バス 133 | ライセンス:ODPT基本 134 | ライセンス_url:https://developer.odpt.org/terms 135 | URLs:GTFS, VehiclePosition 136 | GTFS_url:https://api.odpt.org/api/v4/files/YokohamaMunicipal/data/YokohamaMunicipal-Bus-GTFS.zip?acl:consumerKey='''+conf['odptAPIKey']+''' 137 | VehiclePosition_url:https://api.odpt.org/api/v4/gtfs/realtime/YokohamaMunicipalBus_vehicle?acl:consumerKey='''+conf['odptAPIKey']+''' 138 | 詳細:詳細 139 | 最新GTFS開始日:'''+dateStr+''' 140 | 最新GTFS終了日:'''+dateStr+''' 141 | 最終更新日:'''+dateStr+''' 142 | organization_id:YokohamaMunicipal 143 | feed_id:YokohamaMunicipal_FEEDID_YokohamaMunicipal 144 | ------------- 145 | ''' 146 | # 西武バスを後から追加 147 | output_text += '''事業者名:西武バス 148 | 事業者名_url:https://www.seibubus.co.jp/sp 149 | 都道府県:東京都 150 | GTFSフィード名:西武バス 151 | ライセンス:ODPT基本 152 | ライセンス_url:https://developer.odpt.org/terms 153 | URLs:GTFS, VehiclePosition 154 | GTFS_url:https://api.odpt.org/api/v4/files/SeibuBus/data/SeibuBus-GTFS.zip?acl:consumerKey='''+conf['odptAPIKey']+''' 155 | VehiclePosition_url:https://api.odpt.org/api/v4/gtfs/realtime/SeibuBus_vehicle?acl:consumerKey='''+conf['odptAPIKey']+''' 156 | 詳細:詳細 157 | 最新GTFS開始日:'''+dateStr+''' 158 | 最新GTFS終了日:'''+dateStr+''' 159 | 最終更新日:'''+dateStr+''' 160 | organization_id:SeibuBus 161 | feed_id:SeibuBus_FEEDID_SeibuBus 162 | ------------- 163 | ''' 164 | 165 | # ファイルへ保存 166 | output_file_path = './dataList.txt' 167 | with open(output_file_path, 'w', encoding='utf-8') as file: 168 | file.write(output_text) 169 | -------------------------------------------------------------------------------- /cmd/5_split.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "math" 7 | "net/url" 8 | "os" 9 | "time" 10 | 11 | "io/ioutil" 12 | 13 | "github.com/takoyaki-3/goc" 14 | 15 | csvtag "github.com/takoyaki-3/go-csv-tag/v3" 16 | filetool "github.com/takoyaki-3/go-file-tool" 17 | gtfs "github.com/takoyaki-3/go-gtfs/v2" 18 | json "github.com/takoyaki-3/go-json" 19 | 20 | . "github.com/takoyaki-3/butter/cmd/helper" 21 | ) 22 | 23 | var privateKeyBytes []byte 24 | 25 | func main() { 26 | fmt.Println("start") 27 | 28 | var err error 29 | 30 | // RSA秘密鍵を読み込む 31 | privateKeyBytes, err = ioutil.ReadFile("key.pem") 32 | if err != nil { 33 | log.Fatalln(err) 34 | } 35 | 36 | _, files := filetool.DirWalk("./feed", filetool.DirWalkOption{}) 37 | goc.Parallel(8, len(files), func(i, rank int) { 38 | file := files[i] 39 | if file.Name[len(file.Name)-len(".zip"):] == ".zip" { 40 | fmt.Println("feed:",file.Name) 41 | split("feed_dir_out", "v1.0.0", file.Name, time.Now().Format("2006-01-02T15_04_05Z07_00")) 42 | } 43 | }) 44 | 45 | _, files = filetool.DirWalk("./gtfs", filetool.DirWalkOption{}) 46 | goc.Parallel(8, len(files), func(i, rank int) { 47 | file := files[i] 48 | if file.Name[len(file.Name)-len(".zip"):] == ".zip" { 49 | fmt.Println("gtfs:",file.Name) 50 | split("dir_out", "v0.0.0", file.Name, time.Now().Format("2006-01-02T15_04_05Z07_00")) 51 | } 52 | }) 53 | } 54 | 55 | type Info struct { 56 | ByStopHashValueSize int `json:"by_stop_hash_value_size"` 57 | ByTripHashValueSize int `json:"by_trip_hash_value_size"` 58 | } 59 | 60 | type StopTime struct { 61 | StopName string `csv:"stop_name" json:"stop_name"` 62 | 63 | StopID string `csv:"stop_id" json:"stop_id"` 64 | StopSeq string `csv:"stop_sequence" json:"stop_sequence"` 65 | StopHeadSign string `csv:"stop_headsign" json:"stop_headsign"` 66 | TripID string `csv:"trip_id" json:"trip_id"` 67 | Shape int `csv:"shape_dist_traveled" json:"shape_dist_traveled"` 68 | Departure string `csv:"departure_time" json:"departure_time"` 69 | Arrival string `csv:"arrival_time" json:"arrival_time"` 70 | PickupType int `csv:"pickup_type" json:"pickup_type"` 71 | DropOffType int `csv:"drop_off_type" json:"drop_off_type"` 72 | 73 | TripName string `csv:"trip_short_name" json:"trip_short_name"` 74 | RouteID string `csv:"route_id" json:"route_id"` 75 | ServiceID string `csv:"service_id" json:"service_id"` 76 | ShapeID string `csv:"shape_id" json:"shape_id"` 77 | DirectionID string `csv:"direction_id" json:"direction_id"` 78 | Headsign string `csv:"trip_headsign" json:"trip_headsign"` 79 | } 80 | 81 | func split(srcDirRoot string, dstDirRoot string, file string, version string) error { 82 | 83 | // 入力ファイルのディレクトリを設定 84 | srcDir := srcDirRoot + "/" + file 85 | 86 | // 出力ファイルのディレクトリを設定 87 | dstDir := dstDirRoot + "/" + file[:len(file)-len(".zip")] + "/" + version 88 | 89 | // stop_times.txtからStopTimeのスライスをロード 90 | stopTimes := []StopTime{} 91 | err := csvtag.LoadFromPath(srcDir+"/stop_times.txt", &stopTimes) 92 | if err != nil { 93 | return err 94 | } 95 | // trips.txtからTripのスライスをロード 96 | trips := []gtfs.Trip{} 97 | err = csvtag.LoadFromPath(srcDir+"/trips.txt", &trips) 98 | if err != nil { 99 | return err 100 | } 101 | // stops.txtからStopのスライスをロード 102 | stops := []gtfs.Stop{} 103 | err = csvtag.LoadFromPath(srcDir+"/stops.txt", &stops) 104 | if err != nil { 105 | return err 106 | } 107 | stopID2Name := map[string]string{} 108 | for _,stop := range stops{ 109 | stopID2Name[stop.ID] = stop.Name 110 | } 111 | 112 | // StopTimesの各要素に対応するTrip情報を追加 113 | for i, _ := range stopTimes { 114 | tripID := stopTimes[i].TripID 115 | stopTimes[i].StopName = stopID2Name[stopTimes[i].StopID] 116 | for _, trip := range trips { 117 | if trip.ID == tripID { 118 | stopTimes[i].TripName = trip.Name 119 | stopTimes[i].RouteID = trip.RouteID 120 | stopTimes[i].ServiceID = trip.ServiceID 121 | stopTimes[i].ShapeID = trip.ServiceID 122 | stopTimes[i].DirectionID = trip.DirectionID 123 | stopTimes[i].Headsign = trip.Headsign 124 | break 125 | } 126 | } 127 | } 128 | 129 | // location_typeが1の停留所と、それに関連するlocation_typeが0の停留所を特定 130 | relatedStopIDs := map[string][]string{} 131 | for _, stop := range stops { 132 | if stop.Type == "1" { 133 | for _, relatedStop := range stops { 134 | if relatedStop.Parent == stop.ID && relatedStop.Type == "0" { 135 | if _,ok:=relatedStopIDs[stop.ID];!ok{ 136 | relatedStopIDs[stop.ID] = []string{} 137 | } 138 | relatedStopIDs[stop.ID] = append(relatedStopIDs[stop.ID], relatedStop.ID) 139 | } 140 | } 141 | } 142 | } 143 | // location_typeが0の関連する停留所のstop_timesを取得 144 | var relatedStopTimes []StopTime 145 | for _, stopTime := range stopTimes { 146 | for baseStopID, stopIDs := range relatedStopIDs { 147 | for _,stopID := range stopIDs{ 148 | if stopTime.StopID == stopID { 149 | stopTime.StopID = baseStopID 150 | relatedStopTimes = append(relatedStopTimes, stopTime) 151 | break 152 | } 153 | } 154 | } 155 | } 156 | 157 | // StopTimeのデータを停留所IDとTripIDでグループ化 158 | byStop := map[string][]StopTime{} 159 | byTrip := map[string][]StopTime{} 160 | 161 | for _, stopTime := range stopTimes { 162 | byStop[url.QueryEscape(stopTime.StopID)] = append(byStop[url.QueryEscape(stopTime.StopID)], stopTime) 163 | byTrip[url.QueryEscape(stopTime.TripID)] = append(byTrip[url.QueryEscape(stopTime.TripID)], stopTime) 164 | } 165 | for _, stopTime := range relatedStopTimes { 166 | byStop[url.QueryEscape(stopTime.StopID)] = append(byStop[url.QueryEscape(stopTime.StopID)], stopTime) 167 | } 168 | 169 | // グループ化されたデータをさらにハッシュ値によってサブグループ化 170 | tarByStop := map[string]map[string][]StopTime{} 171 | tarByTrip := map[string]map[string][]StopTime{} 172 | fileNum := int(math.Log2(float64(len(stopTimes))))/4/4 + 1 173 | 174 | // 停留所IDでグループ化されたデータをハッシュ値によってサブグループ化 175 | for stopID, stopTimes := range byStop { 176 | hid := GetBinaryBySHA256(stopID)[:fileNum] 177 | if _, ok := tarByStop[hid]; !ok { 178 | tarByStop[hid] = map[string][]StopTime{} 179 | } 180 | tarByStop[hid][stopID] = stopTimes 181 | } 182 | // TripIDでグループ化されたデータをハッシュ値によってサブグループ化 183 | for tripID, stopTimes := range byTrip { 184 | hid := GetBinaryBySHA256(tripID)[:fileNum] 185 | if _, ok := tarByTrip[hid]; !ok { 186 | tarByTrip[hid] = map[string][]StopTime{} 187 | } 188 | tarByTrip[hid][tripID] = stopTimes 189 | } 190 | 191 | // 出力 192 | os.MkdirAll(dstDir+"/byStops", 0777) 193 | os.MkdirAll(dstDir+"/byTrips", 0777) 194 | 195 | // tarByStopのデータをアーカイブして保存 196 | for hid, data := range tarByStop { 197 | // TarGzWriterを作成し、指定したファイル名で開く 198 | tgz, err := NewTarGzWriter(dstDir + "/byStops/" + hid + ".tar.gz") 199 | if err != nil { 200 | panic(err) 201 | } 202 | defer tgz.Close() // TarGzWriterを閉じることを保証 203 | 204 | // data内の各停留所IDとその停留所の時刻データをTarGzWriterに追加 205 | for stopID, stopTimes := range data { 206 | str, _ := csvtag.DumpToString(&stopTimes) // StopTimesを文字列に変換 207 | b := []byte(str) // 文字列をバイト配列に変換 208 | 209 | // データを署名付きでTarGzWriterに追加 210 | err := tgz.AddDataWithSign(stopID, b, privateKeyBytes) 211 | if err != nil { 212 | log.Fatalln(err) 213 | } 214 | } 215 | } 216 | 217 | // tarByTripのデータをアーカイブして保存 218 | for hid, data := range tarByTrip { 219 | // TarGzWriterを作成し、指定したファイル名で開く 220 | tgz, err := NewTarGzWriter(dstDir + "/byTrips/" + hid + ".tar.gz") 221 | if err != nil { 222 | panic(err) 223 | } 224 | defer tgz.Close() // TarGzWriterを閉じることを保証 225 | 226 | // data内の各TripIDとそのトリップの時刻データをTarGzWriterに追加 227 | for tripID, stopTimes := range data { 228 | str, _ := csvtag.DumpToString(&stopTimes) // StopTimesを文字列に変換 229 | b := []byte(str) // 文字列をバイト配列に変換 230 | 231 | // データを署名付きでTarGzWriterに追加 232 | err := tgz.AddDataWithSign(tripID, b, privateKeyBytes) 233 | if err != nil { 234 | log.Fatalln(err) 235 | } 236 | } 237 | } 238 | 239 | // GTFSのコピー 240 | err = CopyDir(srcDir, dstDir+"/GTFS") 241 | if err != nil { 242 | return err 243 | } 244 | 245 | // 設定ファイル作成 246 | info := Info{ 247 | ByStopHashValueSize: fileNum, 248 | ByTripHashValueSize: fileNum, 249 | } 250 | err = json.DumpToFile(&info, dstDir+"/info.json") 251 | if err != nil { 252 | return err 253 | } 254 | return err 255 | } 256 | --------------------------------------------------------------------------------