├── .prettierignore ├── otp-data-server ├── .dockerignore ├── Dockerfile ├── deploy.sh └── lighttpd.conf ├── .dockerignore ├── .git-blame-ignore-revs ├── .prettierrc ├── finland ├── gtfs-rules │ ├── hsl-no-trains.rule │ ├── matka-cleaned.rule │ └── matka.rule ├── otp-config.json ├── build-config.json ├── config.js └── router-config.json ├── waltti ├── gtfs-rules │ └── only-kotka-ferries.rule ├── otp-config.json ├── build-config.json ├── config.js └── router-config.json ├── .gitignore ├── kela ├── gtfs-rules │ ├── remove-route-color.rule │ ├── remove-matching-route.rule │ └── matkahuolto.rule ├── otp-config.json ├── build-config.json ├── config.js └── router-config.json ├── opentripplanner ├── entrypoint.sh ├── Dockerfile └── deploy-otp.sh ├── eslint.config.mjs ├── varely ├── otp-config.json ├── build-config.json ├── config.js └── router-config.json ├── waltti-alt ├── otp-config.json ├── gtfs-rules │ └── waltti.rule ├── build-config.json ├── config.js └── router-config.json ├── Dockerfile ├── hsl ├── otp-config.json ├── osm-preprocessing │ └── hsl.sh ├── config.js ├── build-config.json └── router-config.json ├── task ├── GTFSRename.js ├── Download.js ├── BlobValidation.js ├── Seed.js ├── GTFSReplace.js ├── StorageCleanup.js ├── OBAFilter.js ├── DownloadDEMBlob.js ├── SetFeedId.js ├── OTPTest.js ├── MapFit.js ├── BuildOTPGraph.js ├── OSMPreprocessing.js ├── PrepareFit.js ├── PrepareRouterData.js ├── ZipTask.js └── Update.js ├── otp-data-tools ├── README.md └── Dockerfile ├── index.js ├── .github └── workflows │ ├── scripts │ ├── push_prod.sh │ └── build_and_push_dev.sh │ ├── prod-pipeline.yml │ └── dev-pipeline.yml ├── logback-include-extensions.xml ├── package.json ├── test.sh ├── config.js ├── util.js ├── gulpfile.js └── README.md /.prettierignore: -------------------------------------------------------------------------------- 1 | README.md 2 | -------------------------------------------------------------------------------- /otp-data-server/.dockerignore: -------------------------------------------------------------------------------- 1 | deploy.sh -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | data/ 2 | storage/ 3 | node_modules/ 4 | otp-data-tools/ 5 | *.zip 6 | Dockerfile 7 | *~ 8 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Prettier and eslint automatic formatting/linting 2 | db27603ac9b4ce52a33aa67796f26038b0068fd8 3 | 4 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "arrowParens": "avoid", 3 | "endOfLine": "auto", 4 | "singleQuote": true, 5 | "trailingComma": "all" 6 | } 7 | -------------------------------------------------------------------------------- /finland/gtfs-rules/hsl-no-trains.rule: -------------------------------------------------------------------------------- 1 | # Remove all trains from HSL gtfs 2 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"109"}} 3 | -------------------------------------------------------------------------------- /waltti/gtfs-rules/only-kotka-ferries.rule: -------------------------------------------------------------------------------- 1 | # Only keep routes available in Kotka 2 | {"op":"retain", "match":{"file":"agency.txt", "agency_name":"Finferries"}} 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | storage 3 | node_modules/ 4 | .DS_Store 5 | router-finland.zip 6 | router-hsl.zip 7 | router-waltti.zip 8 | router-waltti-alt.zip 9 | 10 | .tmp -------------------------------------------------------------------------------- /kela/gtfs-rules/remove-route-color.rule: -------------------------------------------------------------------------------- 1 | # remove route_color field if it exists 2 | {"op":"update", "match":{"file":"routes.txt"}, "update":{"route_color":"s/.*//"}} 3 | -------------------------------------------------------------------------------- /kela/gtfs-rules/remove-matching-route.rule: -------------------------------------------------------------------------------- 1 | # remove routes matching the route_short_name regex 2 | {"op":"remove", "match":{"file":"routes.txt", "route_short_name":"m/^OB(2|3|36|37|38|4|41|44|6|9|91)$/"}} 3 | -------------------------------------------------------------------------------- /opentripplanner/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | echo "OTP_GRAPH_DIR: ${OTP_GRAPH_DIR}" 5 | 6 | java $JAVA_OPTS -cp @/app/jib-classpath-file @/app/jib-main-class-file /var/otp/${OTP_GRAPH_DIR} --load --serve 7 | -------------------------------------------------------------------------------- /opentripplanner/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG OTP_TAG="v2" 2 | FROM hsldevcom/opentripplanner:${OTP_TAG} 3 | 4 | ARG OTP_GRAPH_DIR 5 | ENV OTP_GRAPH_DIR=$OTP_GRAPH_DIR 6 | 7 | ADD entrypoint.sh /var/entrypoint.sh 8 | 9 | WORKDIR /var/otp/${OTP_GRAPH_DIR} 10 | 11 | ENTRYPOINT ["/var/entrypoint.sh"] -------------------------------------------------------------------------------- /kela/gtfs-rules/matkahuolto.rule: -------------------------------------------------------------------------------- 1 | # Onnibus mega (has no support for Kela fare) 2 | {"op":"remove", "match":{"file":"agency.txt", "agency_id":"4820"}} 3 | {"op":"remove", "match":{"file":"agency.txt", "agency_id":"7247"}} 4 | 5 | # Länsilinjat duplicates 6 | {"op":"remove", "match":{"file":"agency.txt", "agency_id":"299"}} -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- 1 | import globals from 'globals'; 2 | import pluginJs from '@eslint/js'; 3 | 4 | /** @type {import('eslint').Linter.Config[]} */ 5 | export default [ 6 | { files: ['**/*.js'], languageOptions: { sourceType: 'commonjs' } }, 7 | { languageOptions: { globals: globals.node } }, 8 | pluginJs.configs.recommended, 9 | ]; 10 | -------------------------------------------------------------------------------- /varely/otp-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "otpFeatures": { 3 | "GtfsGraphQlApi": true, 4 | "DebugUi": true, 5 | "APIServerInfo": true, 6 | "APIUpdaterStatus": false, 7 | "SandboxAPIMapboxVectorTilesApi": true, 8 | "ActuatorAPI": true, 9 | "AsyncGraphQLFetchers": false, 10 | "DebugRasterTiles": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /waltti-alt/otp-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "otpFeatures": { 3 | "GtfsGraphQlApi": true, 4 | "DebugUi": true, 5 | "APIServerInfo": true, 6 | "APIUpdaterStatus": false, 7 | "SandboxAPIMapboxVectorTilesApi": true, 8 | "ActuatorAPI": true, 9 | "AsyncGraphQLFetchers": false, 10 | "DebugRasterTiles": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker:dind 2 | 3 | RUN apk add --update --no-cache bash zip p7zip curl nodejs yarn && rm -rf /var/cache/apk/* 4 | 5 | WORKDIR /opt/otp-data-builder 6 | 7 | ADD . /opt/otp-data-builder/ 8 | 9 | RUN yarn install 10 | 11 | CMD ( dockerd-entrypoint.sh --log-level=error > /dev/null 2>&1 & ) && unset DOCKER_HOST && sleep 300 && node index.js 12 | -------------------------------------------------------------------------------- /waltti-alt/gtfs-rules/waltti.rule: -------------------------------------------------------------------------------- 1 | # map white route color to 'bus blue' 2 | {"op":"update", "match":{"file":"routes.txt", "route_color":"FFFFFF"}, "update":{"route_color":""}} 3 | 4 | # Remove shape_dist_traveled from stop_times.txt as values in waltti GTFS are bogus 5 | {"op":"update", "match":{"file":"stop_times.txt", }, "update":{"shape_dist_traveled":-999.0}} 6 | 7 | -------------------------------------------------------------------------------- /kela/otp-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "otpFeatures": { 3 | "GtfsGraphQlApi": true, 4 | "DebugUi": true, 5 | "APIServerInfo": true, 6 | "APIUpdaterStatus": false, 7 | "SandboxAPIMapboxVectorTilesApi": true, 8 | "FloatingBike": false, 9 | "ActuatorAPI": true, 10 | "AsyncGraphQLFetchers": false, 11 | "DebugRasterTiles": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /waltti/otp-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "otpFeatures": { 3 | "GtfsGraphQlApi": true, 4 | "DebugUi": true, 5 | "APIServerInfo": true, 6 | "APIUpdaterStatus": false, 7 | "SandboxAPIMapboxVectorTilesApi": true, 8 | "FloatingBike": false, 9 | "ActuatorAPI": true, 10 | "AsyncGraphQLFetchers": false, 11 | "Emission": true, 12 | "DebugRasterTiles": true 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /hsl/otp-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "otpFeatures": { 3 | "GtfsGraphQlApi": true, 4 | "DebugUi": true, 5 | "APIServerInfo": true, 6 | "APIUpdaterStatus": false, 7 | "SandboxAPIMapboxVectorTilesApi": true, 8 | "FloatingBike": true, 9 | "ActuatorAPI": true, 10 | "AsyncGraphQLFetchers": false, 11 | "Emission": true, 12 | "DebugRasterTiles": true, 13 | "FlexRouting": true 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /finland/otp-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "otpFeatures": { 3 | "GtfsGraphQlApi": true, 4 | "DebugUi": true, 5 | "APIServerInfo": true, 6 | "APIUpdaterStatus": false, 7 | "SandboxAPIMapboxVectorTilesApi": true, 8 | "FloatingBike": true, 9 | "ActuatorAPI": true, 10 | "AsyncGraphQLFetchers": false, 11 | "Emission": true, 12 | "DebugRasterTiles": true, 13 | "FlexRouting": true 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /task/GTFSRename.js: -------------------------------------------------------------------------------- 1 | const through = require('through2'); 2 | 3 | module.exports = { 4 | renameGTFSFile: () => { 5 | return through.obj(function (file, encoding, callback) { 6 | if (!file.stem.includes('-gtfs')) { 7 | file.stem = file.stem + '-gtfs'; 8 | } 9 | if (file.extname !== '.zip') { 10 | file.extname = '.zip'; 11 | } 12 | callback(null, file); 13 | }); 14 | }, 15 | }; 16 | -------------------------------------------------------------------------------- /otp-data-tools/README.md: -------------------------------------------------------------------------------- 1 | # OpenTripPlanner-data-tools 2 | 3 | ## About: 4 | 5 | - This docker container includes all the needed tools for preparing gtfs data ready for consumption. 6 | - It uses opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli as the base image. 7 | - Useful links: 8 | - https://github.com/OneBusAway/onebusaway-gtfs-modules 9 | - Documentation can be found in the `docs` folder 10 | - https://registry.hub.docker.com/r/opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli 11 | -------------------------------------------------------------------------------- /finland/gtfs-rules/matka-cleaned.rule: -------------------------------------------------------------------------------- 1 | # Stops at 0 of some coordinate system 2 | {"op":"remove", "match":{"file":"stops.txt","stop_lon":"20.142573"}} 3 | 4 | # Waltti demand responsive transport. BTW: why real on demand traffic is removed? 5 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"6"}} 6 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"715"}} 7 | 8 | # Remove shape_dist_traveled from stop_times.txt as values in waltti GTFS are bogus 9 | {"op":"update", "match":{"file":"stop_times.txt", }, "update":{"shape_dist_traveled":-999.0}} 10 | -------------------------------------------------------------------------------- /otp-data-tools/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli:8.0.0 2 | LABEL maintainer="Digitransit (digitransit.fi)" \ 3 | version="1.1" \ 4 | repo="https://github.com/HSLdevcom/OpenTripPlanner-data-container" 5 | 6 | RUN apt-get update && apt-get -y install git python3 python3-pip python3-venv osmctools 7 | RUN rm -rf /var/lib/apt/lists/* 8 | RUN python3 -m venv python-venv && python-venv/bin/pip install future grequests numpy 9 | RUN git clone -b v3 --single-branch https://github.com/HSLdevcom/OTPQA.git 10 | 11 | RUN mkdir /data 12 | VOLUME /data 13 | -------------------------------------------------------------------------------- /hsl/osm-preprocessing/hsl.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -x 4 | 5 | osmconvert hsl.pbf -o=hsl.o5m 6 | osmfilter hsl.o5m -o=hsl2.o5m --modify-tags="ref=H0071 to ref=no \ 7 | ref=H0091 to ref=no \ 8 | ref=H0072 to ref=no \ 9 | ref=H0079 to ref=no \ 10 | ref=H0082 to ref=no \ 11 | ref=H0083 to ref=no \ 12 | ref=H0084 to ref=no" 13 | osmconvert hsl2.o5m -o=hsl.pbf 14 | -------------------------------------------------------------------------------- /hsl/config.js: -------------------------------------------------------------------------------- 1 | const { mapSrc } = require('../util'); 2 | 3 | module.exports = { 4 | id: 'hsl', 5 | src: [ 6 | mapSrc( 7 | 'HSL', 8 | 'https://infopalvelut.storage.hsldev.com/gtfs/hsl_google_transit.zip', 9 | false, 10 | undefined, 11 | { 'trips.txt': 'trips2.txt' }, 12 | ), 13 | /* 14 | mapSrc( 15 | 'HSLlautta', 16 | 'https://mobility.mobility-database.fintraffic.fi/static/lautat_new.zip', 17 | ), 18 | */ 19 | // src('Sipoo', 'https://koontikartta.navici.com/tiedostot/rae/sipoon_kunta_sibbo_kommun.zip') 20 | ], 21 | osm: ['hsl'], 22 | dem: 'hsl', 23 | }; 24 | -------------------------------------------------------------------------------- /varely/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataImportReport": true, 3 | "areaVisibility": true, 4 | "maxAreaNodes": 1000, 5 | "maxTransferDuration": "26m", 6 | "transitServiceStart": "-P2W", 7 | "transitServiceEnd": "P12W", 8 | "transitModelTimeZone": "Europe/Helsinki", 9 | "fares": "hsl", 10 | "transferRequests": [ 11 | { "modes": "WALK" }, 12 | { 13 | "modes": "WALK", 14 | "wheelchairAccessibility": { 15 | "enabled": true 16 | } 17 | }, 18 | { "modes": "BICYCLE" } 19 | ], 20 | "osmDefaults": { 21 | "timeZone": "Europe/Helsinki", 22 | "osmTagMapping": "finland" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /varely/config.js: -------------------------------------------------------------------------------- 1 | const { mapSrc } = require('../util'); 2 | 3 | module.exports = { 4 | id: 'varely', 5 | src: [ 6 | mapSrc( 7 | 'VARELY', 8 | 'http://digitransit-proxy:8080/out/varelyadmin.mattersoft.fi/feeds/102.zip', 9 | ), 10 | mapSrc('FOLI', 'http://data.foli.fi/gtfs/gtfs.zip'), 11 | mapSrc( 12 | 'Rauma', 13 | 'http://digitransit-proxy:8080/out/raumaadmin.mattersoft.fi/feeds/233.zip', 14 | ), 15 | mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), 16 | mapSrc('Pori', 'https://tvv.fra1.digitaloceanspaces.com/231.zip', true), 17 | ], 18 | osm: ['varely'], 19 | }; 20 | -------------------------------------------------------------------------------- /kela/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataImportReport": true, 3 | "areaVisibility": true, 4 | "staticParkAndRide": false, 5 | "maxTransferDuration": "1h", 6 | "maxStopToShapeSnapDistance": 300, 7 | "transitServiceStart": "-P2W", 8 | "transitServiceEnd": "P12W", 9 | "transitModelTimeZone": "Europe/Helsinki", 10 | "fares": "hsl", 11 | "transferRequests": [ 12 | { "modes": "WALK" }, 13 | { 14 | "modes": "WALK", 15 | "wheelchairAccessibility": { 16 | "enabled": true 17 | } 18 | } 19 | ], 20 | "boardingLocationTags": ["ref", "ref:findt", "ref:findr"], 21 | "osmDefaults": { 22 | "timeZone": "Europe/Helsinki", 23 | "osmTagMapping": "constant_speed_finland" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /waltti-alt/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataImportReport": true, 3 | "areaVisibility": true, 4 | "maxAreaNodes": 1000, 5 | "maxTransferDuration": "26m", 6 | "transitServiceStart": "-P2W", 7 | "transitServiceEnd": "P12W", 8 | "transitModelTimeZone": "Europe/Helsinki", 9 | "fares": "hsl", 10 | "transferRequests": [ 11 | { "modes": "WALK" }, 12 | { 13 | "modes": "WALK", 14 | "wheelchairAccessibility": { 15 | "enabled": true 16 | } 17 | }, 18 | { "modes": "BICYCLE" } 19 | ], 20 | "gtfsDefaults": { 21 | "stationTransferPreference": "recommended" 22 | }, 23 | "osmDefaults": { 24 | "timeZone": "Europe/Helsinki", 25 | "osmTagMapping": "finland" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /otp-data-server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine:3.21 2 | LABEL authors="Digitransit version: 2" 3 | 4 | RUN apk add --update --no-cache \ 5 | lighttpd \ 6 | lighttpd-mod_auth \ 7 | busybox \ 8 | && rm -rf /var/cache/apk/* 9 | 10 | ARG OTP_GRAPH_DIR 11 | ENV OTP_GRAPH_DIR=${OTP_GRAPH_DIR} 12 | ADD lighttpd.conf /etc/lighttpd/lighttpd.conf 13 | RUN printf '#!/bin/sh\ngrep -v "#" /etc/lighttpd/lighttpd.conf |grep port > /dev/null\nif [ "$?" = "1" ]\nthen\n echo "server.port = $PORT" >> /etc/lighttpd/lighttpd.conf\nfi\n/usr/sbin/lighttpd -D -f /etc/lighttpd/lighttpd.conf 2>&1' > /usr/sbin/startlighttpd && chmod 755 /usr/sbin/startlighttpd 14 | 15 | ENV PORT="8080" 16 | EXPOSE 8080 17 | 18 | CMD /bin/ash -c "/usr/sbin/startlighttpd" 19 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const { postSlackMessage } = require('./util'); 2 | const { update } = require('./task/Update'); 3 | const { SPLIT_BUILD_TYPE } = require('./config.js'); 4 | 5 | let message = ''; 6 | 7 | switch (SPLIT_BUILD_TYPE) { 8 | case 'ONLY_BUILD_STREET_GRAPH': 9 | message = 'Starting street only graph data build'; 10 | break; 11 | case 'USE_PREBUILT_STREET_GRAPH': 12 | message = 'Starting graph data build from prebuilt street graph'; 13 | break; 14 | default: 15 | message = 'Starting data build'; 16 | break; 17 | } 18 | 19 | postSlackMessage(message) 20 | .then(response => { 21 | if (response.ok) { 22 | global.messageTimeStamp = response.ts; 23 | } 24 | }) 25 | .catch(err => { 26 | console.log(err); 27 | }) 28 | .finally(() => { 29 | update(); 30 | }); 31 | -------------------------------------------------------------------------------- /.github/workflows/scripts/push_prod.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | COMMIT_HASH=$(git rev-parse --short "$GITHUB_SHA") 5 | ORG=hsldevcom 6 | 7 | function imagedeploy { 8 | DOCKER_IMAGE=$ORG/$1 9 | DOCKER_TAG=${DOCKER_BASE_TAG:-prod} 10 | DOCKER_DEV_TAG=${DOCKER_DEV_TAG:-latest} 11 | 12 | DOCKER_TAG_LONG=$DOCKER_TAG-$(date +"%Y-%m-%dT%H.%M.%S")-$COMMIT_HASH 13 | DOCKER_IMAGE_TAG=$DOCKER_IMAGE:$DOCKER_TAG 14 | DOCKER_IMAGE_TAG_LONG=$DOCKER_IMAGE:$DOCKER_TAG_LONG 15 | DOCKER_IMAGE_DEV=$DOCKER_IMAGE:$DOCKER_DEV_TAG 16 | 17 | echo "processing prod release" 18 | docker pull $DOCKER_IMAGE_DEV 19 | docker tag $DOCKER_IMAGE_DEV $DOCKER_IMAGE_TAG 20 | docker tag $DOCKER_IMAGE_DEV $DOCKER_IMAGE_TAG_LONG 21 | docker push $DOCKER_IMAGE_TAG 22 | docker push $DOCKER_IMAGE_TAG_LONG 23 | } 24 | 25 | docker login -u $DOCKER_USER -p $DOCKER_AUTH 26 | 27 | imagedeploy "otp-data-builder" 28 | 29 | imagedeploy "otp-data-tools" 30 | 31 | echo Build completed 32 | -------------------------------------------------------------------------------- /.github/workflows/scripts/build_and_push_dev.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | COMMIT_HASH=$(git rev-parse --short "$GITHUB_SHA") 5 | ORG=hsldevcom 6 | 7 | function imagedeploy { 8 | DOCKER_IMAGE=$ORG/$1 9 | DOCKER_TAG=${DOCKER_BASE_TAG:-latest} 10 | 11 | COMMIT_HASH=$(git rev-parse --short "$GITHUB_SHA") 12 | 13 | DOCKER_TAG_LONG=$DOCKER_TAG-$(date +"%Y-%m-%dT%H.%M.%S")-$COMMIT_HASH 14 | DOCKER_IMAGE_TAG=$DOCKER_IMAGE:$DOCKER_TAG 15 | DOCKER_IMAGE_TAG_LONG=$DOCKER_IMAGE:$DOCKER_TAG_LONG 16 | 17 | # Build image 18 | echo "Building $1" 19 | docker build --network=host --tag=$DOCKER_IMAGE_TAG_LONG . 20 | 21 | echo "Pushing $DOCKER_TAG image" 22 | docker push $DOCKER_IMAGE_TAG_LONG 23 | docker tag $DOCKER_IMAGE_TAG_LONG $DOCKER_IMAGE_TAG 24 | docker push $DOCKER_IMAGE_TAG 25 | } 26 | 27 | docker login -u $DOCKER_USER -p $DOCKER_AUTH 28 | 29 | imagedeploy "otp-data-builder" 30 | 31 | cd otp-data-tools 32 | 33 | imagedeploy "otp-data-tools" 34 | 35 | echo Build completed 36 | -------------------------------------------------------------------------------- /hsl/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataImportReport": true, 3 | "areaVisibility": true, 4 | "staticParkAndRide": false, 5 | "subwayAccessTime": 0, 6 | "maxAreaNodes": 1000, 7 | "maxTransferDuration": "26m", 8 | "multiThreadElevationCalculations": true, 9 | "transitServiceStart": "-P2W", 10 | "transitServiceEnd": "P12W", 11 | "transitModelTimeZone": "Europe/Helsinki", 12 | "fares": "hsl", 13 | "transferRequests": [ 14 | { "modes": "WALK" }, 15 | { 16 | "modes": "WALK", 17 | "wheelchairAccessibility": { 18 | "enabled": true, 19 | "maxSlope": 0.125 20 | } 21 | }, 22 | { "modes": "BICYCLE" } 23 | ], 24 | "gtfsDefaults": { 25 | "stationTransferPreference": "recommended" 26 | }, 27 | "osmDefaults": { 28 | "timeZone": "Europe/Helsinki", 29 | "osmTagMapping": "finland", 30 | "includeOsmSubwayEntrances": true 31 | }, 32 | "demDefaults": { 33 | "elevationUnitMultiplier": 0.1 34 | }, 35 | "emission": { 36 | "carAvgCo2PerKm": 170, 37 | "carAvgOccupancy": 1.3 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /waltti-alt/config.js: -------------------------------------------------------------------------------- 1 | const { mapSrc } = require('../util'); 2 | 3 | module.exports = { 4 | id: 'waltti-alt', 5 | src: [ 6 | mapSrc( 7 | 'WalttiTest', 8 | 'http://digitransit-proxy:8080/out/lmjadmin.mattersoft.fi/feeds/229.zip', 9 | true, 10 | ), 11 | mapSrc('TurkuTrunkroutes', 'http://data-test.foli.fi/gtfs/gtfs.zip', true), 12 | mapSrc( 13 | 'tampere', 14 | 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_mattersoft.zip', 15 | false, 16 | undefined, 17 | { 18 | 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 19 | 'fare_rules.txt': 'digitransit_fare_rules.txt', 20 | }, 21 | ), 22 | mapSrc( 23 | 'digitraffic', 24 | 'https://rata.digitraffic.fi/api/v1/trains/gtfs-passenger-stops.zip', 25 | false, 26 | undefined, 27 | undefined, 28 | { 29 | headers: { 30 | 'Accept-Encoding': 'gzip', 31 | 'Digitraffic-User': 'Digitransit/OTP-dataloading', 32 | }, 33 | }, 34 | ), 35 | ], 36 | osm: ['oulu', 'southFinland'], 37 | }; 38 | -------------------------------------------------------------------------------- /.github/workflows/prod-pipeline.yml: -------------------------------------------------------------------------------- 1 | name: Build v3-prod from release 2 | on: 3 | release: 4 | types: 5 | - published 6 | jobs: 7 | v3-release: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout 11 | uses: actions/checkout@v4 12 | - name: Set time zone to Europe/Helsinki 13 | uses: zcong1993/setup-timezone@master 14 | with: 15 | timezone: 'Europe/Helsinki' 16 | - name: Check Tag 17 | id: check-tag 18 | run: | 19 | if [[ ${GITHUB_REF##*/} =~ ^202[0-9][0-1][0-9][0-3][0-9] ]]; then 20 | echo "match=true" >> $GITHUB_OUTPUT 21 | else 22 | echo invalid release tag 23 | exit 1 24 | fi 25 | - name: Push latest image as v3-prod 26 | if: steps.check-tag.outputs.match == 'true' 27 | run: ./.github/workflows/scripts/push_prod.sh 28 | env: 29 | DOCKER_USER: ${{ secrets.DOCKER_USER }} 30 | DOCKER_AUTH: ${{ secrets.DOCKER_AUTH }} 31 | DOCKER_BASE_TAG: v3-prod 32 | DOCKER_DEV_TAG: ${{ github.event.release.target_commitish }} 33 | -------------------------------------------------------------------------------- /waltti/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataImportReport": true, 3 | "areaVisibility": true, 4 | "staticParkAndRide": false, 5 | "maxAreaNodes": 1000, 6 | "maxTransferDuration": "45m", 7 | "multiThreadElevationCalculations": true, 8 | "transitServiceStart": "-P2W", 9 | "transitServiceEnd": "P12W", 10 | "transitModelTimeZone": "Europe/Helsinki", 11 | "fares": "hsl", 12 | "transferRequests": [ 13 | { "modes": "WALK" }, 14 | { 15 | "modes": "WALK", 16 | "wheelchairAccessibility": { 17 | "enabled": true, 18 | "maxSlope": 0.125 19 | } 20 | }, 21 | { "modes": "BICYCLE" } 22 | ], 23 | "transferParametersForMode": { 24 | "BIKE": { 25 | "maxTransferDuration": "30m" 26 | } 27 | }, 28 | "gtfsDefaults": { 29 | "stationTransferPreference": "recommended" 30 | }, 31 | "boardingLocationTags": ["ref", "ref:findt", "ref:findr"], 32 | "osmDefaults": { 33 | "timeZone": "Europe/Helsinki", 34 | "osmTagMapping": "finland" 35 | }, 36 | "demDefaults": { 37 | "elevationUnitMultiplier": 0.1 38 | }, 39 | "emission": { 40 | "carAvgCo2PerKm": 170, 41 | "carAvgOccupancy": 1.3 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /finland/build-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "dataImportReport": true, 3 | "areaVisibility": true, 4 | "staticParkAndRide": false, 5 | "subwayAccessTime": 0, 6 | "maxAreaNodes": 1000, 7 | "maxTransferDuration": "1h", 8 | "maxStopToShapeSnapDistance": 300, 9 | "transitServiceStart": "-P2W", 10 | "transitServiceEnd": "P12W", 11 | "transitModelTimeZone": "Europe/Helsinki", 12 | "fares": "hsl", 13 | "transferRequests": [ 14 | { "modes": "WALK" }, 15 | { 16 | "modes": "WALK", 17 | "wheelchairAccessibility": { 18 | "enabled": true 19 | } 20 | }, 21 | { "modes": "BICYCLE" }, 22 | { "modes": "CAR" } 23 | ], 24 | "transferParametersForMode": { 25 | "CAR": { 26 | "disableDefaultTransfers": true, 27 | "carsAllowedStopMaxTransferDuration": "2h" 28 | }, 29 | "BIKE": { 30 | "maxTransferDuration": "30m", 31 | "carsAllowedStopMaxTransferDuration": "2h" 32 | } 33 | }, 34 | "boardingLocationTags": ["ref", "ref:findt", "ref:findr"], 35 | "osmDefaults": { 36 | "timeZone": "Europe/Helsinki", 37 | "osmTagMapping": "finland", 38 | "includeOsmSubwayEntrances": true 39 | }, 40 | "emission": { 41 | "carAvgCo2PerKm": 170, 42 | "carAvgOccupancy": 1.3 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /otp-data-server/deploy.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Set these environment variables 4 | #DOCKER_USER // dockerhub credentials 5 | #DOCKER_AUTH 6 | set -e 7 | 8 | cd "$(dirname "$0")" 9 | 10 | ROUTER_NAME=${ROUTER_NAME:-hsl} 11 | DATE=$1 12 | 13 | ORG=${ORG:-hsldevcom} 14 | DOCKER_TAG=${DOCKER_TAG:-v3} 15 | CONTAINER=opentripplanner-data-server 16 | DOCKER_IMAGE=$ORG/$CONTAINER 17 | 18 | if [[ -z "${OTP_GRAPH_DIR}" ]]; then 19 | echo "*** OTP_GRAPH_DIR is not defined." 20 | exit 1 21 | fi 22 | 23 | if [[ -z "${DOCKER_USER}" ]]; then 24 | echo "*** DOCKER_USER is not defined. Unable to log in to the registry." 25 | else 26 | docker login -u $DOCKER_USER -p $DOCKER_AUTH 27 | fi 28 | 29 | DOCKER_DATE_IMAGE=$DOCKER_IMAGE:$DOCKER_TAG-$ROUTER_NAME-$DATE 30 | DOCKER_IMAGE_TAGGED=$DOCKER_IMAGE:$DOCKER_TAG-$ROUTER_NAME 31 | 32 | docker build --progress=plain --network=host --build-arg OTP_GRAPH_DIR=$OTP_GRAPH_DIR -t $DOCKER_DATE_IMAGE . 33 | 34 | docker tag $DOCKER_DATE_IMAGE $DOCKER_IMAGE_TAGGED 35 | 36 | if [[ -z "${DOCKER_USER}" ]]; then 37 | echo "*** Not signed into the registry. Image not pushed." 38 | else 39 | echo "*** Pushing $DOCKER_DATE_IMAGE" 40 | docker push $DOCKER_DATE_IMAGE 41 | echo "*** Pushing $DOCKER_IMAGE_TAGGED" 42 | docker push $DOCKER_IMAGE_TAGGED 43 | fi 44 | -------------------------------------------------------------------------------- /logback-include-extensions.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | /var/opentripplanner/taggedStops.log 6 | false 7 | 8 | false 9 | 11 | 12 | %msg%n 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /opentripplanner/deploy-otp.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #deploys otp image with updated ENTRYPOINT 3 | 4 | set -e 5 | 6 | cd "$(dirname "$0")" 7 | 8 | ROUTER_NAME=${ROUTER_NAME:-hsl} 9 | DATE=$1 10 | 11 | ORG=${ORG:-hsldevcom} 12 | OTP_TAG=${OTP_TAG:-v2} 13 | DOCKER_TAG=$OTP_TAG-$ROUTER_NAME 14 | PROJECT=opentripplanner 15 | DOCKER_IMAGE=$ORG/$PROJECT 16 | DOCKER_DATE_IMAGE=$DOCKER_IMAGE:$DOCKER_TAG-$DATE 17 | DOCKER_IMAGE_TAGGED=$DOCKER_IMAGE:$DOCKER_TAG 18 | 19 | if [[ -z "${OTP_GRAPH_DIR}" ]]; then 20 | echo "*** OTP_GRAPH_DIR is not defined." 21 | exit 1 22 | fi 23 | 24 | if [[ -z "${DOCKER_USER}" ]]; then 25 | echo "*** DOCKER_USER is not defined. Unable to log in to the registry." 26 | else 27 | docker login -u $DOCKER_USER -p $DOCKER_AUTH 28 | fi 29 | 30 | # remove old version (may be necessary in local use) 31 | docker rmi --force $DOCKER_IMAGE_TAGGED &> /dev/null 32 | docker rmi --force $DOCKER_DATE_IMAGE &> /dev/null 33 | 34 | echo "Building router's opentripplanner image..." 35 | 36 | docker build --progress=plain --network=host --build-arg OTP_TAG=$OTP_TAG --build-arg OTP_GRAPH_DIR=$OTP_GRAPH_DIR -t $DOCKER_IMAGE_TAGGED . 37 | 38 | docker tag $DOCKER_IMAGE_TAGGED $DOCKER_DATE_IMAGE 39 | 40 | if [[ -z "${DOCKER_USER}" ]]; then 41 | echo "*** Not signed into the registry. Image not pushed." 42 | else 43 | echo "*** Pushing $DOCKER_IMAGE_TAGGED" 44 | docker push $DOCKER_IMAGE_TAGGED 45 | echo "*** Pushing $DOCKER_DATE_IMAGE" 46 | docker push $DOCKER_DATE_IMAGE 47 | fi 48 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "otp-data-builder", 3 | "version": "2.0.0", 4 | "description": "[![Build](https://github.com/hsldevcom/OpenTripPlanner-data-container/workflows/Process%20master%20push%20or%20pr/badge.svg?branch=master)](https://github.com/HSLdevcom/OpenTripPlanner-data-container/actions)", 5 | "scripts": { 6 | "format": "prettier --write .", 7 | "lint": "eslint .", 8 | "test": "prettier --check . && npm run lint", 9 | "start": "node index.js" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/HSLdevcom/OpenTripPlanner-data-container.git" 14 | }, 15 | "author": "Digitransit", 16 | "license": "ISC", 17 | "bugs": { 18 | "url": "https://github.com/HSLdevcom/OpenTripPlanner-data-container/issues" 19 | }, 20 | "homepage": "https://github.com/HSLdevcom/OpenTripPlanner-data-container#readme", 21 | "dependencies": { 22 | "axios": "^1.6.7", 23 | "cloneable-readable": "^3.0.0", 24 | "csv-parser": "^3.0.0", 25 | "csv-stringify": "^6.4.4", 26 | "del": "^6.1.1", 27 | "fs-extra": "^11.1.1", 28 | "gulp": "^4.0.2", 29 | "gulp-rename": "^2.1.0", 30 | "json-2-csv": "^5.0.1", 31 | "osm-pbf-parser": "^2.3.0", 32 | "remove-bom-stream": "^2.0.0", 33 | "through2": "^4.0.2", 34 | "vinyl": "^3.0.0" 35 | }, 36 | "devDependencies": { 37 | "@eslint/js": "^9.18.0", 38 | "eslint": "^9.18.0", 39 | "globals": "^15.14.0", 40 | "prettier": "^3.4.2" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /task/Download.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const axios = require('axios'); 3 | const { postSlackMessage, createDir } = require('../util'); 4 | 5 | function handleFail(url, err) { 6 | postSlackMessage(`${url} Download failed: ${err} :boom:`); 7 | global.hasFailures = true; 8 | } 9 | 10 | /** 11 | * Download external data (gtfs, osm) resources. 12 | */ 13 | function download(entry, dir) { 14 | return new Promise(resolve => { 15 | createDir(dir); 16 | process.stdout.write('Downloading ' + entry.url + '...\n'); 17 | const name = entry.url.split('/').pop(); 18 | const ext = name.indexOf('.') > 0 ? '.' + name.split('.').pop() : ''; 19 | const filePath = `${dir}/${entry.id + ext}`; 20 | const request = { 21 | method: 'GET', 22 | url: entry.url, 23 | responseType: 'stream', 24 | ...entry.request, 25 | }; 26 | axios(request) 27 | .then(response => { 28 | response.data.pipe(fs.createWriteStream(filePath)); 29 | response.data.on('error', err => { 30 | handleFail(entry.url, err); 31 | resolve(); 32 | }); 33 | response.data.on('end', () => { 34 | process.stdout.write(entry.url + ' Download SUCCESS\n'); 35 | resolve(); 36 | }); 37 | }) 38 | .catch(err => { 39 | handleFail(entry.url, err); 40 | resolve(); 41 | }); 42 | }); 43 | } 44 | 45 | module.exports = async function dlSequentially(entries, dir) { 46 | for (const e of entries) { 47 | await download(e, dir); 48 | } 49 | }; 50 | -------------------------------------------------------------------------------- /task/BlobValidation.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const through = require('through2'); 3 | 4 | /** 5 | * Checks if downloaded file is at most 1% smaller than the seeded file. 6 | * If there was no seeded file, validation is always successful 7 | */ 8 | function validateSize(seededFile, downloadedFile) { 9 | if (!fs.existsSync(downloadedFile)) { 10 | process.stdout.write(downloadedFile + ' does not exist!\n'); 11 | return false; 12 | } 13 | if (process.env.DISABLE_BLOB_VALIDATION || !fs.existsSync(seededFile)) { 14 | process.stdout.write('Skipping blob size validation\n'); 15 | global.blobSizeOk = true; 16 | return true; 17 | } 18 | const downloadedFileSize = fs.statSync(downloadedFile).size; 19 | const seedFileSize = fs.statSync(seededFile).size; 20 | if (seedFileSize * 0.99 <= downloadedFileSize) { 21 | process.stdout.write('Blob size validated\n'); 22 | global.blobSizeOk = true; 23 | return true; 24 | } else { 25 | process.stdout.write( 26 | downloadedFile + ': file had different size than the seeded file\n', 27 | ); 28 | return false; 29 | } 30 | } 31 | 32 | module.exports = { 33 | validateBlobSize: () => { 34 | return through.obj(function (file, encoding, callback) { 35 | const localFile = file.history[file.history.length - 1]; 36 | const seededFile = localFile.replace('/downloads/', '/ready/'); 37 | if (validateSize(seededFile, localFile)) { 38 | callback(null, file); 39 | } else { 40 | callback(null, null); 41 | } 42 | }); 43 | }, 44 | }; 45 | -------------------------------------------------------------------------------- /task/Seed.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const { extractAllFiles } = require('./ZipTask'); 3 | const { postSlackMessage, dirNameToDate } = require('../util'); 4 | 5 | function findLatestZip(sourceDir, routerId, tag) { 6 | const basePath = `${sourceDir}/${tag}`; 7 | let latestDirName; 8 | let latestDate = null; 9 | fs.readdirSync(basePath).forEach(file => { 10 | const filePath = `${basePath}/${file}`; 11 | if (!fs.lstatSync(filePath).isDirectory()) { 12 | return; 13 | } 14 | const routers = fs.readdirSync(filePath); 15 | if (routers.length !== 1 || routers[0] !== routerId) { 16 | return; 17 | } 18 | // Directory names follow ISO 8601 format without milliseconds and 19 | // with ':' replaced with '.'. 20 | const date = dirNameToDate(file); 21 | if (date != null && (latestDate == null || date > latestDate)) { 22 | latestDate = date; 23 | latestDirName = file; 24 | } 25 | }); 26 | return `${basePath}/${latestDirName}/${routerId}/router-${routerId}.zip`; 27 | } 28 | 29 | /** 30 | * Unzip latest version of the data for router into destinationDir so it can be used as 31 | * the basis for the new build if some parts of the data can't be updated. 32 | */ 33 | module.exports = function (sourceDir, destinationDir, routerId, tag) { 34 | return new Promise((resolve, reject) => { 35 | try { 36 | extractAllFiles(findLatestZip(sourceDir, routerId, tag), destinationDir); 37 | resolve(); 38 | } catch (err) { 39 | postSlackMessage(`Seed failed due to: ${err}`); 40 | reject(err); 41 | } 42 | }); 43 | }; 44 | -------------------------------------------------------------------------------- /kela/config.js: -------------------------------------------------------------------------------- 1 | const { mapSrc } = require('../util'); 2 | 3 | // matkahuolto data source often fails when accessed through digitransit proxy 4 | // here we exceptionally set up direct calls with basic auth 5 | let mhAddress; 6 | if (process.env.MH_BASIC_AUTH) { 7 | const basic = Buffer.from(process.env.MH_BASIC_AUTH, 'base64').toString( 8 | 'utf8', 9 | ); 10 | mhAddress = `https://${basic}@minfoapi.matkahuolto.fi/gtfs/kokomaa-fi/gtfs.zip`; 11 | } else { 12 | mhAddress = 13 | 'http://digitransit-proxy:8080/out/minfoapi.matkahuolto.fi/gtfs/kokomaa-fi/gtfs.zip'; 14 | } 15 | 16 | module.exports = { 17 | id: 'kela', 18 | src: [ 19 | mapSrc( 20 | 'kela', 21 | 'https://mobility.mobility-database.fintraffic.fi/static/Kela_suuret.zip', 22 | false, 23 | ['kela/gtfs-rules/remove-route-color.rule'], 24 | ), 25 | mapSrc( 26 | 'kela_varely', 27 | 'https://mobility.mobility-database.fintraffic.fi/static/Kela_varely.zip', 28 | false, 29 | ['kela/gtfs-rules/remove-route-color.rule'], 30 | ), 31 | mapSrc( 32 | 'kela_waltti', 33 | 'https://mobility.mobility-database.fintraffic.fi/static/kela_waltti.zip', 34 | false, 35 | ['kela/gtfs-rules/remove-route-color.rule'], 36 | ), 37 | mapSrc( 38 | 'matkahuolto', 39 | mhAddress, 40 | false, 41 | [ 42 | 'kela/gtfs-rules/matkahuolto.rule', 43 | 'kela/gtfs-rules/remove-matching-route.rule', 44 | 'kela/gtfs-rules/remove-route-color.rule', 45 | ], 46 | { 'transfers.txt': null }, 47 | ), 48 | ], 49 | osm: ['finland'], 50 | }; 51 | -------------------------------------------------------------------------------- /task/GTFSReplace.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const cloneable = require('cloneable-readable'); 3 | const through = require('through2'); 4 | const { parseId, postSlackMessage } = require('../util'); 5 | const { 6 | renameFilesInZip, 7 | removeFilesFromZip, 8 | zipHasFile, 9 | } = require('./ZipTask'); 10 | 11 | const replaceGTFSFiles = (replacements, fileName) => { 12 | const filesToRemove = []; 13 | const replacementsForFiles = {}; 14 | for (const [fileToReplace, replacementFile] of Object.entries(replacements)) { 15 | if (replacementFile) { 16 | // If replacement file doesn't exist (anymore), don't do anything else than message 17 | if (!zipHasFile(fileName, replacementFile)) { 18 | const msg = `${replacementFile} not found in ${fileName}. ${fileToReplace} is not replaced.`; 19 | postSlackMessage(msg); 20 | process.stdout.write(`${msg}\n`); 21 | continue; 22 | } 23 | replacementsForFiles[fileToReplace] = replacementFile; 24 | } 25 | filesToRemove.push(fileToReplace); 26 | } 27 | removeFilesFromZip(fileName, filesToRemove); 28 | renameFilesInZip(fileName, replacementsForFiles); 29 | }; 30 | 31 | module.exports = { 32 | replaceGTFSFilesTask: configMap => { 33 | return through.obj(function (file, encoding, callback) { 34 | const gtfsFile = file.history[file.history.length - 1]; 35 | const id = parseId(gtfsFile); 36 | const config = configMap[id]; 37 | const replacements = config ? config.replacements : null; 38 | if (!replacements) { 39 | callback(null, file); 40 | } else { 41 | process.stdout.write(`Replacing files in source ${id} \n`); 42 | replaceGTFSFiles(replacements, file.path); 43 | file.contents = cloneable(fs.createReadStream(file.path)); 44 | callback(null, file); 45 | } 46 | }); 47 | }, 48 | }; 49 | -------------------------------------------------------------------------------- /.github/workflows/dev-pipeline.yml: -------------------------------------------------------------------------------- 1 | name: Process push or pr 2 | on: 3 | push: 4 | branches: 5 | - v3 6 | - custom-release 7 | pull_request: 8 | branches: 9 | - v3 10 | jobs: 11 | lint: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | node-version: [22.x] 16 | steps: 17 | - name: Checkout 18 | uses: actions/checkout@v4 19 | - name: Use Node.js ${{ matrix.node-version }} 20 | uses: actions/setup-node@v4 21 | with: 22 | node-version: ${{ matrix.node-version }} 23 | - name: Cache node modules 24 | uses: actions/cache@v4 25 | with: 26 | path: '**/node_modules' 27 | key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }} 28 | - name: Install dependencies 29 | run: npm install 30 | - name: Run linters 31 | run: npm run test 32 | docker-push: 33 | if: github.event_name != 'pull_request' 34 | needs: 35 | - lint 36 | runs-on: ubuntu-latest 37 | strategy: 38 | matrix: 39 | node-version: [22.x] 40 | steps: 41 | - name: Checkout 42 | uses: actions/checkout@v4 43 | - name: Set time zone to Europe/Helsinki 44 | uses: zcong1993/setup-timezone@master 45 | with: 46 | timezone: 'Europe/Helsinki' 47 | - name: Use Node.js ${{ matrix.node-version }} 48 | uses: actions/setup-node@v4 49 | with: 50 | node-version: ${{ matrix.node-version }} 51 | - name: Cache node modules 52 | uses: actions/cache@v4 53 | with: 54 | path: '**/node_modules' 55 | key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }} 56 | - name: Install dependencies 57 | run: npm install 58 | - name: Build docker image from ${{ github.ref_name }} and push it 59 | run: ./.github/workflows/scripts/build_and_push_dev.sh 60 | env: 61 | DOCKER_USER: ${{ secrets.DOCKER_USER }} 62 | DOCKER_AUTH: ${{ secrets.DOCKER_AUTH }} 63 | DOCKER_BASE_TAG: ${{ github.ref_name }} 64 | -------------------------------------------------------------------------------- /task/StorageCleanup.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const del = require('del'); 3 | const { dirNameToDate } = require('../util'); 4 | 5 | /* 6 | * Removes files which are not directories, directories which are empty or can't be parsed into dates. 7 | */ 8 | function deleteInvalidVersions(basePath) { 9 | const deletionPromises = []; 10 | fs.readdirSync(basePath).forEach(file => { 11 | const filePath = `${basePath}/${file}`; 12 | if (!fs.lstatSync(filePath).isDirectory()) { 13 | deletionPromises.push(del(filePath)); 14 | return; 15 | } 16 | if (dirNameToDate(file) === null) { 17 | deletionPromises.push(del(filePath)); 18 | return; 19 | } 20 | const files = fs.readdirSync(filePath); 21 | if (files.length === 0) { 22 | deletionPromises.push(del(filePath)); 23 | } 24 | }); 25 | return deletionPromises; 26 | } 27 | 28 | function sortByDate(firstFile, secondFile) { 29 | const firstDate = dirNameToDate(firstFile); 30 | const secondDate = dirNameToDate(secondFile); 31 | if (firstDate < secondDate) { 32 | return -1; 33 | } 34 | if (firstDate > secondDate) { 35 | return 1; 36 | } 37 | return 0; 38 | } 39 | 40 | function isCorrectRouter(basePath, file, routerId) { 41 | const routers = fs.readdirSync(`${basePath}/${file}`); 42 | return routers.length === 1 && routers[0] === routerId; 43 | } 44 | 45 | /* 46 | * Keeps 10 valid latest versions and removes the rest. 47 | */ 48 | function deleteOldVersions(sourceDir, routerId, tag) { 49 | const basePath = `${sourceDir}/${tag}`; 50 | if (!fs.existsSync(basePath)) { 51 | return Promise(res => res()); 52 | } 53 | return Promise.all(deleteInvalidVersions(basePath)).then(() => { 54 | const filesToDelete = fs 55 | .readdirSync(basePath) 56 | .filter(file => isCorrectRouter(basePath, file, routerId)) 57 | .sort(sortByDate) 58 | .slice(0, -10); 59 | return filesToDelete.map(file => del(`${basePath}/${file}/${routerId}`)); 60 | }); 61 | } 62 | 63 | module.exports = function (sourceDir, routerId, tag) { 64 | return deleteOldVersions(sourceDir, routerId, tag); 65 | }; 66 | -------------------------------------------------------------------------------- /finland/gtfs-rules/matka.rule: -------------------------------------------------------------------------------- 1 | # Stops at 0 of some coordinate system 2 | {"op":"remove", "match":{"file":"stops.txt","stop_lon":"20.142573"}} 3 | # All trains 4 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"2"}} 5 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"100"}} 6 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"101"}} 7 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"102"}} 8 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"103"}} 9 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"104"}} 10 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"105"}} 11 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"106"}} 12 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"107"}} 13 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"108"}} 14 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"109"}} 15 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"110"}} 16 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"111"}} 17 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"112"}} 18 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"113"}} 19 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"114"}} 20 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"115"}} 21 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"116"}} 22 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"117"}} 23 | # HSL 24 | {"op":"remove", "match":{"file":"agency.txt", "agency_id":"2"}} 25 | # Tampere old 26 | {"op":"remove", "match":{"file":"agency.txt", "agency_id":"3"}} 27 | # Tampere new 28 | {"op":"remove", "match":{"file":"agency.txt", "agency_name":"Nysse"}} 29 | # Oulu 30 | {"op":"remove", "match":{"file":"agency.txt", "agency_name":"Oulun joukkoliikenne"}} 31 | 32 | # Jyväskylä does not need filtering at the moment. Koontikanta does not include local Linkki traffic 33 | # and Waltti GTFS for Jyväskylä does not include long distance routes operated by Jyväskylän liikenne 34 | 35 | # Waltti demand responsive transport. BTW: why real on demand traffic is removed? 36 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"6"}} 37 | {"op":"remove", "match":{"file":"routes.txt", "route_type":"715"}} 38 | 39 | # Remove shape_dist_traveled from stop_times.txt as values in waltti GTFS are bogus 40 | {"op":"update", "match":{"file":"stop_times.txt", }, "update":{"shape_dist_traveled":-999.0}} 41 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set +e 3 | 4 | # set defaults 5 | ORG=${ORG:-hsldevcom} 6 | JAVA_OPTS=${JAVA_OPTS:--Xmx12g} 7 | ROUTER_NAME=${ROUTER_NAME:-hsl} 8 | OTP_TAG=${OTP_TAG:-v2} 9 | TOOLS_TAG=${TOOLS_TAG:-v3} 10 | 11 | # set useful variables 12 | TOOL_IMAGE=$ORG/otp-data-tools:$TOOLS_TAG 13 | 14 | OTPCONT=otp-$ROUTER_NAME 15 | TOOLCONT=otp-data-tools 16 | 17 | function shutdown() { 18 | echo shutting down 19 | docker stop $OTPCONT 20 | docker rm $TOOLCONT &> /dev/null 21 | } 22 | 23 | echo -e "\n##### Testing new data #####\n" 24 | 25 | echo Starting otp... 26 | 27 | docker run --rm --name $OTPCONT -e JAVA_OPTS="$JAVA_OPTS" \ 28 | --mount type=bind,source=$(pwd)/logback-include-extensions.xml,target=/logback-include-extensions.xml \ 29 | --mount type=bind,source=$(pwd)/data/build/$ROUTER_NAME/graph.obj,target=/var/opentripplanner/graph.obj \ 30 | --mount type=bind,source=$(pwd)/data/build/$ROUTER_NAME/otp-config.json,target=/var/opentripplanner/otp-config.json \ 31 | --mount type=bind,source=$(pwd)/data/build/$ROUTER_NAME/router-config.json,target=/var/opentripplanner/router-config.json \ 32 | $ORG/opentripplanner:$OTP_TAG --load --serve & 33 | sleep 10 34 | 35 | echo Getting otp ip.. 36 | timeout=$(($(date +%s) + 480)) 37 | until IP=$(docker inspect --format '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $OTPCONT) || [[ $(date +%s) -gt $timeout ]]; do sleep 1;done; 38 | 39 | if [ "$IP" == "" ]; then 40 | echo Could not get ip. failing test 41 | shutdown 42 | exit 1 43 | fi 44 | 45 | echo Got otp ip: $IP 46 | 47 | OTP_URL=http://$IP:8080/otp 48 | 49 | for (( c=1; c<=20; c++ ));do 50 | STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" $OTP_URL/actuators/health || true) 51 | 52 | if [ $STATUS_CODE = 200 ]; then 53 | echo OTP started 54 | curl -s $OTP_URL/gtfs/v1 -H "Content-Type: application/graphql" --data "{agencies {name}}" |grep error 55 | if [ $? = 1 ]; then #grep finds no error 56 | echo OTP works 57 | break 58 | else 59 | echo OTP has errors 60 | shutdown 61 | exit 1 62 | fi 63 | else 64 | echo waiting for service 65 | sleep 30 66 | fi 67 | done 68 | 69 | echo running otpqa 70 | 71 | docker pull $TOOL_IMAGE 72 | docker run --entrypoint /bin/bash --name $TOOLCONT $TOOL_IMAGE -c "cd OTPQA; /python-venv/bin/python3 otpprofiler_json.py $OTP_URL/gtfs/v1 $ROUTER_NAME $SKIPPED_SITES" 73 | 74 | if [ $? == 0 ]; then 75 | echo getting failed feed list from container 76 | docker cp $TOOLCONT:/OTPQA/failed_feeds.txt . &> /dev/null 77 | shutdown 78 | exit 0 79 | else 80 | shutdown 81 | exit 1 82 | fi 83 | -------------------------------------------------------------------------------- /task/OBAFilter.js: -------------------------------------------------------------------------------- 1 | const del = require('del'); 2 | const execSync = require('child_process').execSync; 3 | const through = require('through2'); 4 | const fs = require('fs-extra'); 5 | const path = require('path'); 6 | const cloneable = require('cloneable-readable'); 7 | const { zipDirContents } = require('./ZipTask'); 8 | const { dataToolImage } = require('../config.js'); 9 | const { dataDir } = require('../config.js'); 10 | const { postSlackMessage, parseId } = require('../util'); 11 | 12 | function OBAFilter(src, dst, rule) { 13 | process.stdout.write(`filtering ${src} with ${rule}...\n`); 14 | 15 | const cmd = `docker run -v ${dataDir}:/data --rm ${dataToolImage} --transform=/data/${rule} /data/${src} /data/${dst}`; 16 | 17 | try { 18 | execSync(cmd, { stdio: [0, 1, 2] }); 19 | return true; 20 | } catch { 21 | return false; 22 | } 23 | } 24 | 25 | module.exports = { 26 | OBAFilterTask: gtfsMap => { 27 | return through.obj(function (file, encoding, callback) { 28 | const gtfsFile = file.history[file.history.length - 1]; 29 | const relativeFilename = path.relative(dataDir, gtfsFile); 30 | const id = parseId(gtfsFile); 31 | const source = gtfsMap[id]; 32 | const rules = source.rules; 33 | if (rules) { 34 | const src = `${relativeFilename}`; 35 | const dst = `${relativeFilename}-filtered`; 36 | const dstDir = `${dataDir}/${dst}`; 37 | 38 | // execute all rules 39 | // result zip of a rule is input data for next rule 40 | // async zip creation is synchronized using recursion: 41 | // next recursion call is launched from zip callback 42 | let i = 0; 43 | function processRule() { 44 | if (i < rules.length) { 45 | const rule = rules[i++]; 46 | if (OBAFilter(src, dst, rule)) { 47 | fs.unlinkSync(`${dataDir}/${src}`); 48 | /* create zip named src from files in dst */ 49 | if (zipDirContents(`${dataDir}/${src}`, `${dataDir}/${dst}`)) { 50 | del(dstDir); 51 | process.stdout.write( 52 | `Filter ${gtfsFile} with rule ${rule} SUCCESS\n`, 53 | ); 54 | processRule(); // handle next rule 55 | } else { 56 | del(dstDir); 57 | postSlackMessage(`OBA zip task failed :boom:`); 58 | callback(null, null); 59 | } 60 | } else { 61 | // failure 62 | del(dstDir); 63 | postSlackMessage(`Rule ${rule} on ${gtfsFile} failed :boom:`); 64 | callback(null, null); 65 | } 66 | } else { 67 | // all rules done successfully 68 | file.contents = cloneable(fs.createReadStream(gtfsFile)); 69 | callback(null, file); 70 | } 71 | } 72 | processRule(); // start recursive rule processing 73 | } else { 74 | process.stdout.write(gtfsFile + ' filter skipped\n'); 75 | callback(null, file); 76 | } 77 | }); 78 | }, 79 | }; 80 | -------------------------------------------------------------------------------- /kela/router-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "routingDefaults": { 3 | "transferSlack": "1m30s", 4 | "waitReluctance": 0.99, 5 | "elevatorBoardTime": 60, 6 | "bicycle": { 7 | "boardCost": 120, 8 | "reluctance": 1.7 9 | }, 10 | "car": { 11 | "reluctance": 10.0 12 | }, 13 | "walk": { 14 | "speed": 1.3, 15 | "reluctance": 1.75, 16 | "stairsReluctance": 1.2, 17 | "stairsTimeFactor": 2, 18 | "escalator": { 19 | "speed": 0.65 20 | }, 21 | "boardCost": 120 22 | }, 23 | "accessEgress": { 24 | "maxDuration": "8h", 25 | "maxStopCount": "400" 26 | }, 27 | "maxDirectStreetDuration": "100h", 28 | "maxDirectStreetDurationForMode": { 29 | "walk": "90m" 30 | }, 31 | "maxJourneyDuration": "24h", 32 | "streetRoutingTimeout": "9s", 33 | "wheelchairAccessibility": { 34 | "stop": { 35 | "onlyConsiderAccessible": false, 36 | "unknownCost": 0, 37 | "inaccessibleCost": 100000 38 | }, 39 | "maxSlope": 0.125 40 | }, 41 | "itineraryFilters": { 42 | "transitGeneralizedCostLimit": { 43 | "costLimitFunction": "600 + 1.5x" 44 | }, 45 | "nonTransitGeneralizedCostLimit": "400 + 1.5x" 46 | }, 47 | "intersectionTraversalModel": "CONSTANT" 48 | }, 49 | "gtfsApi": { 50 | "tracingTags": ["digitransit-subscription-id"] 51 | }, 52 | "transit": { 53 | "dynamicSearchWindow": { 54 | "minWindow": "2h" 55 | }, 56 | "pagingSearchWindowAdjustments": ["8h", "4h", "4h", "4h", "4h"], 57 | "transferCacheRequests": [ 58 | { 59 | "modes": "WALK", 60 | "walk": { 61 | "speed": 1.2, 62 | "reluctance": 1.8 63 | } 64 | }, 65 | { 66 | "modes": "WALK", 67 | "walk": { 68 | "speed": 1.2, 69 | "reluctance": 1.8 70 | }, 71 | "wheelchairAccessibility": { 72 | "enabled": true 73 | } 74 | }, 75 | { 76 | "modes": "WALK", 77 | "walk": { 78 | "speed": 1.67, 79 | "reluctance": 1.8 80 | } 81 | } 82 | ] 83 | }, 84 | "vectorTiles": { 85 | "attribution": "Digitransit data is licensed under CC BY 4.0.", 86 | "layers": [ 87 | { 88 | "name": "stops", 89 | "type": "Stop", 90 | "mapper": "Digitransit", 91 | "maxZoom": 20, 92 | "minZoom": 5, 93 | "cacheMaxSeconds": 43200 94 | }, 95 | { 96 | "name": "realtimeStops", 97 | "type": "Stop", 98 | "mapper": "DigitransitRealtime", 99 | "maxZoom": 20, 100 | "minZoom": 5, 101 | "cacheMaxSeconds": 60 102 | }, 103 | { 104 | "name": "stations", 105 | "type": "Station", 106 | "mapper": "Digitransit", 107 | "maxZoom": 20, 108 | "minZoom": 5, 109 | "cacheMaxSeconds": 43200 110 | } 111 | ] 112 | }, 113 | "updaters": [] 114 | } 115 | -------------------------------------------------------------------------------- /task/DownloadDEMBlob.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const axios = require('axios'); 3 | const { dataDir } = require('../config'); 4 | 5 | /** 6 | * Download DEM files from Azure blob storage. 7 | */ 8 | module.exports = function (entries) { 9 | return entries.map( 10 | entry => 11 | new Promise((resolve, reject) => { 12 | const filePath = `${dataDir}/downloads/dem/${entry.id}.tif`; 13 | const readyPath = `${dataDir}/ready/dem/${entry.id}.tif`; 14 | let dataAlreadyExists = false; 15 | let downloadSize; 16 | let readySize; 17 | const abortController = new AbortController(); 18 | 19 | if (fs.existsSync(readyPath)) { 20 | readySize = fs.statSync(readyPath).size; 21 | } 22 | axios({ 23 | method: 'GET', 24 | url: entry.url, 25 | responseType: 'stream', 26 | signal: abortController.signal, 27 | }) 28 | .then(response => { 29 | if (response.status === 200) { 30 | downloadSize = response.headers['content-length']; 31 | if (readySize && readySize === parseInt(downloadSize)) { 32 | process.stdout.write( 33 | `Local DEM data for ${entry.id} was already up-to-date\n`, 34 | ); 35 | dataAlreadyExists = true; 36 | // Abort download as remote has same size as local copy 37 | abortController.abort(); 38 | resolve(); 39 | } else { 40 | response.data.pipe(fs.createWriteStream(filePath)); 41 | process.stdout.write( 42 | `Downloading new DEM data from ${entry.url}\n`, 43 | ); 44 | } 45 | } 46 | response.data.on('error', err => { 47 | if (!dataAlreadyExists) { 48 | process.stdout.write( 49 | `${entry.url} download failed: ${JSON.stringify(err)} \n`, 50 | ); 51 | reject(err); 52 | } else { 53 | resolve(); 54 | } 55 | }); 56 | response.data.on('end', () => { 57 | // If new file was downloaded, this resolves with the file's path 58 | // This is also called when request is aborted but new call to resolve shouldn't do anything 59 | // However, if the file is really small, this could in theory be called before call to abort request 60 | // but that situation shouldn't happen with DEM data sizes. 61 | if (!dataAlreadyExists) { 62 | process.stdout.write( 63 | `Downloaded updated DEM data to ${filePath}\n`, 64 | ); 65 | fs.rename(filePath, readyPath, err => { 66 | if (err) { 67 | process.stdout.write(JSON.stringify(err)); 68 | process.stdout.write( 69 | `Failed to move DEM data from ${readyPath}\n`, 70 | ); 71 | reject(err); 72 | } else { 73 | process.stdout.write(`DEM data updated for ${entry.id}\n`); 74 | resolve(); 75 | } 76 | }); 77 | } else { 78 | resolve(); 79 | } 80 | }); 81 | }) 82 | .catch(err => { 83 | process.stdout.write( 84 | `${entry.url} download failed: ${JSON.stringify(err)}\n`, 85 | ); 86 | reject(err); 87 | }); 88 | }), 89 | ); 90 | }; 91 | -------------------------------------------------------------------------------- /task/SetFeedId.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const cloneable = require('cloneable-readable'); 3 | const converter = require('json-2-csv'); 4 | const through = require('through2'); 5 | // const cloneable = require('cloneable-readable'); 6 | const { postSlackMessage, parseId } = require('../util'); 7 | const { dataDir } = require('../config.js'); 8 | 9 | function createFeedInfo(file, id) { 10 | process.stdout.write(`Generating new feed_info for ${id}\n`); 11 | const csv = `feed_publisher_name,feed_publisher_url,feed_lang,feed_id 12 | ${id}-fake-name,${id}-fake-url,${id}-fake-lang,${id}\n`; 13 | fs.writeFileSync(file, csv); 14 | } 15 | 16 | function setFeedId(file, id) { 17 | try { 18 | if (!fs.existsSync(file)) { 19 | createFeedInfo(file, id); 20 | } else { 21 | const data = fs.readFileSync(file, { 22 | encoding: 'utf8', 23 | flag: 'r', 24 | }); 25 | // Remove unnecessary control characters that break things 26 | let filteredData = data.replace(/\r/g, ''); 27 | if (filteredData.charAt(filteredData.length - 1) === '\n') { 28 | filteredData = filteredData.slice(0, -1); 29 | } 30 | if (filteredData.charCodeAt(0) === 0xfeff) { 31 | // remove BOM 32 | filteredData = filteredData.substr(1); 33 | } 34 | const json = converter.csv2json(filteredData); 35 | if (json.length > 0) { 36 | if (process.env.VERSION_CHECK) { 37 | const EIGHT_HOURS = 8 * 60 * 60 * 1000; 38 | const idsToCheck = process.env.VERSION_CHECK.replace(/ /g, '').split( 39 | ',', 40 | ); 41 | const now = new Date(); 42 | // check if a warning should be shown about feed_version timestamp being over 8 hours in the past 43 | if ( 44 | idsToCheck.includes(id) && 45 | json[0].feed_version !== undefined && 46 | now - new Date(json[0].feed_version) > EIGHT_HOURS 47 | ) { 48 | const msg = `GTFS data for ${id} is older than 8 hours`; 49 | process.stdout.write(`${msg}\n`); 50 | // send warning also to slack between monday and friday 51 | const day = now.getDay(); 52 | if (day !== 1) { 53 | postSlackMessage(`${msg} :boom:`); 54 | } 55 | } 56 | } 57 | // no id or id is wrong 58 | if (json[0].feed_id === undefined || json[0].feed_id !== id) { 59 | json[0].feed_id = id; 60 | const csv = converter.json2csv(json); 61 | fs.writeFileSync(file, csv); 62 | } else { 63 | process.stdout.write('Correct feed id was already set\n'); 64 | return 'ok'; 65 | } 66 | } else { 67 | createFeedInfo(file, id); 68 | } 69 | } 70 | } catch (err) { 71 | return err; 72 | } 73 | return 'ok'; 74 | } 75 | 76 | module.exports = { 77 | /** 78 | * Sets gtfs feed id into feed_info.txt 79 | */ 80 | setFeedIdTask: () => { 81 | return through.obj(function (file, encoding, callback) { 82 | const gtfsFile = file.history[file.history.length - 1]; 83 | const id = parseId(gtfsFile); 84 | const infoFile = `${dataDir}/tmp/${id}/feed_info.txt`; 85 | 86 | process.stdout.write(`${gtfsFile} setting GTFS feed id to ${id} \n`); 87 | const action = setFeedId(infoFile, id); 88 | if (action !== 'ok') { 89 | process.stdout.write(`Feed id editing failed: ${action}\n`); 90 | throw new Error(`Failed to edit feed id for ${id}`); 91 | } 92 | process.stdout.write(`${gtfsFile} feed id SUCCESS\n`); 93 | file.contents = cloneable(fs.createReadStream(gtfsFile)); 94 | callback(null, file); 95 | }); 96 | }, 97 | }; 98 | -------------------------------------------------------------------------------- /config.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | 3 | // OBA filter erases files which it does not recognize from GTFS packages 4 | // this array specifies the file names which should be preserved 5 | const passOBAfilter = ['emissions.txt', 'translations.txt']; 6 | 7 | assert(process.env.ROUTER_NAME !== undefined, 'ROUTER_NAME must be defined'); 8 | 9 | // Require router config from router directory 10 | const router = require(`./${process.env.ROUTER_NAME}/config`); 11 | 12 | // EXTRA_SRC format should be {"FOLI": {"url": "https://data.foli.fi/gtfs/gtfs.zip", "fit": false, "rules": ["waltti/gtfs-rules/waltti.rule"]}} 13 | // but you can only define, for example, new url and the other key value pairs will remain the same as they are defined in this file. 14 | // It is also possible to add completely new src by defining object with unused id or to remove a src by defining "remove": true 15 | const extraSrc = 16 | process.env.EXTRA_SRC !== undefined ? JSON.parse(process.env.EXTRA_SRC) : {}; 17 | 18 | const SPLIT_BUILD_TYPE = process.env.SPLIT_BUILD_TYPE || 'NO_SPLIT_BUILD'; 19 | 20 | const usedSrc = []; 21 | 22 | // override source values if they are defined in extraSrc 23 | const rt = router; 24 | const sources = rt.src; 25 | for (let j = sources.length - 1; j >= 0; j--) { 26 | const src = sources[j]; 27 | const id = src.id; 28 | if (extraSrc[id]) { 29 | usedSrc.push(id); 30 | if (extraSrc[id].remove) { 31 | sources.splice(j, 1); 32 | continue; 33 | } 34 | sources[j] = { ...src, ...extraSrc[id] }; 35 | } 36 | sources[j].config = rt; 37 | } 38 | 39 | // Go through extraSrc keys to find keys that don't already exist in src and add those as new src 40 | Object.keys(extraSrc).forEach(id => { 41 | if (!usedSrc.includes(id)) { 42 | router.src.push({ ...extraSrc[id], id }); 43 | } 44 | }); 45 | 46 | // create id->src-entry map 47 | const gtfsMap = {}; 48 | router.src.forEach(src => { 49 | gtfsMap[src.id] = src; 50 | }); 51 | 52 | const extraOSM = 53 | process.env.EXTRA_OSM !== undefined ? JSON.parse(process.env.EXTRA_OSM) : {}; 54 | 55 | const osm = { 56 | estonia: 'https://download.geofabrik.de/europe/estonia-latest.osm.pbf', 57 | finland: 'https://download.geofabrik.de/europe/finland-latest.osm.pbf', 58 | hsl: 'https://karttapalvelu.storage.hsldev.com/hsl.osm/hsl.osm.pbf', 59 | kajaani: 60 | 'https://karttapalvelu.storage.hsldev.com/waltti.osm/kajaani.osm.pbf', 61 | oulu: 'https://karttapalvelu.storage.hsldev.com/waltti.osm/oulu.osm.pbf', 62 | rovaniemi: 63 | 'https://karttapalvelu.storage.hsldev.com/waltti.osm/rovaniemi.osm.pbf', 64 | southFinland: 65 | 'https://karttapalvelu.storage.hsldev.com/waltti.osm/south_finland.osm.pbf', 66 | vaasa: 'https://karttapalvelu.storage.hsldev.com/waltti.osm/vaasa.osm.pbf', 67 | varely: 'https://karttapalvelu.storage.hsldev.com/finland.osm/varely.osm.pbf', 68 | ...extraOSM, 69 | }; 70 | 71 | const dem = { 72 | waltti: 73 | 'https://elevdata.blob.core.windows.net/elevation/waltti/waltti-10m-elevation-model_20190927.tif', 74 | hsl: 'https://elevdata.blob.core.windows.net/elevation/hsl/hsl-10m-elevation-model_20190920.tif', 75 | }; 76 | 77 | const constants = { 78 | BUFFER_SIZE: 1024 * 1024 * 32, 79 | }; 80 | 81 | module.exports = { 82 | router, 83 | gtfsMap, 84 | osm: router.osm.map(id => { 85 | return { id, url: osm[id] }; 86 | }), // array of id, url (OSM data) pairs 87 | dem: router.dem ? [{ id: router.dem, url: dem[router.dem] }] : null, // currently only one DEM file is used 88 | dataToolImage: `hsldevcom/otp-data-tools:${process.env.TOOLS_TAG || 'v3'}`, 89 | dataDir: `${process.cwd()}/data`, 90 | storageDir: `${process.cwd()}/storage`, 91 | constants, 92 | passOBAfilter, 93 | SPLIT_BUILD_TYPE, 94 | }; 95 | -------------------------------------------------------------------------------- /task/OTPTest.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const fse = require('fs-extra'); 3 | const exec = require('child_process').exec; 4 | const through = require('through2'); 5 | const { dataDir, constants } = require('../config'); 6 | const { postSlackMessage, createDir } = require('../util'); 7 | const testTag = process.env.OTP_TAG || 'v2'; 8 | const JAVA_OPTS = process.env.JAVA_OPTS || '-Xmx9g'; 9 | 10 | /** 11 | * Builds an OTP graph with a source data file. If the build is successful we can trust 12 | * the file is good enough to be used. 13 | */ 14 | function testWithOTP(otpFile, quiet = false) { 15 | const lastLog = []; 16 | 17 | return new Promise((resolve, reject) => { 18 | if (!fs.existsSync(otpFile)) { 19 | reject(new Error(`${otpFile} does not exist!\n`)); 20 | } else { 21 | createDir(`${dataDir}/tmp`); 22 | fs.mkdtemp(`${dataDir}/tmp/router-build-test`, (err, folder) => { 23 | if (err) throw err; 24 | process.stdout.write( 25 | 'Testing ' + otpFile + ' in directory ' + folder + '...\n', 26 | ); 27 | const dir = folder.split('/').pop(); 28 | const r = fs.createReadStream(otpFile); 29 | r.on('end', () => { 30 | try { 31 | const build = exec( 32 | `docker run --rm -e JAVA_OPTS="${JAVA_OPTS}" -v ${dataDir}/tmp/${dir}:/var/opentripplanner hsldevcom/opentripplanner:${testTag} --build --save`, 33 | { maxBuffer: constants.BUFFER_SIZE }, 34 | ); 35 | build.on('exit', function (c) { 36 | if (c === 0) { 37 | resolve(true); 38 | process.stdout.write(otpFile + ' Test SUCCESS\n'); 39 | } else { 40 | const log = lastLog.join(''); 41 | postSlackMessage(`${otpFile} test failed: ${log} :boom:`); 42 | global.hasFailures = true; 43 | resolve(false); 44 | } 45 | fse.removeSync(folder); 46 | }); 47 | build.stdout.on('data', function (data) { 48 | lastLog.push(data.toString()); 49 | if (lastLog.length === 20) { 50 | delete lastLog[0]; 51 | } 52 | if (!quiet) { 53 | process.stdout.write(data.toString()); 54 | } 55 | }); 56 | build.stderr.on('data', function (data) { 57 | lastLog.push(data.toString()); 58 | if (lastLog.length > 20) { 59 | lastLog.splice(0, 1); 60 | } 61 | if (!quiet) { 62 | process.stderr.write(data.toString()); 63 | } 64 | }); 65 | } catch (e) { 66 | const log = lastLog.join(''); 67 | postSlackMessage(`${otpFile} test failed: ${log} :boom:`); 68 | fse.removeSync(folder); 69 | reject(e); 70 | } 71 | }); 72 | r.pipe(fs.createWriteStream(`${folder}/${otpFile.split('/').pop()}`)); 73 | }); 74 | } 75 | }); 76 | } 77 | 78 | module.exports = { 79 | testOTPFile: () => { 80 | return through.obj(function (file, encoding, callback) { 81 | const otpFile = file.history[file.history.length - 1]; 82 | if (process.env.SKIP_OTP_TESTS) { 83 | process.stdout.write( 84 | 'OTP test skipped because the SKIP_OTP_TESTS environment variable is set\n', 85 | ); 86 | return callback(null, file); 87 | } 88 | testWithOTP(otpFile, true) 89 | .then(success => { 90 | if (success) { 91 | callback(null, file); 92 | } else { 93 | callback(null, null); 94 | } 95 | }) 96 | .catch(() => { 97 | callback(null, null); 98 | }); 99 | }); 100 | }, 101 | }; 102 | -------------------------------------------------------------------------------- /waltti/config.js: -------------------------------------------------------------------------------- 1 | const { mapSrc } = require('../util'); 2 | 3 | module.exports = { 4 | id: 'waltti', 5 | src: [ 6 | mapSrc( 7 | 'Hameenlinna', 8 | 'https://tvv.fra1.digitaloceanspaces.com/203.zip', 9 | true, 10 | ), 11 | mapSrc( 12 | 'Kotka', 13 | 'https://tvv.fra1.digitaloceanspaces.com/217.zip', 14 | true, 15 | undefined, 16 | { 17 | 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 18 | 'fare_rules.txt': 'digitransit_fare_rules.txt', 19 | }, 20 | ), 21 | mapSrc('Kouvola', 'https://tvv.fra1.digitaloceanspaces.com/219.zip', true), 22 | mapSrc( 23 | 'Lappeenranta', 24 | 'https://tvv.fra1.digitaloceanspaces.com/225.zip', 25 | true, 26 | ), 27 | mapSrc('Mikkeli', 'https://tvv.fra1.digitaloceanspaces.com/227.zip', true), 28 | mapSrc('Vaasa', 'https://tvv.fra1.digitaloceanspaces.com/249.zip', true), 29 | mapSrc( 30 | 'Joensuu', 31 | 'https://tvv.fra1.digitaloceanspaces.com/207.zip', 32 | true, 33 | undefined, 34 | { 35 | 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 36 | 'fare_rules.txt': 'digitransit_fare_rules.txt', 37 | }, 38 | ), 39 | mapSrc('FOLI', 'http://data.foli.fi/gtfs/gtfs.zip'), 40 | mapSrc( 41 | 'Lahti', 42 | 'https://tvv.fra1.digitaloceanspaces.com/223.zip', 43 | true, 44 | undefined, 45 | { 46 | 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 47 | 'fare_rules.txt': 'digitransit_fare_rules.txt', 48 | }, 49 | ), 50 | mapSrc( 51 | 'Kuopio', 52 | 'https://karttapalvelu.kuopio.fi/google_transit/google_transit.zip', 53 | ), 54 | mapSrc( 55 | 'OULU', 56 | 'https://tvv.fra1.digitaloceanspaces.com/229.zip', 57 | true, 58 | undefined, 59 | { 60 | 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 61 | 'fare_rules.txt': 'digitransit_fare_rules.txt', 62 | }, 63 | ), 64 | mapSrc( 65 | 'LINKKI', 66 | 'https://tvv.fra1.digitaloceanspaces.com/209.zip', 67 | true, 68 | undefined, 69 | { 70 | 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 71 | 'fare_rules.txt': 'digitransit_fare_rules.txt', 72 | }, 73 | ), 74 | mapSrc( 75 | 'tampere', 76 | 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_deprecated.zip', 77 | ), 78 | mapSrc( 79 | 'Rovaniemi', 80 | 'https://tvv.fra1.digitaloceanspaces.com/237.zip', 81 | true, 82 | ), 83 | mapSrc( 84 | 'digitraffic', 85 | 'https://rata.digitraffic.fi/api/v1/trains/gtfs-passenger-stops.zip', 86 | false, 87 | undefined, 88 | undefined, 89 | { 90 | headers: { 91 | 'Accept-Encoding': 'gzip', 92 | 'Digitraffic-User': 'Digitransit/OTP-dataloading', 93 | }, 94 | }, 95 | ), 96 | mapSrc( 97 | 'Pori', 98 | 'https://tvv.fra1.digitaloceanspaces.com/231.zip', 99 | true, 100 | undefined, 101 | { 102 | 'fare_attributes.txt': 'digitransit_fare_attributes.txt', 103 | 'fare_rules.txt': 'digitransit_fare_rules.txt', 104 | }, 105 | ), 106 | mapSrc( 107 | 'FUNI', 108 | 'https://foligtfs.blob.core.windows.net/routeplanner/gtfs-foli-ff.zip', 109 | true, 110 | ), 111 | mapSrc( 112 | 'Raasepori', 113 | 'https://tvv.fra1.digitaloceanspaces.com/232.zip', 114 | true, 115 | ), 116 | mapSrc( 117 | 'KotkaLautat', 118 | 'https://mobility.mobility-database.fintraffic.fi/static/ferries_cars.zip', 119 | true, 120 | ['waltti/gtfs-rules/only-kotka-ferries.rule'], 121 | ), 122 | mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), 123 | mapSrc('Kajaani', 'https://tvv.fra1.digitaloceanspaces.com/211.zip', true), 124 | ], 125 | osm: ['kajaani', 'oulu', 'rovaniemi', 'southFinland', 'vaasa'], 126 | }; 127 | -------------------------------------------------------------------------------- /finland/config.js: -------------------------------------------------------------------------------- 1 | const { mapSrc } = require('../util'); 2 | 3 | module.exports = { 4 | id: 'finland', 5 | src: [ 6 | mapSrc( 7 | 'HSL', 8 | 'https://infopalvelut.storage.hsldev.com/gtfs/hsl_google_transit.zip', 9 | false, 10 | ['finland/gtfs-rules/hsl-no-trains.rule'], 11 | { 'trips.txt': 'trips2.txt' }, 12 | ), 13 | mapSrc( 14 | 'MATKA', 15 | 'https://mobility.mobility-database.fintraffic.fi/static/digitransit_new.zip', 16 | true, 17 | ), 18 | mapSrc( 19 | 'CAR_FERRIES', 20 | 'https://mobility.mobility-database.fintraffic.fi/static/ferries_cars.zip', 21 | true, 22 | ), 23 | mapSrc( 24 | 'tampere', 25 | 'https://ekstrat.tampere.fi/ekstrat/ptdata/tamperefeed_deprecated.zip', 26 | ), 27 | mapSrc('LINKKI', 'https://tvv.fra1.digitaloceanspaces.com/209.zip', true), 28 | mapSrc('OULU', 'https://tvv.fra1.digitaloceanspaces.com/229.zip'), 29 | mapSrc( 30 | 'digitraffic', 31 | 'https://rata.digitraffic.fi/api/v1/trains/gtfs-passenger-stops.zip', 32 | false, 33 | undefined, 34 | undefined, 35 | { 36 | headers: { 37 | 'Accept-Encoding': 'gzip', 38 | 'Digitraffic-User': 'Digitransit/OTP-dataloading', 39 | }, 40 | }, 41 | ), 42 | mapSrc( 43 | 'Rauma', 44 | 'http://digitransit-proxy:8080/out/raumaadmin.mattersoft.fi/feeds/233.zip', 45 | ), 46 | mapSrc( 47 | 'Hameenlinna', 48 | 'https://tvv.fra1.digitaloceanspaces.com/203.zip', 49 | true, 50 | ), 51 | mapSrc('Kotka', 'https://tvv.fra1.digitaloceanspaces.com/217.zip', true), 52 | mapSrc('Kouvola', 'https://tvv.fra1.digitaloceanspaces.com/219.zip', true), 53 | mapSrc( 54 | 'Lappeenranta', 55 | 'https://tvv.fra1.digitaloceanspaces.com/225.zip', 56 | true, 57 | ), 58 | mapSrc('Mikkeli', 'https://tvv.fra1.digitaloceanspaces.com/227.zip', true), 59 | mapSrc('Vaasa', 'https://tvv.fra1.digitaloceanspaces.com/249.zip', true), 60 | mapSrc('Joensuu', 'https://tvv.fra1.digitaloceanspaces.com/207.zip', true), 61 | mapSrc('FOLI', 'http://data.foli.fi/gtfs/gtfs.zip'), 62 | mapSrc('Lahti', 'https://tvv.fra1.digitaloceanspaces.com/223.zip', true), 63 | mapSrc( 64 | 'Kuopio', 65 | 'https://karttapalvelu.kuopio.fi/google_transit/google_transit.zip', 66 | ), 67 | mapSrc( 68 | 'Rovaniemi', 69 | 'https://tvv.fra1.digitaloceanspaces.com/237.zip', 70 | true, 71 | ), 72 | mapSrc('Kajaani', 'https://tvv.fra1.digitaloceanspaces.com/211.zip', true), 73 | mapSrc('Salo', 'https://tvv.fra1.digitaloceanspaces.com/239.zip', true), 74 | mapSrc('Pori', 'https://tvv.fra1.digitaloceanspaces.com/231.zip', true), 75 | mapSrc('Viro', 'https://peatus.ee/gtfs/gtfs.zip'), 76 | mapSrc( 77 | 'Raasepori', 78 | 'https://tvv.fra1.digitaloceanspaces.com/232.zip', 79 | true, 80 | ), 81 | mapSrc( 82 | 'VARELY', 83 | 'http://digitransit-proxy:8080/out/varelyadmin.mattersoft.fi/feeds/102.zip', 84 | false, 85 | ), 86 | mapSrc( 87 | 'Harma', 88 | 'https://harmanliikenne.bussikaista.fi/sites/harma/files/gtfs/export/latest.zip', 89 | true, 90 | ), 91 | mapSrc( 92 | 'PohjolanMatka', 93 | 'https://minfoapi.matkahuolto.fi/gtfs/458/gtfs.zip', 94 | true, 95 | ), 96 | mapSrc( 97 | 'Korsisaari', 98 | 'https://minfoapi.matkahuolto.fi/gtfs/036/gtfs.zip', 99 | true, 100 | ), 101 | mapSrc( 102 | 'KoivistonAuto', 103 | 'https://minfoapi.matkahuolto.fi/gtfs/020/gtfs.zip', 104 | true, 105 | ), 106 | mapSrc( 107 | 'PahkakankaanLiikenne', 108 | 'https://minfoapi.matkahuolto.fi/gtfs/198/gtfs.zip', 109 | true, 110 | ), 111 | mapSrc( 112 | 'IngvesSvanback', 113 | 'https://minfoapi.matkahuolto.fi/gtfs/177/gtfs.zip', 114 | true, 115 | ), 116 | mapSrc( 117 | '02Taksi', 118 | 'https://resources.02taksi.fi/digitransit_02_taksi.zip', 119 | false, 120 | ), 121 | ], 122 | osm: ['finland', 'estonia'], 123 | }; 124 | -------------------------------------------------------------------------------- /task/MapFit.js: -------------------------------------------------------------------------------- 1 | const through = require('through2'); 2 | const fs = require('fs'); 3 | const csvParser = require('csv-parser'); 4 | const cloneable = require('cloneable-readable'); 5 | const removeBOM = require('remove-bom-stream'); 6 | const { parseId } = require('../util'); 7 | const { stringify } = require('csv-stringify'); 8 | const { dataDir } = require('../config.js'); 9 | 10 | const limit = 200; // do not fit if distance is more than this many meters 11 | 12 | // The radius of the earth. 13 | const radius = 6371000; 14 | const dr = Math.PI / 180; // degree to radian 15 | 16 | function distance(a, b) { 17 | const lat1 = a[0] * dr; 18 | const lat2 = b[0] * dr; 19 | const sinDLat = Math.sin((b[0] - a[0]) * dr * 0.5); 20 | const sinDLon = Math.sin((b[1] - a[1]) * dr * 0.5); 21 | const r = 22 | sinDLat * sinDLat + Math.cos(lat1) * Math.cos(lat2) * sinDLon * sinDLon; 23 | const c = 2 * Math.atan2(Math.sqrt(r), Math.sqrt(1 - r)); 24 | return radius * c; 25 | } 26 | 27 | function readHeader(fileName) { 28 | return new Promise(resolve => { 29 | const readStream = fs.createReadStream(fileName); 30 | readStream 31 | .pipe(removeBOM('utf-8')) 32 | .pipe(csvParser()) 33 | .on('headers', headers => { 34 | readStream.destroy(); 35 | resolve(headers); 36 | }); 37 | }); 38 | } 39 | 40 | function fitStopCoordinates(map, stats) { 41 | return through.obj(function (stop, enc, next) { 42 | const id = stop.stop_id; 43 | // koontikanta uses x_ like gtfs ids, strip the prefix 44 | const noprefix = id.slice(id.indexOf('_') + 1); 45 | for (const k of [id, stop.stop_code, noprefix]) { 46 | if (k === undefined) { 47 | continue; 48 | } 49 | const osmPos = map[k]; 50 | if (osmPos && stop.stop_lat && stop.stop_lon) { 51 | const dist = distance(osmPos, [stop.stop_lat, stop.stop_lon]); 52 | if (dist > stats.maxDist) stats.maxDist = dist; 53 | if (dist < limit) { 54 | stats.fitted++; 55 | stats.dsum += dist; 56 | stop.stop_lat = osmPos[0]; 57 | stop.stop_lon = osmPos[1]; 58 | break; 59 | } else { 60 | stats.bad++; 61 | } 62 | } 63 | } 64 | next(null, stop); 65 | }); 66 | } 67 | 68 | function transformStops(folder, map, cb) { 69 | const fileName = `${folder}/stops.txt`; 70 | const result = `${folder}/transformed_stops.txt`; 71 | const stats = { 72 | bad: 0, 73 | fitted: 0, 74 | dsum: 0, 75 | maxDist: 0, 76 | }; 77 | 78 | readHeader(fileName).then(headers => { 79 | const stringifier = stringify({ header: true, columns: headers }); 80 | 81 | fs.createReadStream(fileName) 82 | .pipe(removeBOM('utf-8')) 83 | .pipe(csvParser()) 84 | .pipe(fitStopCoordinates(map, stats)) 85 | .pipe(stringifier) 86 | .pipe(fs.createWriteStream(result)) 87 | .on('finish', () => { 88 | fs.copyFileSync(result, fileName); 89 | process.stdout.write( 90 | `Fitted ${stats.fitted} stops, skipped ${stats.bad} bad fits\n`, 91 | ); 92 | if (stats.fitted) { 93 | process.stdout.write( 94 | `Average fit distance ${stats.dsum / stats.fitted}, max distance ${stats.maxDist}\n`, 95 | ); 96 | } 97 | cb(); 98 | }); 99 | }); 100 | } 101 | 102 | module.exports = function mapFit(config) { 103 | return through.obj((file, encoding, callback) => { 104 | const gtfsFile = file.history[file.history.length - 1]; 105 | const id = parseId(gtfsFile); 106 | const folder = `${dataDir}/tmp/${id}`; 107 | const source = config.gtfsMap[id]; 108 | 109 | if (!source.fit) { 110 | process.stdout.write(gtfsFile + ' fit disabled\n'); 111 | callback(null, file); 112 | return; 113 | } 114 | if (!config.fitMap) { 115 | process.stdout.write( 116 | `PrepareFit task not run before fitting, skipping ${gtfsFile} map fit\n`, 117 | ); 118 | callback(null, file); 119 | return; 120 | } 121 | 122 | process.stdout.write(`Fitting ${gtfsFile} to OSM stop locations ...\n`); 123 | transformStops(folder, config.fitMap, () => { 124 | process.stdout.write(gtfsFile + ' fit SUCCESS\n'); 125 | file.contents = cloneable(fs.createReadStream(gtfsFile)); 126 | callback(null, file); 127 | }); 128 | }); 129 | }; 130 | -------------------------------------------------------------------------------- /task/BuildOTPGraph.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const { exec, execSync } = require('child_process'); 3 | const del = require('del'); 4 | const { otpMatching, postSlackMessage } = require('../util'); 5 | const { zipWithGlobIntoDir } = require('./ZipTask'); 6 | const { dataDir, constants, SPLIT_BUILD_TYPE } = require('../config.js'); 7 | const graphBuildTag = process.env.OTP_TAG || 'v2'; 8 | const JAVA_OPTS = process.env.JAVA_OPTS || '-Xmx12g'; 9 | const dockerImage = `hsldevcom/opentripplanner:${graphBuildTag}`; 10 | 11 | const buildGraph = function (router) { 12 | const lastLog = []; 13 | const collectLog = data => { 14 | lastLog.push(data.toString()); 15 | if (lastLog.length > 20) { 16 | lastLog.splice(0, 1); 17 | } 18 | }; 19 | return new Promise((resolve, reject) => { 20 | const version = execSync( 21 | `docker pull ${dockerImage};docker run --rm ${dockerImage} --version`, 22 | ); 23 | const commit = version.toString().match(/commit: ([0-9a-f]+)/)[1]; 24 | 25 | let command; 26 | switch (SPLIT_BUILD_TYPE) { 27 | case 'ONLY_BUILD_STREET_GRAPH': 28 | command = `docker run -e JAVA_OPTS="${JAVA_OPTS}" -v ${dataDir}/build/${router.id}:/var/opentripplanner --mount type=bind,source=${dataDir}/../logback-include-extensions.xml,target=/logback-include-extensions.xml ${dockerImage} --buildStreet --save`; 29 | break; 30 | case 'USE_PREBUILT_STREET_GRAPH': 31 | command = `docker run -e JAVA_OPTS="${JAVA_OPTS}" -v ${dataDir}/build/${router.id}:/var/opentripplanner --mount type=bind,source=${dataDir}/../logback-include-extensions.xml,target=/logback-include-extensions.xml ${dockerImage} --loadStreet --save`; 32 | break; 33 | default: 34 | command = `docker run -e JAVA_OPTS="${JAVA_OPTS}" -v ${dataDir}/build/${router.id}:/var/opentripplanner --mount type=bind,source=${dataDir}/../logback-include-extensions.xml,target=/logback-include-extensions.xml ${dockerImage} --build --save`; 35 | break; 36 | } 37 | 38 | const buildGraph = exec(command, { maxBuffer: constants.BUFFER_SIZE }); 39 | const buildLog = fs.openSync( 40 | `${dataDir}/build/${router.id}/build.log`, 41 | 'w+', 42 | ); 43 | 44 | buildGraph.stdout.on('data', function (data) { 45 | collectLog(data); 46 | process.stdout.write(data.toString()); 47 | fs.writeSync(buildLog, data); 48 | }); 49 | 50 | buildGraph.stderr.on('data', function (data) { 51 | collectLog(data); 52 | process.stdout.write(data.toString()); 53 | fs.writeSync(buildLog, data); 54 | }); 55 | 56 | buildGraph.on('exit', status => { 57 | fs.closeSync(buildLog); 58 | if (status === 0) { 59 | resolve({ commit, router }); 60 | } else { 61 | const log = lastLog.join(''); 62 | postSlackMessage(`${router.id} build failed: ${status}:${log} :boom:`); 63 | reject('could not build'); 64 | } 65 | }); 66 | }); 67 | }; 68 | 69 | const packData = function (commit, router) { 70 | const path = `${dataDir}/build/${router.id}`; 71 | 72 | const p1 = new Promise((resolve, reject) => { 73 | process.stdout.write('Creating zip file for router data\n'); 74 | const osmFiles = router.osm.map(osm => `${path}/${osm}.pbf`); 75 | 76 | // create a zip file which includes all data required 77 | // for graph build and routing: gtfs, osm, dem + otp configs 78 | if ( 79 | zipWithGlobIntoDir( 80 | `${path}/router-${router.id}.zip`, 81 | [`${path}/*gtfs.zip`, `${path}/*.json`, ...osmFiles, `${path}/*.tif`], 82 | `router-${router.id}`, 83 | ) 84 | ) { 85 | resolve(); 86 | } else { 87 | reject('Zip creation failed'); 88 | } 89 | }); 90 | const p2 = new Promise((resolve, reject) => { 91 | process.stdout.write('Creating zip file for otp graph\n'); 92 | // create a zip file for routing only 93 | // include graph.obj, router-config.json and otp-config.json 94 | if ( 95 | zipWithGlobIntoDir( 96 | `${path}/graph-${router.id}-${commit}.zip`, 97 | [ 98 | `${path}/graph.obj`, 99 | `${path}/router-config.json`, 100 | `${path}/otp-config.json`, 101 | ], 102 | router.id, 103 | ) 104 | ) { 105 | resolve(); 106 | } else { 107 | reject('Zip creation failed'); 108 | } 109 | }); 110 | const p3 = new Promise((resolve, reject) => { 111 | fs.writeFile( 112 | `${path}/version.txt`, 113 | new Date().toISOString(), 114 | function (err) { 115 | if (err) { 116 | reject(err); 117 | } else { 118 | resolve(); 119 | } 120 | }, 121 | ); 122 | }); 123 | return Promise.all([p1, p2, p3]); 124 | }; 125 | 126 | module.exports = { 127 | buildOTPGraphTask: router => 128 | buildGraph(router) 129 | .then(resp => packData(resp.commit, resp.router)) 130 | .then(() => otpMatching(`${dataDir}/build/${router.id}`)) 131 | .then(() => del(`${dataDir}/build/${router.id}/taggedStops.log`)) 132 | .then(() => process.stdout.write('Graph build SUCCESS\n')), 133 | buildOTPStreetOnlyGraphTask: router => 134 | buildGraph(router).then(() => 135 | process.stdout.write('Street only graph build SUCCESS\n'), 136 | ), 137 | }; 138 | -------------------------------------------------------------------------------- /task/OSMPreprocessing.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const fse = require('fs-extra'); 3 | const exec = require('child_process').exec; 4 | const through = require('through2'); 5 | const { dataDir, constants, dataToolImage } = require('../config'); 6 | const { postSlackMessage, createDir } = require('../util'); 7 | 8 | /** 9 | * Runs the instructions listed in the OSM preprocessing bash script. 10 | */ 11 | function preprocessWithFile( 12 | osmFile, 13 | quiet = false, 14 | osmPreprocessingDir, 15 | osmId, 16 | osmFileName, 17 | ) { 18 | const lastLog = []; 19 | 20 | return new Promise((resolve, reject) => { 21 | const preprocessingInstructionsFile = `${osmPreprocessingDir}/${osmId}.sh`; 22 | 23 | if (!fs.existsSync(osmFile)) { 24 | reject(new Error(`${osmFile} does not exist!\n`)); 25 | } else if (!fs.existsSync(preprocessingInstructionsFile)) { 26 | reject( 27 | new Error( 28 | `No OSM preprocessing instructions for ${osmId}. ${preprocessingInstructionsFile} does not exist!\n`, 29 | ), 30 | ); 31 | } else { 32 | createDir(`${dataDir}/tmp`); 33 | fs.mkdtemp(`${dataDir}/tmp/osm-preprocessing`, (err, folder) => { 34 | if (err) throw err; 35 | process.stdout.write( 36 | 'Running OSM preprocessing instructions from ' + 37 | preprocessingInstructionsFile + 38 | ' for ' + 39 | osmFile + 40 | ' in directory ' + 41 | folder + 42 | '...\n', 43 | ); 44 | const r = fs.createReadStream(osmFile); 45 | r.on('end', async () => { 46 | try { 47 | process.stdout.write( 48 | 'Running commands from file: ' + 49 | preprocessingInstructionsFile + 50 | '\n', 51 | ); 52 | const preprocessingCommand = exec( 53 | `docker run -v ${folder}:/tmp/osm-preprocessing:rw -v ${preprocessingInstructionsFile}:/tmp/preprocessing.sh:ro -w /tmp/osm-preprocessing --rm --entrypoint /bin/bash ${dataToolImage} /tmp/preprocessing.sh`, 54 | { maxBuffer: constants.BUFFER_SIZE }, 55 | ); 56 | preprocessingCommand.on('exit', function (c) { 57 | if (c === 0) { 58 | resolve(fs.readFileSync(`${folder}/${osmFileName}`)); 59 | process.stdout.write( 60 | osmFile + 61 | ' + ' + 62 | preprocessingInstructionsFile + 63 | ' OSM preprocessing SUCCESS\n', 64 | ); 65 | } else { 66 | const log = lastLog.join(''); 67 | postSlackMessage( 68 | `${osmFile} + ${preprocessingInstructionsFile} OSM preprocessing failed: ${log} :boom:`, 69 | ); 70 | global.hasFailures = true; 71 | resolve(null); 72 | } 73 | fse.removeSync(folder); 74 | }); 75 | preprocessingCommand.stdout.on('data', function (data) { 76 | lastLog.push(data.toString()); 77 | if (lastLog.length === 20) { 78 | delete lastLog[0]; 79 | } 80 | if (!quiet) { 81 | process.stdout.write(data.toString()); 82 | } 83 | }); 84 | preprocessingCommand.stderr.on('data', function (data) { 85 | lastLog.push(data.toString()); 86 | if (lastLog.length > 20) { 87 | lastLog.splice(0, 1); 88 | } 89 | if (!quiet) { 90 | process.stderr.write(data.toString()); 91 | } 92 | }); 93 | } catch (e) { 94 | const log = lastLog.join(''); 95 | postSlackMessage( 96 | `${osmFile} + ${preprocessingInstructionsFile} OSM preprocessing failed: ${log} :boom: ${e}`, 97 | ); 98 | fse.removeSync(folder); 99 | reject(e); 100 | } 101 | }); 102 | r.pipe(fs.createWriteStream(`${folder}/${osmFileName}`)); 103 | }); 104 | } 105 | }); 106 | } 107 | 108 | module.exports = { 109 | runOSMPreprocessing: osmPreprocessingDir => { 110 | return through.obj(function (file, encoding, callback) { 111 | const osmFile = file.history[file.history.length - 1]; 112 | if (process.env.SKIP_OSM_PREPROCESSING) { 113 | process.stdout.write( 114 | 'OSM preprocessing skipped because the SKIP_OSM_PREPROCESSING environment variable is set\n', 115 | ); 116 | return callback(null, file); 117 | } 118 | const osmFileName = osmFile.split('/').pop(); 119 | // This can be, for example, hsl, finland, or southFinland. 120 | const osmId = osmFileName.split('.')[0]; 121 | preprocessWithFile(osmFile, true, osmPreprocessingDir, osmId, osmFileName) 122 | .then(outputContents => { 123 | if (outputContents) { 124 | file.contents = outputContents; 125 | callback(null, file); 126 | } else { 127 | callback(null, file); 128 | } 129 | }) 130 | .catch(err => { 131 | process.stdout.write(err.message); 132 | callback(null, file); 133 | }); 134 | }); 135 | }, 136 | }; 137 | -------------------------------------------------------------------------------- /task/PrepareFit.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const through = require('through2'); 3 | const parseOSM = require('osm-pbf-parser'); 4 | 5 | const map = {}; 6 | const nodePositions = {}; 7 | const referredNodes = {}; 8 | 9 | // some stats 10 | let count = 0; 11 | let wcount = 0; 12 | 13 | function isBoardingLocation(tags) { 14 | // same as OTP's boarding location concept 15 | return ( 16 | tags.highway === 'bus_stop' || 17 | tags.railway === 'tram_stop' || 18 | tags.railway === 'station' || 19 | tags.railway === 'halt' || 20 | tags.amenity === 'bus_station' || 21 | tags.amenity === 'ferry_terminal' || 22 | ((tags.public_transport === 'platform' || tags.railway === 'platform') && 23 | tags.usage !== 'tourism') 24 | ); 25 | } 26 | 27 | // first pass: find out which nodes are needed when computing way centers 28 | function collectRefs(config, cb) { 29 | function refsFromOSM(i) { 30 | const osm = parseOSM(); 31 | return fs 32 | .createReadStream(`${config.dataDir}/ready/osm/${config.osm[i].id}.pbf`) 33 | .pipe(osm) 34 | .pipe( 35 | through 36 | .obj(function (items, enc, next) { 37 | items.forEach(function (item) { 38 | const tags = item.tags; 39 | if ( 40 | item.type === 'way' && 41 | isBoardingLocation(tags) && 42 | (tags.ref || tags['ref:findr'] || tags['ref:findt']) 43 | ) { 44 | item.refs.forEach(n => { 45 | referredNodes[n] = true; 46 | }); 47 | } 48 | }); 49 | next(); 50 | }) 51 | .on('end', () => { 52 | if (i === config.osm.length - 1) { 53 | createMap(config, cb); 54 | } else { 55 | refsFromOSM(i + 1); // recurse to next OSM entry 56 | } 57 | }), 58 | ) 59 | .resume(); 60 | } 61 | return refsFromOSM(0); 62 | } 63 | 64 | // second pass: collect a map of OSM stop coordinates 65 | function createMap(config, cb) { 66 | function mapFromOsm(i) { 67 | const osm = parseOSM(); 68 | fs.createReadStream(`${config.dataDir}/ready/osm/${config.osm[i].id}.pbf`) 69 | .pipe(osm) 70 | .pipe( 71 | through.obj(function (items, enc, next) { 72 | items.forEach(function (item) { 73 | if (item.type === 'node' && referredNodes[item.id]) { 74 | // this node is needed later 75 | nodePositions[item.id] = [item.lat, item.lon]; 76 | } 77 | const tags = item.tags; 78 | if (isBoardingLocation(tags)) { 79 | const ref = tags.ref || tags['ref:findr'] || tags['ref:findt']; 80 | if (ref) { 81 | let pos; 82 | if (item.type === 'node') { 83 | pos = [item.lat, item.lon]; 84 | } else if (item.type === 'way' && item.refs.length > 1) { 85 | wcount++; 86 | pos = [0, 0]; 87 | // do not sum the start and end point of a closed loop twice 88 | const last = 89 | item.refs[0] === item.refs[item.refs.length - 1] 90 | ? item.refs.length - 1 91 | : item.refs.length; 92 | let i; 93 | for (i = 0; i < last; i++) { 94 | const n = item.refs[0]; 95 | if (nodePositions[n]) { 96 | pos[0] += nodePositions[n][0]; 97 | pos[1] += nodePositions[n][1]; 98 | } else { 99 | break; // bad data 100 | } 101 | } 102 | if (i === last) { 103 | pos[0] /= i; 104 | pos[1] /= i; 105 | } else { 106 | pos = null; 107 | } 108 | } 109 | if (pos) { 110 | if (tags.ref && !map[tags.ref]) { 111 | map[tags.ref] = pos; 112 | } 113 | if (tags['ref:findr'] && !map[tags['ref:findr']]) { 114 | map[tags['ref:findr']] = pos; 115 | } 116 | if (tags['ref:findt'] && !map[tags['ref:findt']]) { 117 | map[tags['ref:findt']] = pos; 118 | } 119 | count++; 120 | } 121 | } 122 | } 123 | }); 124 | next(); 125 | }), 126 | ) 127 | .on('end', () => { 128 | if (i < config.osm.length - 1) { 129 | mapFromOsm(i + 1); // recurse to next OSM entry 130 | } else { 131 | console.log( 132 | 'Number of ref mapped stops is ' + 133 | count + 134 | ', of which OSM ways ' + 135 | wcount, 136 | ); 137 | config.fitMap = map; 138 | cb(); 139 | } 140 | }) 141 | .resume(); 142 | } 143 | 144 | mapFromOsm(0); 145 | } 146 | 147 | module.exports = function (config) { 148 | return new Promise((resolve, reject) => { 149 | if (!config.router.src.some(src => src.fit)) { 150 | resolve(); 151 | } 152 | try { 153 | collectRefs(config, resolve); 154 | } catch (err) { 155 | reject(err); 156 | } 157 | }); 158 | }; 159 | -------------------------------------------------------------------------------- /util.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const readline = require('readline'); 3 | const path = require('path'); 4 | const axios = require('axios'); 5 | 6 | const username = `OTP data builder ${process.env.BUILDER_TYPE || 'dev'}`; 7 | const channel = process.env.SLACK_CHANNEL_ID; 8 | const headers = { 9 | Authorization: `Bearer ${process.env.SLACK_ACCESS_TOKEN}`, 10 | 'Content-Type': 'application/json', 11 | Accept: '*/*', 12 | }; 13 | 14 | async function postSlackMessage(text) { 15 | process.stdout.write(`${text}\n`); // write important messages also to log 16 | try { 17 | const { data } = await axios.post( 18 | 'https://slack.com/api/chat.postMessage', 19 | { 20 | channel, 21 | text, 22 | username, 23 | thread_ts: global.messageTimeStamp, // either null (will be a new message) or pointing to parent message (will be a reply) 24 | }, 25 | { headers }, 26 | ); 27 | // Return the response, it contains information such as the message timestamp that is needed to reply to messages 28 | return data; 29 | } catch (e) { 30 | // Something went wrong in the Slack-cycle... log it and continue build 31 | process.stdout.write( 32 | `Something went wrong when trying to send message to Slack: ${e}\n`, 33 | ); 34 | return e; 35 | } 36 | } 37 | 38 | async function updateSlackMessage(text) { 39 | process.stdout.write(`${text}\n`); 40 | try { 41 | const { data } = await axios.post( 42 | 'https://slack.com/api/chat.update', 43 | { 44 | channel: process.env.SLACK_CHANNEL_ID, 45 | text, 46 | username, 47 | ts: global.messageTimeStamp, 48 | }, 49 | { headers }, 50 | ); 51 | // Return response data, it contains information such as the message timestamp that is needed to reply to messages 52 | return data; 53 | } catch (e) { 54 | // Something went wrong in the Slack-cycle... log it and continue build 55 | process.stdout.write( 56 | `Something went wrong when trying to update Slack message: ${e}\n`, 57 | ); 58 | return e; 59 | } 60 | } 61 | 62 | const UNCONNECTED = 63 | /Could not connect ([A-Z]?[a-z]?\d{4}) at \((\d+\.\d+), (\d+\.\d+)/; 64 | const CONNECTED = 65 | /Connected {.*:(\d*) lat,lng=(\d+\.\d+),(\d+\.\d+)} \(([A-Z]?[a-z]?\d{4})\) to (.*) at \((\d+\.\d+), (\d+\.\d+)/; 66 | 67 | function distance(lat1, lon1, lat2, lon2) { 68 | const p = Math.PI / 180; 69 | const a = 70 | 0.5 - 71 | Math.cos((lat2 - lat1) * p) / 2 + 72 | (Math.cos(lat1 * p) * 73 | Math.cos(lat2 * p) * 74 | (1 - Math.cos((lon2 - lon1) * p))) / 75 | 2; 76 | 77 | return 12742 * 1000 * Math.asin(Math.sqrt(a)); // 2 * R; R = 6371 km 78 | } 79 | 80 | async function match(line, connectedStream, unconnectedStream) { 81 | let res = UNCONNECTED.exec(line); 82 | if (res != null) { 83 | const [stopcode, jorelon, jorelat] = res.slice(1); 84 | unconnectedStream.write([stopcode, jorelat, jorelon].join(',') + '\n'); 85 | return; 86 | } 87 | res = CONNECTED.exec(line); 88 | if (res != null) { 89 | const [stopid, jorelat, jorelon, stopcode, osmnode, osmlon, osmlat] = 90 | res.slice(1); 91 | const dist = distance(jorelat, jorelon, osmlat, osmlon); 92 | connectedStream.write( 93 | [stopid, stopcode, jorelat, jorelon, osmnode, osmlat, osmlon, dist].join( 94 | ',', 95 | ) + '\n', 96 | ); 97 | } 98 | } 99 | 100 | // process taggedStops.log file into connected.csv and unconnected.csv in given dir path 101 | const otpMatching = function (directory) { 102 | return new Promise(resolve => { 103 | const promises = []; 104 | 105 | const connectedStream = fs.createWriteStream( 106 | path.join(directory, 'connected.csv'), 107 | ); 108 | const unconnectedStream = fs.createWriteStream( 109 | path.join(directory, 'unconnected.csv'), 110 | ); 111 | connectedStream.write( 112 | 'stop_id,stop_code,jore_lat,jore_lon,osm_node,osm_lat,osm_lon,distance\n', 113 | ); 114 | unconnectedStream.write('stop_code,jore_lat,jore_lon\n'); 115 | 116 | const rl = readline.createInterface({ 117 | input: fs.createReadStream(path.join(directory, 'taggedStops.log')), 118 | }); 119 | 120 | rl.on('line', line => { 121 | promises.push(match(line, connectedStream, unconnectedStream)); 122 | }); 123 | 124 | rl.on('close', () => { 125 | Promise.all(promises).then(resolve); 126 | }); 127 | }); 128 | }; 129 | 130 | // extract feed id from zip file name: 'path/HSL-gtfs.zip' -> HSL 131 | const parseId = function (gtfsFile) { 132 | const fileName = gtfsFile.split('/').pop(); 133 | return fileName.substring(0, fileName.indexOf('-gtfs')); 134 | }; 135 | 136 | /* 137 | * Directory names follow ISO 8601 format without milliseconds and 138 | * with ':' replaced with '.'. Returns null if date can't be parsed. 139 | */ 140 | function dirNameToDate(dirName) { 141 | const date = new Date(dirName.replace(/\./g, ':')); 142 | return date instanceof Date && !isNaN(date) ? date : null; 143 | } 144 | 145 | /** 146 | * @param {string} dirPath dir to create including its path 147 | */ 148 | function createDir(dirPath) { 149 | if (!fs.existsSync(dirPath)) { 150 | fs.mkdirSync(dirPath, { recursive: true }); 151 | } 152 | } 153 | 154 | /* 155 | * id = feedid (String) 156 | * url = feed url (String) 157 | * fit = mapfit shapes (true/falsy) 158 | * rules = OBA Filter rules to apply (array of strings or undefined) 159 | * replacements = replace or remove file from gtfs package (format: {'file_to_replace': 'file_to_replace_with' or null}) 160 | * request options = optional special options for request 161 | */ 162 | const mapSrc = (id, url, fit, rules, replacements, request) => ({ 163 | id, 164 | url, 165 | fit, 166 | rules, 167 | replacements, 168 | request, 169 | }); 170 | 171 | module.exports = { 172 | postSlackMessage, 173 | updateSlackMessage, 174 | otpMatching, 175 | parseId, 176 | dirNameToDate, 177 | mapSrc, 178 | createDir, 179 | }; 180 | -------------------------------------------------------------------------------- /varely/router-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "routingDefaults": { 3 | "transferSlack": "1m30s", 4 | "waitReluctance": 0.95, 5 | "elevatorBoardTime": 60, 6 | "bicycle": { 7 | "boardCost": 120, 8 | "reluctance": 1.7, 9 | "optimization": "safest-streets" 10 | }, 11 | "car": { 12 | "reluctance": 10.0 13 | }, 14 | "walk": { 15 | "speed": 1.3, 16 | "reluctance": 1.75, 17 | "stairsReluctance": 1.2, 18 | "stairsTimeFactor": 2, 19 | "escalator": { 20 | "speed": 0.65 21 | }, 22 | "boardCost": 120 23 | }, 24 | "accessEgress": { 25 | "maxDuration": "1h" 26 | }, 27 | "maxDirectStreetDuration": "2h", 28 | "maxDirectStreetDurationForMode": { 29 | "walk": "90m" 30 | }, 31 | "maxJourneyDuration": "12h", 32 | "streetRoutingTimeout": "8s", 33 | "wheelchairAccessibility": { 34 | "stop": { 35 | "onlyConsiderAccessible": false, 36 | "unknownCost": 0, 37 | "inaccessibleCost": 100000 38 | }, 39 | "maxSlope": 0.125 40 | }, 41 | "itineraryFilters": { 42 | "transitGeneralizedCostLimit": { 43 | "costLimitFunction": "600 + 1.5x" 44 | }, 45 | "nonTransitGeneralizedCostLimit": "400 + 1.5x" 46 | } 47 | }, 48 | "gtfsApi": { 49 | "tracingTags": ["digitransit-subscription-id"] 50 | }, 51 | "transit": { 52 | "pagingSearchWindowAdjustments": ["8h", "4h", "4h", "4h", "4h"], 53 | "dynamicSearchWindow": { 54 | "minWindow": "2h" 55 | }, 56 | "maxNumberOfTransfers": 12, 57 | "transferCacheRequests": [ 58 | { 59 | "modes": "WALK", 60 | "walk": { 61 | "speed": 1.2, 62 | "reluctance": 1.8 63 | } 64 | }, 65 | { 66 | "modes": "WALK", 67 | "walk": { 68 | "speed": 1.2, 69 | "reluctance": 1.8 70 | }, 71 | "wheelchairAccessibility": { 72 | "enabled": true 73 | } 74 | }, 75 | { 76 | "modes": "WALK", 77 | "walk": { 78 | "speed": 1.67, 79 | "reluctance": 1.8 80 | } 81 | }, 82 | { 83 | "modes": "BICYCLE", 84 | "walk": { 85 | "speed": 1.2, 86 | "reluctance": 1.8 87 | }, 88 | "bicycle": { 89 | "speed": 5.55, 90 | "rental": { 91 | "useAvailabilityInformation": true 92 | } 93 | } 94 | }, 95 | { 96 | "modes": "BICYCLE", 97 | "walk": { 98 | "speed": 1.67, 99 | "reluctance": 1.8 100 | }, 101 | "bicycle": { 102 | "speed": 5.55, 103 | "rental": { 104 | "useAvailabilityInformation": true 105 | } 106 | } 107 | } 108 | ] 109 | }, 110 | "vectorTiles": { 111 | "attribution": "Digitransit data is licensed under CC BY 4.0.", 112 | "layers": [ 113 | { 114 | "name": "stops", 115 | "type": "Stop", 116 | "mapper": "Digitransit", 117 | "maxZoom": 20, 118 | "minZoom": 5, 119 | "cacheMaxSeconds": 43200 120 | }, 121 | { 122 | "name": "realtimeStops", 123 | "type": "Stop", 124 | "mapper": "DigitransitRealtime", 125 | "maxZoom": 20, 126 | "minZoom": 5, 127 | "cacheMaxSeconds": 60 128 | }, 129 | { 130 | "name": "stations", 131 | "type": "Station", 132 | "mapper": "Digitransit", 133 | "maxZoom": 20, 134 | "minZoom": 5, 135 | "cacheMaxSeconds": 43200 136 | } 137 | ] 138 | }, 139 | "updaters": [ 140 | { 141 | "id": "varely-trip-updates", 142 | "type": "stop-time-updater", 143 | "frequency": "60s", 144 | "url": "http://digitransit-proxy:8080/out/varely.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 145 | "feedId": "VARELY", 146 | "fuzzyTripMatching": false, 147 | "backwardsDelayPropagationType": "ALWAYS" 148 | }, 149 | { 150 | "id": "varely-alerts", 151 | "type": "real-time-alerts", 152 | "frequency": "30s", 153 | "url": "http://digitransit-proxy:8080/out/varely.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 154 | "feedId": "VARELY", 155 | "fuzzyTripMatching": false 156 | }, 157 | { 158 | "id": "rauma-trip-updates", 159 | "type": "stop-time-updater", 160 | "frequency": "60s", 161 | "url": "http://digitransit-proxy:8080/out/rauma.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 162 | "feedId": "Rauma", 163 | "fuzzyTripMatching": false, 164 | "backwardsDelayPropagationType": "ALWAYS" 165 | }, 166 | { 167 | "id": "rauma-alerts", 168 | "type": "real-time-alerts", 169 | "frequency": "30s", 170 | "url": "http://digitransit-proxy:8080/out/rauma.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 171 | "feedId": "Rauma", 172 | "fuzzyTripMatching": false 173 | }, 174 | { 175 | "id": "foli-trip-updates", 176 | "type": "stop-time-updater", 177 | "frequency": "60s", 178 | "url": "http://siri2gtfsrt:8080/FOLI", 179 | "feedId": "FOLI", 180 | "fuzzyTripMatching": true, 181 | "backwardsDelayPropagationType": "ALWAYS" 182 | }, 183 | { 184 | "id": "foli-alerts", 185 | "type": "real-time-alerts", 186 | "frequency": "30s", 187 | "url": "http://digitransit-proxy:8080/out/data.foli.fi/gtfs-rt/reittiopas", 188 | "feedId": "FOLI", 189 | "fuzzyTripMatching": false 190 | }, 191 | { 192 | "id": "pori-trip-updates", 193 | "type": "stop-time-updater", 194 | "frequency": "60s", 195 | "url": "http://digitransit-proxy:8080/out/pjl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 196 | "feedId": "Pori", 197 | "fuzzyTripMatching": false, 198 | "backwardsDelayPropagationType": "ALWAYS" 199 | }, 200 | { 201 | "id": "pori-alerts", 202 | "type": "real-time-alerts", 203 | "frequency": "30s", 204 | "url": "http://digitransit-proxy:8080/out/pjl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 205 | "feedId": "Pori", 206 | "fuzzyTripMatching": false 207 | }, 208 | { 209 | "id": "salo-trip-updates", 210 | "type": "stop-time-updater", 211 | "frequency": "60s", 212 | "url": "http://digitransit-proxy:8080/out/paikku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 213 | "feedId": "Salo", 214 | "fuzzyTripMatching": false, 215 | "backwardsDelayPropagationType": "ALWAYS" 216 | }, 217 | { 218 | "id": "salo-alerts", 219 | "type": "real-time-alerts", 220 | "frequency": "30s", 221 | "url": "http://digitransit-proxy:8080/out/paikku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 222 | "feedId": "Salo", 223 | "fuzzyTripMatching": false 224 | } 225 | ] 226 | } 227 | -------------------------------------------------------------------------------- /task/PrepareRouterData.js: -------------------------------------------------------------------------------- 1 | const through = require('through2'); 2 | const Vinyl = require('vinyl'); 3 | const fs = require('fs'); 4 | const cloneable = require('cloneable-readable'); 5 | const { dataDir, storageDir } = require('../config'); 6 | const { dirNameToDate } = require('../util'); 7 | const assert = require('assert'); 8 | 9 | function createFile(config, fileName, sourcePath) { 10 | process.stdout.write(`copying ${fileName}...\n`); 11 | return new Vinyl({ 12 | path: fileName, 13 | contents: cloneable(fs.createReadStream(`${sourcePath}/${fileName}`)), 14 | }); 15 | } 16 | 17 | // EXTRA_UPDATERS format should be {"turku-alerts": {"type": "real-time-alerts", "frequencySec": 30, "url": "https://foli-beta.nanona.fi/gtfs-rt/reittiopas", "feedId": "FOLI", "fuzzyTripMatching": true}} 18 | // but you can only define, for example, new url and the other key value pairs will remain the same as they are defined in this file. 19 | // It is also possible to add completely new src by defining object with unused id or to remove a src by defining "remove": true 20 | const extraUpdaters = 21 | process.env.EXTRA_UPDATERS !== undefined 22 | ? JSON.parse(process.env.EXTRA_UPDATERS) 23 | : {}; 24 | 25 | // Prepares router-config.json data for opentripplanner and applies edits/additions made in EXTRA_UPDATERS env var 26 | function createAndProcessRouterConfig(router) { 27 | process.stdout.write('copying router-config.json...\n'); 28 | const configName = `${router.id}/router-config.json`; 29 | const routerConfig = JSON.parse(fs.readFileSync(configName, 'utf8')); 30 | const updaters = routerConfig.updaters; 31 | const usedPatches = []; 32 | for (let i = updaters.length - 1; i >= 0; i--) { 33 | const updaterId = updaters[i].id; 34 | const updaterPatch = extraUpdaters[updaterId]; 35 | if (updaterPatch !== undefined) { 36 | if (updaterPatch.remove === true) { 37 | updaters.splice(i, 1); 38 | } else { 39 | const mergedUpdaters = { ...updaters[i], ...updaterPatch }; 40 | delete mergedUpdaters.remove; 41 | updaters[i] = mergedUpdaters; 42 | } 43 | usedPatches.push(updaterId); 44 | } 45 | } 46 | Object.keys(extraUpdaters).forEach(id => { 47 | if (!usedPatches.includes(id)) { 48 | const patchClone = Object.assign({}, extraUpdaters[id]); 49 | delete patchClone.remove; 50 | updaters.push({ ...patchClone, id }); 51 | } 52 | }); 53 | const file = new Vinyl({ 54 | path: 'router-config.json', 55 | contents: Buffer.from(JSON.stringify(routerConfig, null, 2)), 56 | }); 57 | return file; 58 | } 59 | 60 | /** 61 | * Make router data ready for inclusion in opentripplanner. 62 | * In the whole build case, all osm, dem, and gtfs data is fetched from the data directory. 63 | */ 64 | function prepareRouterData(router) { 65 | const stream = through.obj(); 66 | 67 | process.stdout.write( 68 | 'Collecting data and configuration files for graph build\n', 69 | ); 70 | 71 | stream.push(createFile(router, 'build-config.json', router.id)); 72 | stream.push(createFile(router, 'otp-config.json', router.id)); 73 | stream.push(createAndProcessRouterConfig(router)); 74 | router.osm.forEach(osmId => { 75 | const name = osmId + '.pbf'; 76 | stream.push(createFile(router, name, `${dataDir}/ready/osm`)); 77 | }); 78 | if (router.dem) { 79 | const name = router.dem + '.tif'; 80 | stream.push(createFile(router, name, `${dataDir}/ready/dem`)); 81 | } 82 | router.src.forEach(src => { 83 | const name = src.id + '-gtfs.zip'; 84 | stream.push(createFile(router, name, `${dataDir}/ready/gtfs`)); 85 | }); 86 | stream.end(); 87 | 88 | return stream; 89 | } 90 | 91 | /** 92 | * Make router data ready for the street only graph build in opentripplanner. 93 | * In the street only build case, only osm and dem data is fetched from the data directory, gtfs data is not fetched at all. 94 | */ 95 | function prepareRouterDataForStreetOnlyGraphBuild(router) { 96 | const stream = through.obj(); 97 | 98 | process.stdout.write( 99 | 'Collecting data and configuration files for street only graph build\n', 100 | ); 101 | 102 | stream.push(createFile(router, 'build-config.json', router.id)); 103 | stream.push(createFile(router, 'otp-config.json', router.id)); 104 | stream.push(createAndProcessRouterConfig(router)); 105 | router.osm.forEach(osmId => { 106 | const name = osmId + '.pbf'; 107 | stream.push(createFile(router, name, `${dataDir}/ready/osm`)); 108 | }); 109 | if (router.dem) { 110 | const name = router.dem + '.tif'; 111 | stream.push(createFile(router, name, `${dataDir}/ready/dem`)); 112 | } 113 | stream.end(); 114 | 115 | return stream; 116 | } 117 | 118 | function getDirectories(path) { 119 | const directoryContents = fs.readdirSync(path); 120 | const directories = directoryContents.filter(element => { 121 | return fs.statSync(path + '/' + element).isDirectory(); 122 | }); 123 | return directories; 124 | } 125 | 126 | /** 127 | * Make router data ready for the graph build from prebuilt data in opentripplanner. 128 | * In the prebuilt build case, only gtfs data is fetched from the data directory, 129 | * osm and dem data, as well as the prebuilt streetGraph.obj file is fetched from the osm-builds directory. 130 | */ 131 | function prepareRouterDataForPrebuiltStreetGraphBuild(router) { 132 | // check environmental variables which needs to be defined 133 | assert(process.env.DOCKER_TAG !== undefined, 'DOCKER_TAG must be defined'); 134 | 135 | const stream = through.obj(); 136 | 137 | process.stdout.write( 138 | 'Collecting data and configuration files for graph build based on prebuilt street graph data\n', 139 | ); 140 | 141 | stream.push(createFile(router, 'build-config.json', router.id)); 142 | stream.push(createFile(router, 'otp-config.json', router.id)); 143 | stream.push(createAndProcessRouterConfig(router)); 144 | router.src.forEach(src => { 145 | const name = src.id + '-gtfs.zip'; 146 | stream.push(createFile(router, name, `${dataDir}/ready/gtfs`)); 147 | }); 148 | 149 | const osmDirectories = getDirectories( 150 | `${storageDir}/osm-builds/${process.env.DOCKER_TAG}`, 151 | ); 152 | if (osmDirectories.length > 0) { 153 | osmDirectories.sort( 154 | (date1, date2) => dirNameToDate(date2) - dirNameToDate(date1), 155 | ); 156 | global.osmPrebuildDir = `${storageDir}/osm-builds/${process.env.DOCKER_TAG}/${osmDirectories[0]}/${router.id}`; 157 | process.stdout.write(`Using OSM data from ${global.osmPrebuildDir} \n`); 158 | // This is needed for gtfs data fitting and seeding. 159 | router.osm.forEach(osmId => { 160 | const name = osmId + '.pbf'; 161 | stream.push(createFile(router, name, global.osmPrebuildDir)); 162 | }); 163 | // This is needed for seeding. 164 | if (router.dem) { 165 | const name = router.dem + '.tif'; 166 | stream.push(createFile(router, name, global.osmPrebuildDir)); 167 | } 168 | // This is the prebuilt street graph. 169 | stream.push(createFile(router, 'streetGraph.obj', global.osmPrebuildDir)); 170 | } else { 171 | throw new Error(`No OSM directories can be found!\n`); 172 | } 173 | 174 | stream.end(); 175 | 176 | return stream; 177 | } 178 | 179 | module.exports = { 180 | prepareRouterData, 181 | prepareRouterDataForStreetOnlyGraphBuild, 182 | prepareRouterDataForPrebuiltStreetGraphBuild, 183 | }; 184 | -------------------------------------------------------------------------------- /task/ZipTask.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const cloneable = require('cloneable-readable'); 3 | const { execSync } = require('child_process'); 4 | const through = require('through2'); 5 | const { parseId, createDir } = require('../util'); 6 | const { dataDir } = require('../config.js'); 7 | 8 | /** 9 | * Moves files to a zip. 10 | * @param {string} zipFile - The name of the zip file 11 | * @param {string} path - The path to the data directory containing files to be restored 12 | * @param {string[]} filesToAdd - An array of filenames to add to the zip file 13 | */ 14 | function addToZip(zipFile, path, filesToAdd) { 15 | const existingFilePaths = filesToAdd 16 | .map(fileName => `${path}/${fileName}`) 17 | .filter(filePath => fs.existsSync(filePath)); 18 | if (existingFilePaths.length > 0) { 19 | const names = filesToAdd.join(' '); 20 | const params = `${zipFile} ${names}`; 21 | try { 22 | // remove old versions 23 | execSync(`cd ${path} && zip -d ${params}`, { stdio: 'pipe' }); 24 | } catch (err) { 25 | // Zip returns error 12 if file does not exist in zip 26 | if (err.status !== 12) { 27 | throw err; 28 | } 29 | } 30 | try { 31 | execSync(`cd ${path} && zip -u ${params}`, { stdio: 'pipe' }); 32 | } catch (err) { 33 | // Zip returns 12 code when the file(s) don't need to be updated as they already 34 | // exist in the zip in identical state. 35 | if (err.status !== 12) { 36 | throw err; 37 | } 38 | } 39 | process.stdout.write( 40 | `Added ${existingFilePaths.join(', ')} to ${zipFile}\n`, 41 | ); 42 | } 43 | } 44 | 45 | /** 46 | * Extracts files from a zip archive and saves them to given path 47 | * @param {string} zipName - zip file name 48 | * @param {string[]} filesToExtract - An array of filenames to extract from the archive 49 | * @param {string} path - The path to the data directory where files are put 50 | */ 51 | function extractFromZip(zipName, filesToExtract, path) { 52 | const filesString = filesToExtract 53 | .filter(name => zipHasFile(zipName, name)) 54 | .join(' '); 55 | execSync(`unzip -o -j ${zipName} ${filesString} -d ${path}`); 56 | process.stdout.write(`Extracted ${filesString} from ${zipName} to ${path}\n`); 57 | } 58 | 59 | /** 60 | * Delete files from a zip archive 61 | * @param {string} zipName - zip file name 62 | * @param {string[]} filesToRemove - An array of filenames to remove from the archive 63 | */ 64 | function removeFilesFromZip(zipName, filesToRemove) { 65 | const filesString = filesToRemove 66 | .filter(name => zipHasFile(zipName, name)) 67 | .join(' '); 68 | if (filesString.length > 0) { 69 | execSync(`zip -d ${zipName} ${filesString}`); 70 | process.stdout.write(`Removed ${filesString} from ${zipName}\n`); 71 | } 72 | } 73 | 74 | /** 75 | * Rename files in a zip archive 76 | * @param {string} zipName - zip file name 77 | * @param {object} oldNamesForFiles - object where the keys are the new names and values are the old names 78 | */ 79 | function renameFilesInZip(zipName, oldNamesForFiles) { 80 | for (const [newName, oldName] of Object.entries(oldNamesForFiles)) { 81 | if (zipHasFile(zipName, oldName)) { 82 | process.stdout.write(`renaming ${oldName} to ${newName}\n`); 83 | renameFileInZip(zipName, oldName, newName); 84 | } else { 85 | process.stdout.write(`${oldName} not in ${zipName}\n`); 86 | } 87 | } 88 | } 89 | 90 | /** 91 | * Rename a file in a zip archive 92 | * @param {string} zipName - zip file name 93 | * @param {string} oldName - original name for the file in zip 94 | * @param {string} newName - new name for the file in zip 95 | */ 96 | function renameFileInZip(zipName, oldName, newName) { 97 | try { 98 | // Don't output anything to logs as E_NOTIMPL errors can be verbose 99 | execSync(`7z rn ${zipName} ${oldName} ${newName}`, { stdio: 'pipe' }); 100 | } catch (err) { 101 | if (!err.message.match(/E_NOTIMPL/)) { 102 | throw err; 103 | } 104 | 105 | // Some zip files don't support renaming files properly so we need to extract the files and rename them. 106 | const tmpPathForFile = createTmpDir(parseId(zipName), 'tmp-rename'); 107 | extractFromZip(zipName, [oldName], tmpPathForFile); 108 | removeFilesFromZip(zipName, [oldName]); 109 | fs.renameSync( 110 | `${tmpPathForFile}/${oldName}`, 111 | `${tmpPathForFile}/${newName}`, 112 | ); 113 | addToZip(zipName, tmpPathForFile, [newName]); 114 | } 115 | process.stdout.write(`Renamed ${oldName} to ${newName} in ${zipName}\n`); 116 | } 117 | 118 | function zipHasFile(zipName, file) { 119 | try { 120 | execSync(`unzip -l ${zipName} | grep -qE '(^|\\s)${file}(\\s|$)'`); 121 | return true; 122 | // eslint-disable-next-line no-unused-vars 123 | } catch (err) { 124 | return false; 125 | } 126 | } 127 | 128 | /** 129 | * Extracts files from a zip archive and saves them to given path 130 | * @param {string} zipPath - zip file name 131 | * @param {string} destinationPath - The path to the data directory where files are put 132 | */ 133 | function extractAllFiles(zipPath, destinationPath) { 134 | execSync(`unzip -o ${zipPath} -d ${destinationPath}`); 135 | process.stdout.write(`Unzipped ${zipPath} to ${destinationPath}\n`); 136 | } 137 | 138 | /** 139 | * @param {string} zipFile file to create 140 | * @param {string[]} glob patterns for source files 141 | * @param {string} zipDir files are put into this directory inside the zip 142 | */ 143 | function zipWithGlobIntoDir(zipFile, glob, zipDir) { 144 | try { 145 | execSync(`rm -rf ${zipDir} && mkdir ${zipDir}`); 146 | // We don't want to command to fail if nothing matching a glob is found 147 | execSync(`cp ${glob.join(' ')} ${zipDir} 2>/dev/null || :`); 148 | execSync(`zip -rm ${zipFile} ${zipDir}`); 149 | process.stdout.write(`Created ${zipFile}\n`); 150 | return true; 151 | // eslint-disable-next-line no-unused-vars 152 | } catch (err) { 153 | process.stderr.write(`Error creating ${zipFile}\n`); 154 | return false; 155 | } 156 | } 157 | 158 | /** 159 | * @param {string} zipFile file to create 160 | * @param {string} dir source directory for files 161 | */ 162 | function zipDirContents(zipFile, dir) { 163 | try { 164 | execSync(`zip -j ${zipFile} ${dir}/*`); 165 | process.stdout.write(`Created ${zipFile}\n`); 166 | return true; 167 | // eslint-disable-next-line no-unused-vars 168 | } catch (err) { 169 | process.stderr.write(`Error creating ${zipFile}\n`); 170 | return false; 171 | } 172 | } 173 | 174 | function createTmpDir(dirName, baseDirectory) { 175 | const path = `${dataDir}/${baseDirectory}/${dirName}`; 176 | createDir(`${dataDir}/${baseDirectory}/${dirName}`); 177 | return path; 178 | } 179 | 180 | module.exports = { 181 | extractAllFiles, 182 | extractFilesTask: names => { 183 | if (!names?.length) { 184 | return through.obj(function (file, encoding, callback) { 185 | callback(null, file); 186 | }); 187 | } 188 | return through.obj(function (file, encoding, callback) { 189 | const localFile = file.history[file.history.length - 1]; 190 | const path = createTmpDir(parseId(localFile), 'tmp'); 191 | extractFromZip(localFile, names, path); 192 | file.contents = cloneable(fs.createReadStream(localFile)); 193 | callback(null, file); 194 | }); 195 | }, 196 | extractFromZip, 197 | addFilesTask: names => { 198 | if (!names?.length) { 199 | return through.obj(function (file, encoding, callback) { 200 | callback(null, file); 201 | }); 202 | } 203 | return through.obj(function (file, encoding, callback) { 204 | const localFile = file.history[file.history.length - 1]; 205 | const path = createTmpDir(parseId(localFile), 'tmp'); 206 | addToZip(localFile, path, names); 207 | file.contents = cloneable(fs.createReadStream(localFile)); 208 | callback(null, file); 209 | }); 210 | }, 211 | addToZip, 212 | removeFilesFromZip, 213 | renameFilesInZip, 214 | zipHasFile, 215 | zipWithGlobIntoDir, 216 | zipDirContents, 217 | }; 218 | -------------------------------------------------------------------------------- /hsl/router-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "routingDefaults": { 3 | "transferSlack": "1m30s", 4 | "waitReluctance": 0.95, 5 | "elevatorBoardTime": 60, 6 | "bicycle": { 7 | "boardCost": 120, 8 | "reluctance": 1.7, 9 | "optimization": "safest-streets" 10 | }, 11 | "car": { 12 | "reluctance": 10.0 13 | }, 14 | "walk": { 15 | "speed": 1.28, 16 | "reluctance": 1.75, 17 | "stairsReluctance": 1.2, 18 | "stairsTimeFactor": 2, 19 | "escalator": { 20 | "speed": 0.65 21 | }, 22 | "boardCost": 120 23 | }, 24 | "accessEgress": { 25 | "maxDuration": "1h", 26 | "penalty": { 27 | "FLEXIBLE": { 28 | "timePenalty": "10m + 1.9t", 29 | "costFactor": 2 30 | } 31 | } 32 | }, 33 | "maxDirectStreetDuration": "90m", 34 | "maxDirectStreetDurationForMode": { 35 | "bike": "360m" 36 | }, 37 | "maxJourneyDuration": "12h", 38 | "streetRoutingTimeout": "6s", 39 | "wheelchairAccessibility": { 40 | "stop": { 41 | "onlyConsiderAccessible": false, 42 | "unknownCost": 0, 43 | "inaccessibleCost": 100000 44 | }, 45 | "maxSlope": 0.125 46 | }, 47 | "itineraryFilters": { 48 | "transitGeneralizedCostLimit": { 49 | "costLimitFunction": "600 + 1.5x" 50 | }, 51 | "nonTransitGeneralizedCostLimit": "400 + 1.5x" 52 | }, 53 | "transitReluctanceForMode": { 54 | "BUS": 1.2, 55 | "SUBWAY": 0.9, 56 | "RAIL": 0.95 57 | }, 58 | "unpreferredCost": "1700 + 1.2x", 59 | "unpreferred": { 60 | "routes": [ 61 | "HSL:7191", 62 | "HSL:7191A", 63 | "HSL:7192", 64 | "HSL:7192KM", 65 | "HSL:7192M", 66 | "HSL:7192V", 67 | "HSL:7193B", 68 | "HSL:7193V", 69 | "HSL:7194", 70 | "HSL:7194K", 71 | "HSL:7194V", 72 | "HSL:7194VK", 73 | "HSL:7194VT", 74 | "HSL:7195K", 75 | "HSL:7195V", 76 | "HSL:7275", 77 | "HSL:7280", 78 | "HSL:7280A", 79 | "HSL:7282", 80 | "HSL:7285", 81 | "HSL:7375", 82 | "HSL:7375V", 83 | "HSL:7385", 84 | "HSL:7385A", 85 | "HSL:7455", 86 | "HSL:7455A", 87 | "HSL:7456", 88 | "HSL:7456A", 89 | "HSL:7456N", 90 | "HSL:7457", 91 | "HSL:7457A", 92 | "HSL:7464", 93 | "HSL:7465", 94 | "HSL:7465B", 95 | "HSL:7848", 96 | "HSL:7848A", 97 | "HSL:7849", 98 | "HSL:7999", 99 | "HSL:4343" 100 | ] 101 | } 102 | }, 103 | "gtfsApi": { 104 | "tracingTags": ["digitransit-subscription-id"] 105 | }, 106 | "flex": { 107 | "maxTransferDuration": "5m", 108 | "maxFlexTripDuration": "30m", 109 | "maxAccessWalkDuration": "5m", 110 | "maxEgressWalkDuration": "5m" 111 | }, 112 | "transit": { 113 | "maxSearchWindow": "8h", 114 | "pagingSearchWindowAdjustments": ["8h", "2h", "2h", "2h", "2h"], 115 | "dynamicSearchWindow": { 116 | "minWindow": "1h" 117 | }, 118 | "transferCacheRequests": [ 119 | { 120 | "modes": "WALK", 121 | "walk": { 122 | "reluctance": 1.8 123 | } 124 | }, 125 | { 126 | "modes": "WALK", 127 | "walk": { 128 | "reluctance": 1.8 129 | }, 130 | "wheelchairAccessibility": { 131 | "enabled": true 132 | } 133 | }, 134 | { 135 | "modes": "WALK", 136 | "walk": { 137 | "speed": 1.67, 138 | "reluctance": 1.8 139 | } 140 | }, 141 | { 142 | "modes": "BICYCLE", 143 | "walk": { 144 | "reluctance": 1.8 145 | }, 146 | "bicycle": { 147 | "speed": 5.55, 148 | "rental": { 149 | "useAvailabilityInformation": true 150 | } 151 | } 152 | }, 153 | { 154 | "modes": "BICYCLE", 155 | "walk": { 156 | "speed": 1.67, 157 | "reluctance": 1.8 158 | }, 159 | "bicycle": { 160 | "speed": 5.55, 161 | "rental": { 162 | "useAvailabilityInformation": true 163 | } 164 | } 165 | } 166 | ] 167 | }, 168 | "vectorTiles": { 169 | "attribution": "Digitransit data is licensed under CC BY 4.0.", 170 | "layers": [ 171 | { 172 | "name": "stops", 173 | "type": "Stop", 174 | "mapper": "Digitransit", 175 | "maxZoom": 20, 176 | "minZoom": 5, 177 | "cacheMaxSeconds": 43200 178 | }, 179 | { 180 | "name": "realtimeStops", 181 | "type": "Stop", 182 | "mapper": "DigitransitRealtime", 183 | "maxZoom": 20, 184 | "minZoom": 5, 185 | "cacheMaxSeconds": 60 186 | }, 187 | { 188 | "name": "stations", 189 | "type": "Station", 190 | "mapper": "Digitransit", 191 | "maxZoom": 20, 192 | "minZoom": 5, 193 | "cacheMaxSeconds": 43200 194 | }, 195 | { 196 | "name": "rentalStations", 197 | "type": "VehicleRentalStation", 198 | "mapper": "Digitransit", 199 | "maxZoom": 20, 200 | "minZoom": 5, 201 | "cacheMaxSeconds": 43200, 202 | "expansionFactor": 0.25 203 | }, 204 | { 205 | "name": "realtimeRentalStations", 206 | "type": "VehicleRentalStation", 207 | "mapper": "DigitransitRealtime", 208 | "maxZoom": 20, 209 | "minZoom": 5, 210 | "cacheMaxSeconds": 45, 211 | "expansionFactor": 0.25 212 | }, 213 | { 214 | "name": "realtimeRentalVehicles", 215 | "type": "VehicleRentalVehicle", 216 | "mapper": "DigitransitRealtime", 217 | "maxZoom": 20, 218 | "minZoom": 5, 219 | "cacheMaxSeconds": 45, 220 | "expansionFactor": 0.25 221 | }, 222 | { 223 | "name": "vehicleParking", 224 | "type": "VehicleParking", 225 | "mapper": "Digitransit", 226 | "maxZoom": 20, 227 | "minZoom": 5, 228 | "cacheMaxSeconds": 43200, 229 | "expansionFactor": 0.25 230 | }, 231 | { 232 | "name": "vehicleParkingGroups", 233 | "type": "VehicleParkingGroup", 234 | "mapper": "Digitransit", 235 | "maxZoom": 20, 236 | "minZoom": 5, 237 | "cacheMaxSeconds": 43200, 238 | "expansionFactor": 0.25 239 | } 240 | ] 241 | }, 242 | "updaters": [ 243 | { 244 | "id": "hsl-trip-updates", 245 | "type": "mqtt-gtfs-rt-updater", 246 | "url": "tcp://pred.rt.hsl.fi", 247 | "topic": "gtfsrt/v2/fi/hsl/tu", 248 | "feedId": "HSL", 249 | "fuzzyTripMatching": true, 250 | "backwardsDelayPropagationType": "ALWAYS" 251 | }, 252 | { 253 | "id": "hsl-alerts", 254 | "type": "real-time-alerts", 255 | "frequency": "30s", 256 | "url": "https://realtime.hsl.fi/realtime/service-alerts/v2/hsl", 257 | "feedId": "HSL", 258 | "fuzzyTripMatching": true 259 | }, 260 | { 261 | "id": "liipi", 262 | "type": "vehicle-parking", 263 | "sourceType": "liipi", 264 | "feedId": "liipi", 265 | "timeZone": "Europe/Helsinki", 266 | "facilitiesFrequencySec": 3600, 267 | "facilitiesUrl": "https://parking.fintraffic.fi/api/v1/facilities.json?limit=-1", 268 | "utilizationsFrequencySec": 600, 269 | "utilizationsUrl": "https://parking.fintraffic.fi/api/v1/utilizations.json?limit=-1", 270 | "hubsUrl": "https://parking.fintraffic.fi/api/v1/hubs.json?limit=-1" 271 | }, 272 | { 273 | "id": "vantaa-bike-rental", 274 | "type": "vehicle-rental", 275 | "sourceType": "gbfs", 276 | "frequency": "30s", 277 | "network": "vantaa", 278 | "url": "http://digitransit-proxy:8080/out/vantaa-api.giravolta.io/api-opendata/gbfs/2_3/gbfs", 279 | "overloadingAllowed": true, 280 | "rentalPickupTypes": ["station"] 281 | }, 282 | { 283 | "id": "hsl-bike-rental", 284 | "type": "vehicle-rental", 285 | "sourceType": "smoove", 286 | "frequency": "30s", 287 | "network": "smoove", 288 | "url": "http://digitransit-proxy:8080/out/helsinki-fi.smoove.pro/api-public/stations", 289 | "overloadingAllowed": true 290 | }, 291 | { 292 | "type": "vehicle-positions", 293 | "url": "https://realtime.hsl.fi/realtime/vehicle-positions/v2/hsl", 294 | "feedId": "HSL", 295 | "frequency": "30s", 296 | "fuzzyTripMatching": true, 297 | "features": ["stop-position", "occupancy"] 298 | } 299 | ] 300 | } 301 | -------------------------------------------------------------------------------- /otp-data-server/lighttpd.conf: -------------------------------------------------------------------------------- 1 | 2 | # {{{ variables 3 | var.basedir = "/var/www/localhost" 4 | var.logdir = "/var/log/lighttpd" 5 | var.statedir = "/var/lib/lighttpd" 6 | # }}} 7 | 8 | # {{{ modules 9 | # At the very least, mod_access and mod_accesslog should be enabled. 10 | # All other modules should only be loaded if necessary. 11 | # NOTE: the order of modules is important. 12 | server.modules = ( 13 | # "mod_rewrite", 14 | # "mod_alias", 15 | "mod_access", 16 | # "mod_cml", 17 | # "mod_trigger_b4_dl", 18 | # "mod_auth", 19 | # "mod_status", 20 | # "mod_setenv", 21 | # "mod_proxy", 22 | # "mod_simple_vhost", 23 | # "mod_evhost", 24 | # "mod_userdir", 25 | # "mod_compress", 26 | # "mod_ssi", 27 | # "mod_usertrack", 28 | # "mod_expire", 29 | # "mod_secdownload", 30 | # "mod_rrdtool", 31 | # "mod_webdav", 32 | "mod_accesslog" 33 | ) 34 | # }}} 35 | 36 | # {{{ includes 37 | include "mime-types.conf" 38 | # uncomment for cgi support 39 | # include "mod_cgi.conf" 40 | # uncomment for php/fastcgi support 41 | # include "mod_fastcgi.conf" 42 | # uncomment for php/fastcgi fpm support 43 | # include "mod_fastcgi_fpm.conf" 44 | # }}} 45 | 46 | # {{{ server settings 47 | #server.username = "lighttpd" 48 | #server.groupname = "lighttpd" 49 | 50 | server.document-root = var.basedir + "/htdocs/" + env.OTP_GRAPH_DIR 51 | server.pid-file = "/run/lighttpd.pid" 52 | 53 | server.errorlog = "/dev/stderr" 54 | # log errors to syslog instead 55 | # server.errorlog-use-syslog = "enable" 56 | 57 | #server.indexfiles = ("version.txt") 58 | 59 | # server.tag = "lighttpd" 60 | 61 | server.follow-symlink = "enable" 62 | 63 | # event handler (defaults to "poll") 64 | # see performance.txt 65 | # 66 | # for >= linux-2.4 67 | # server.event-handler = "linux-rtsig" 68 | # for >= linux-2.6 69 | # server.event-handler = "linux-sysepoll" 70 | # for FreeBSD 71 | # server.event-handler = "freebsd-kqueue" 72 | 73 | # chroot to directory (defaults to no chroot) 74 | # server.chroot = "/" 75 | 76 | # bind to port (defaults to 80) 77 | # server.port = 81 78 | 79 | # bind to name (defaults to all interfaces) 80 | # server.bind = "grisu.home.kneschke.de" 81 | 82 | # error-handler for status 404 83 | # server.error-handler-404 = "/error-handler.html" 84 | # server.error-handler-404 = "/error-handler.php" 85 | 86 | # Format: .html 87 | # -> ..../status-404.html for 'File not found' 88 | # server.errorfile-prefix = var.basedir + "/error/status-" 89 | 90 | # FAM support for caching stat() calls 91 | # requires that lighttpd be built with USE=fam 92 | # server.stat-cache-engine = "fam" 93 | # }}} 94 | 95 | # {{{ mod_staticfile 96 | 97 | # which extensions should not be handled via static-file transfer 98 | # (extensions that are usually handled by mod_cgi, mod_fastcgi, etc). 99 | static-file.exclude-extensions = (".php", ".pl", ".cgi", ".fcgi") 100 | # }}} 101 | 102 | # {{{ mod_accesslog 103 | accesslog.filename = "/dev/stderr" 104 | # }}} 105 | 106 | # {{{ mod_dirlisting 107 | # enable directory listings 108 | dir-listing.activate = "enable" 109 | # 110 | # don't list hidden files/directories 111 | dir-listing.hide-dotfiles = "enable" 112 | # 113 | # use a different css for directory listings 114 | # dir-listing.external-css = "/path/to/dir-listing.css" 115 | # 116 | # list of regular expressions. files that match any of the 117 | # specified regular expressions will be excluded from directory 118 | # listings. 119 | # dir-listing.exclude = ("^\.", "~$") 120 | # }}} 121 | 122 | # {{{ mod_access 123 | # see access.txt 124 | 125 | url.access-deny = ("~", ".inc") 126 | # }}} 127 | 128 | # {{{ mod_userdir 129 | # see userdir.txt 130 | # 131 | # userdir.path = "public_html" 132 | # userdir.exclude-user = ("root") 133 | # }}} 134 | 135 | # {{{ mod_ssi 136 | # see ssi.txt 137 | # 138 | # ssi.extension = (".shtml") 139 | # }}} 140 | 141 | # {{{ mod_ssl 142 | # see ssl.txt 143 | # 144 | # ssl.engine = "enable" 145 | # ssl.pemfile = "server.pem" 146 | # }}} 147 | 148 | # {{{ mod_status 149 | # see status.txt 150 | # 151 | # status.status-url = "/server-status" 152 | # status.config-url = "/server-config" 153 | # }}} 154 | 155 | # {{{ mod_simple_vhost 156 | # see simple-vhost.txt 157 | # 158 | # If you want name-based virtual hosting add the next three settings and load 159 | # mod_simple_vhost 160 | # 161 | # document-root = 162 | # virtual-server-root + virtual-server-default-host + virtual-server-docroot 163 | # or 164 | # virtual-server-root + http-host + virtual-server-docroot 165 | # 166 | # simple-vhost.server-root = "/home/weigon/wwwroot/servers/" 167 | # simple-vhost.default-host = "grisu.home.kneschke.de" 168 | # simple-vhost.document-root = "/pages/" 169 | # }}} 170 | 171 | # {{{ mod_compress 172 | # see compress.txt 173 | # 174 | # compress.cache-dir = var.statedir + "/cache/compress" 175 | # compress.filetype = ("text/plain", "text/html") 176 | # }}} 177 | 178 | # {{{ mod_proxy 179 | # see proxy.txt 180 | # 181 | # proxy.server = ( ".php" => 182 | # ( "localhost" => 183 | # ( 184 | # "host" => "192.168.0.101", 185 | # "port" => 80 186 | # ) 187 | # ) 188 | # ) 189 | # }}} 190 | 191 | # {{{ mod_auth 192 | # see authentication.txt 193 | # 194 | # auth.backend = "plain" 195 | # auth.backend.plain.userfile = "lighttpd.user" 196 | # auth.backend.plain.groupfile = "lighttpd.group" 197 | 198 | # auth.backend.ldap.hostname = "localhost" 199 | # auth.backend.ldap.base-dn = "dc=my-domain,dc=com" 200 | # auth.backend.ldap.filter = "(uid=$)" 201 | 202 | # auth.require = ( "/server-status" => 203 | # ( 204 | # "method" => "digest", 205 | # "realm" => "download archiv", 206 | # "require" => "user=jan" 207 | # ), 208 | # "/server-info" => 209 | # ( 210 | # "method" => "digest", 211 | # "realm" => "download archiv", 212 | # "require" => "valid-user" 213 | # ) 214 | # ) 215 | # }}} 216 | 217 | # {{{ mod_redirect 218 | # see redirect.txt 219 | # 220 | # url.redirect = ( 221 | # "^/wishlist/(.+)" => "http://www.123.org/$1" 222 | # ) 223 | # }}} 224 | 225 | # {{{ mod_evhost 226 | # define a pattern for the host url finding 227 | # %% => % sign 228 | # %0 => domain name + tld 229 | # %1 => tld 230 | # %2 => domain name without tld 231 | # %3 => subdomain 1 name 232 | # %4 => subdomain 2 name 233 | # 234 | # evhost.path-pattern = "/home/storage/dev/www/%3/htdocs/" 235 | # }}} 236 | 237 | # {{{ mod_expire 238 | # expire.url = ( 239 | # "/buggy/" => "access 2 hours", 240 | # "/asdhas/" => "access plus 1 seconds 2 minutes" 241 | # ) 242 | # }}} 243 | 244 | # {{{ mod_rrdtool 245 | # see rrdtool.txt 246 | # 247 | # rrdtool.binary = "/usr/bin/rrdtool" 248 | # rrdtool.db-name = var.statedir + "/lighttpd.rrd" 249 | # }}} 250 | 251 | # {{{ mod_setenv 252 | # see setenv.txt 253 | # 254 | # setenv.add-request-header = ( "TRAV_ENV" => "mysql://user@host/db" ) 255 | # setenv.add-response-header = ( "X-Secret-Message" => "42" ) 256 | # }}} 257 | 258 | # {{{ mod_trigger_b4_dl 259 | # see trigger_b4_dl.txt 260 | # 261 | # trigger-before-download.gdbm-filename = "/home/weigon/testbase/trigger.db" 262 | # trigger-before-download.memcache-hosts = ( "127.0.0.1:11211" ) 263 | # trigger-before-download.trigger-url = "^/trigger/" 264 | # trigger-before-download.download-url = "^/download/" 265 | # trigger-before-download.deny-url = "http://127.0.0.1/index.html" 266 | # trigger-before-download.trigger-timeout = 10 267 | # }}} 268 | 269 | # {{{ mod_cml 270 | # see cml.txt 271 | # 272 | # don't forget to add index.cml to server.indexfiles 273 | # cml.extension = ".cml" 274 | # cml.memcache-hosts = ( "127.0.0.1:11211" ) 275 | # }}} 276 | 277 | # {{{ mod_webdav 278 | # see webdav.txt 279 | # 280 | # $HTTP["url"] =~ "^/dav($|/)" { 281 | # webdav.activate = "enable" 282 | # webdav.is-readonly = "enable" 283 | # } 284 | # }}} 285 | 286 | # {{{ extra rules 287 | # 288 | # set Content-Encoding and reset Content-Type for browsers that 289 | # support decompressing on-thy-fly (requires mod_setenv) 290 | # $HTTP["url"] =~ "\.gz$" { 291 | # setenv.add-response-header = ("Content-Encoding" => "x-gzip") 292 | # mimetype.assign = (".gz" => "text/plain") 293 | # } 294 | 295 | # $HTTP["url"] =~ "\.bz2$" { 296 | # setenv.add-response-header = ("Content-Encoding" => "x-bzip2") 297 | # mimetype.assign = (".bz2" => "text/plain") 298 | # } 299 | # 300 | # }}} 301 | 302 | # {{{ debug 303 | # debug.log-request-header = "enable" 304 | # debug.log-response-header = "enable" 305 | # debug.log-request-handling = "enable" 306 | # debug.log-file-not-found = "enable" 307 | # }}} 308 | 309 | # vim: set ft=conf foldmethod=marker et : 310 | -------------------------------------------------------------------------------- /task/Update.js: -------------------------------------------------------------------------------- 1 | /* 2 | Executes gulp tasks which download new data, builds and tests a new graph, saves it in storage and deploys 3 | new opentripplanner-data-server and opentripplanner versions that use that data. 4 | Data errors are detected and tolerated to a certain limit thanks to fallback mechanism to older data. 5 | Unexpected code execution errors and failures in graph build abort the data loading. 6 | */ 7 | const gulp = require('gulp'); 8 | const { promisify } = require('util'); 9 | const { execFileSync } = require('child_process'); 10 | const fs = require('fs'); 11 | const { postSlackMessage, updateSlackMessage } = require('../util'); 12 | require('../gulpfile'); 13 | const { router, SPLIT_BUILD_TYPE } = require('../config'); 14 | const assert = require('assert'); 15 | 16 | const MAX_GTFS_FALLBACK = 2; // threshold for aborting data loading 17 | 18 | const start = promisify((task, cb) => gulp.series(task)(cb)); 19 | 20 | /** 21 | * Docker tags don't work with ':' and file names are also prettier without them. We also need to 22 | * remove milliseconds because they are not relevant and make converting string back to ISO format 23 | * more difficult. 24 | * @returns date as string 25 | */ 26 | function getDateString() { 27 | return new Date().toISOString().slice(0, -5).concat('Z').replace(/:/g, '.'); 28 | } 29 | 30 | async function handleSeeding() { 31 | if (!process.env.NOSEED) { 32 | process.stdout.write('Starting seeding\n'); 33 | await start('seed'); 34 | process.stdout.write('Seeded\n'); 35 | } 36 | } 37 | 38 | async function handleOsmAndDemUpdate() { 39 | if (!process.env.NODEM) { 40 | // we track data rejections using this global variable 41 | global.hasFailures = false; 42 | await start('dem:update'); 43 | if (global.hasFailures) { 44 | postSlackMessage('DEM update failed, using previous version :boom:'); 45 | } 46 | } 47 | 48 | // OSM update is more complicated. Download often fails, so there is a retry loop, 49 | // which breaks when a big enough file gets loaded 50 | if (!process.env.USE_SEEDED_OSM) { 51 | global.blobSizeOk = false; // ugly hack but gulp does not return any values from tasks 52 | for (let i = 0; i < 3; i++) { 53 | await start('osm:update'); 54 | if (global.blobSizeOk) { 55 | break; 56 | } 57 | if (i < 2) { 58 | // sleep 10 mins before next attempt 59 | await new Promise(resolve => setTimeout(resolve, 600000)); 60 | } 61 | } 62 | if (!global.blobSizeOk) { 63 | global.hasFailures = true; 64 | postSlackMessage('OSM data update failed, using previous version :boom:'); 65 | } 66 | } else { 67 | process.stdout.write( 68 | 'Skipping OSM update and using existing seeded data\n', 69 | ); 70 | } 71 | } 72 | 73 | function handleTests() { 74 | if (process.env.SKIPPED_SITES === 'all' || process.env.SKIP_OTP_TESTS) { 75 | process.stdout.write('Skipping all tests\n'); 76 | } else { 77 | process.stdout.write('Test the newly built graph with OTPQA\n'); 78 | execFileSync('./test.sh', [], { stdio: [0, 1, 2] }); 79 | } 80 | } 81 | 82 | async function handleGtfsFallback(logFile) { 83 | // testing detected routing problems 84 | global.hasFailures = true; 85 | 86 | global.failedFeeds = fs.readFileSync(logFile, 'utf8'); // comma separated list of feed ids. No newline at end! 87 | fs.unlinkSync(logFile); // cleanup for local use 88 | 89 | if (global.failedFeeds.split(',').length > MAX_GTFS_FALLBACK) { 90 | updateSlackMessage( 91 | 'Aborting the data update because too many quality tests failed :boom:', 92 | ); 93 | process.exit(1); 94 | } 95 | 96 | postSlackMessage( 97 | `GTFS packages ${global.failedFeeds} rejected, using fallback to current data`, 98 | ); 99 | // use seed packages for failed feeds 100 | await start('gtfs:fallback'); 101 | } 102 | 103 | function buildAndDeployDockerImages(date) { 104 | process.stdout.write('Build and deploy Docker images\n'); 105 | execFileSync('./otp-data-server/deploy.sh', [date], { 106 | stdio: [0, 1, 2], 107 | env: { 108 | OTP_TAG: process.env.OTP_TAG, 109 | OTP_GRAPH_DIR: global.storageDirName, 110 | ROUTER_NAME: process.env.ROUTER_NAME, 111 | ORG: process.env.ORG, 112 | DOCKER_TAG: process.env.DOCKER_TAG, 113 | DOCKER_USER: process.env.DOCKER_USER, 114 | DOCKER_AUTH: process.env.DOCKER_AUTH, 115 | }, 116 | }); 117 | execFileSync('./opentripplanner/deploy-otp.sh', [date], { 118 | stdio: [0, 1, 2], 119 | env: { 120 | OTP_TAG: process.env.OTP_TAG, 121 | OTP_GRAPH_DIR: global.storageDirName, 122 | ROUTER_NAME: process.env.ROUTER_NAME, 123 | ORG: process.env.ORG, 124 | DOCKER_TAG: process.env.DOCKER_TAG, 125 | DOCKER_USER: process.env.DOCKER_USER, 126 | DOCKER_AUTH: process.env.DOCKER_AUTH, 127 | }, 128 | }); 129 | } 130 | 131 | async function handleCleanup() { 132 | if (!process.env.NOCLEANUP) { 133 | process.stdout.write('Remove oldest data versions from storage\n'); 134 | await start('storage:cleanup'); 135 | } 136 | } 137 | 138 | /** 139 | * This function only builds the street graph with OSM and DEM data. 140 | */ 141 | async function buildStreetOnlyGraph(name) { 142 | await handleSeeding(); 143 | 144 | await handleOsmAndDemUpdate(); 145 | 146 | process.stdout.write('Build street only graph\n'); 147 | await start('router:buildStreetOnlyGraph'); 148 | 149 | const date = getDateString(); 150 | global.storageDirName = `osm-builds/${process.env.DOCKER_TAG}/${date}/${name}`; 151 | 152 | process.stdout.write('Uploading street graph only build data to storage\n'); 153 | await start('router:store'); 154 | 155 | if (!process.env.NOCLEANUP) { 156 | process.stdout.write( 157 | 'Remove oldest street only graph data versions from storage\n', 158 | ); 159 | await start('storage:cleanupStreetOnlyGraphData'); 160 | } 161 | 162 | if (global.hasFailures) { 163 | updateSlackMessage( 164 | `${name} street only graph data updated, but partially falling back to older data :boom:`, 165 | ); 166 | } else { 167 | updateSlackMessage( 168 | `${name} street only graph data updated :white_check_mark:`, 169 | ); 170 | } 171 | } 172 | 173 | /** 174 | * This function does the whole build. 175 | */ 176 | async function buildGraph(name) { 177 | await handleSeeding(); 178 | 179 | await handleOsmAndDemUpdate(); 180 | 181 | await start('gtfs:update'); 182 | 183 | process.stdout.write('Build routing graph\n'); 184 | await start('router:buildGraph'); 185 | 186 | handleTests(); 187 | 188 | const logFile = 'failed_feeds.txt'; 189 | if (fs.existsSync(logFile)) { 190 | await handleGtfsFallback(logFile); 191 | // rebuild the graph 192 | process.stdout.write('Rebuild graph using fallback data\n'); 193 | await start('router:buildGraph'); 194 | } 195 | 196 | const date = getDateString(); 197 | global.storageDirName = `${process.env.DOCKER_TAG}/${date}/${name}`; 198 | 199 | process.stdout.write('Uploading data to storage\n'); 200 | await start('router:store'); 201 | 202 | buildAndDeployDockerImages(date); 203 | 204 | await handleCleanup(); 205 | 206 | if (global.hasFailures) { 207 | updateSlackMessage( 208 | `${name} data updated, but partially falling back to older data :boom:`, 209 | ); 210 | } else { 211 | updateSlackMessage(`${name} data updated :white_check_mark:`); 212 | } 213 | } 214 | 215 | /** 216 | * This function builds the graph from prebuilt street graph data. 217 | */ 218 | async function buildWithPrebuiltStreetGraph(name) { 219 | await handleSeeding(); 220 | 221 | await start('gtfs:update'); 222 | 223 | process.stdout.write('Build routing graph from prebuilt street only graph\n'); 224 | await start('router:buildWithPrebuiltStreetGraph'); 225 | 226 | handleTests(); 227 | 228 | const logFile = 'failed_feeds.txt'; 229 | if (fs.existsSync(logFile)) { 230 | await handleGtfsFallback(logFile); 231 | // rebuild the graph 232 | process.stdout.write('Rebuild graph using fallback data\n'); 233 | await start('router:buildWithPrebuiltStreetGraph'); 234 | } 235 | 236 | const date = getDateString(); 237 | global.storageDirName = `${process.env.DOCKER_TAG}/${date}/${name}`; 238 | 239 | process.stdout.write('Uploading data to storage\n'); 240 | await start('router:storeForPrebuiltStreetGraphDataBuild'); 241 | 242 | buildAndDeployDockerImages(date); 243 | 244 | await handleCleanup(); 245 | 246 | if (global.hasFailures) { 247 | updateSlackMessage( 248 | `${name} data updated from prebuilt street only graph, but partially falling back to older data :boom:`, 249 | ); 250 | } else { 251 | updateSlackMessage( 252 | `${name} data updated from prebuilt street only graph :white_check_mark:`, 253 | ); 254 | } 255 | } 256 | 257 | async function update() { 258 | // check environmental variables which needs to be defined 259 | assert(process.env.DOCKER_TAG !== undefined, 'DOCKER_TAG must be defined'); 260 | 261 | const name = router.id; 262 | try { 263 | switch (SPLIT_BUILD_TYPE) { 264 | case 'ONLY_BUILD_STREET_GRAPH': 265 | await buildStreetOnlyGraph(name); 266 | break; 267 | case 'USE_PREBUILT_STREET_GRAPH': 268 | await buildWithPrebuiltStreetGraph(name); 269 | break; 270 | default: 271 | await buildGraph(name); 272 | break; 273 | } 274 | } catch (err) { 275 | postSlackMessage(`${name} data update failed: ` + err.message); 276 | updateSlackMessage('Something went wrong with the data update :boom:'); 277 | } 278 | } 279 | 280 | module.exports = { 281 | update, 282 | }; 283 | -------------------------------------------------------------------------------- /waltti-alt/router-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "routingDefaults": { 3 | "transferSlack": "1m30s", 4 | "waitReluctance": 0.95, 5 | "elevatorBoardTime": 60, 6 | "bicycle": { 7 | "boardCost": 120, 8 | "reluctance": 1.7, 9 | "optimization": "safest-streets" 10 | }, 11 | "car": { 12 | "reluctance": 10.0 13 | }, 14 | "walk": { 15 | "speed": 1.3, 16 | "reluctance": 1.75, 17 | "stairsReluctance": 1.2, 18 | "stairsTimeFactor": 2, 19 | "escalator": { 20 | "speed": 0.65 21 | }, 22 | "boardCost": 120 23 | }, 24 | "accessEgress": { 25 | "maxDuration": "1h" 26 | }, 27 | "maxDirectStreetDuration": "4h", 28 | "maxDirectStreetDurationForMode": { 29 | "walk": "90m" 30 | }, 31 | "maxJourneyDuration": "12h", 32 | "streetRoutingTimeout": "8s", 33 | "wheelchairAccessibility": { 34 | "stop": { 35 | "onlyConsiderAccessible": false, 36 | "unknownCost": 0, 37 | "inaccessibleCost": 100000 38 | }, 39 | "maxSlope": 0.125 40 | }, 41 | "itineraryFilters": { 42 | "transitGeneralizedCostLimit": { 43 | "costLimitFunction": "600 + 1.5x" 44 | }, 45 | "nonTransitGeneralizedCostLimit": "400 + 1.5x" 46 | } 47 | }, 48 | "gtfsApi": { 49 | "tracingTags": ["digitransit-subscription-id"] 50 | }, 51 | "transit": { 52 | "pagingSearchWindowAdjustments": ["8h", "4h", "4h", "4h", "4h"], 53 | "dynamicSearchWindow": { 54 | "minWindow": "3h" 55 | }, 56 | "maxNumberOfTransfers": 12, 57 | "transferCacheRequests": [ 58 | { 59 | "modes": "WALK", 60 | "walk": { 61 | "speed": 1.2, 62 | "reluctance": 1.8 63 | } 64 | }, 65 | { 66 | "modes": "WALK", 67 | "walk": { 68 | "speed": 1.2, 69 | "reluctance": 1.8 70 | }, 71 | "wheelchairAccessibility": { 72 | "enabled": true 73 | } 74 | }, 75 | { 76 | "modes": "WALK", 77 | "walk": { 78 | "speed": 1.67, 79 | "reluctance": 1.8 80 | } 81 | }, 82 | { 83 | "modes": "BICYCLE", 84 | "walk": { 85 | "speed": 1.2, 86 | "reluctance": 1.8 87 | }, 88 | "bicycle": { 89 | "speed": 5.55, 90 | "rental": { 91 | "useAvailabilityInformation": true 92 | } 93 | } 94 | }, 95 | { 96 | "modes": "BICYCLE", 97 | "walk": { 98 | "speed": 1.67, 99 | "reluctance": 1.8 100 | }, 101 | "bicycle": { 102 | "speed": 5.55, 103 | "rental": { 104 | "useAvailabilityInformation": true 105 | } 106 | } 107 | } 108 | ] 109 | }, 110 | "vectorTiles": { 111 | "attribution": "Digitransit data is licensed under CC BY 4.0.", 112 | "layers": [ 113 | { 114 | "name": "stops", 115 | "type": "Stop", 116 | "mapper": "Digitransit", 117 | "maxZoom": 20, 118 | "minZoom": 5, 119 | "cacheMaxSeconds": 43200 120 | }, 121 | { 122 | "name": "realtimeStops", 123 | "type": "Stop", 124 | "mapper": "DigitransitRealtime", 125 | "maxZoom": 20, 126 | "minZoom": 5, 127 | "cacheMaxSeconds": 60 128 | }, 129 | { 130 | "name": "stations", 131 | "type": "Station", 132 | "mapper": "Digitransit", 133 | "maxZoom": 20, 134 | "minZoom": 5, 135 | "cacheMaxSeconds": 43200 136 | }, 137 | { 138 | "name": "rentalStations", 139 | "type": "VehicleRentalStation", 140 | "mapper": "Digitransit", 141 | "maxZoom": 20, 142 | "minZoom": 5, 143 | "cacheMaxSeconds": 43200, 144 | "expansionFactor": 0.25 145 | }, 146 | { 147 | "name": "realtimeRentalStations", 148 | "type": "VehicleRentalStation", 149 | "mapper": "DigitransitRealtime", 150 | "maxZoom": 20, 151 | "minZoom": 5, 152 | "cacheMaxSeconds": 45, 153 | "expansionFactor": 0.25 154 | }, 155 | { 156 | "name": "vehicleParking", 157 | "type": "VehicleParking", 158 | "mapper": "Digitransit", 159 | "maxZoom": 20, 160 | "minZoom": 5, 161 | "cacheMaxSeconds": 43200, 162 | "expansionFactor": 0.25 163 | }, 164 | { 165 | "name": "vehicleParkingGroups", 166 | "type": "VehicleParkingGroup", 167 | "mapper": "Digitransit", 168 | "maxZoom": 20, 169 | "minZoom": 5, 170 | "cacheMaxSeconds": 43200, 171 | "expansionFactor": 0.25 172 | } 173 | ] 174 | }, 175 | "updaters": [ 176 | { 177 | "id": "walttitest-trip-updates", 178 | "type": "stop-time-updater", 179 | "frequency": "60s", 180 | "url": "http://digitransit-proxy:8080/out/lmj.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 181 | "feedId": "WalttiTest", 182 | "fuzzyTripMatching": false, 183 | "backwardsDelayPropagationType": "ALWAYS" 184 | }, 185 | { 186 | "id": "walttitest-alerts", 187 | "type": "real-time-alerts", 188 | "frequency": "30s", 189 | "url": "http://digitransit-proxy:8080/out/lmj.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 190 | "feedId": "WalttiTest", 191 | "fuzzyTripMatching": false 192 | }, 193 | { 194 | "id": "foli-trip-updates", 195 | "type": "stop-time-updater", 196 | "frequency": "60s", 197 | "url": "http://siri2gtfsrt:8080/FOLI", 198 | "feedId": "FOLI", 199 | "fuzzyTripMatching": true, 200 | "backwardsDelayPropagationType": "ALWAYS" 201 | }, 202 | { 203 | "id": "foli-alerts", 204 | "type": "real-time-alerts", 205 | "frequency": "30s", 206 | "url": "http://digitransit-proxy:8080/out/data.foli.fi/gtfs-rt/reittiopas", 207 | "feedId": "FOLI", 208 | "fuzzyTripMatching": false 209 | }, 210 | { 211 | "id": "turku-bike-rental", 212 | "type": "vehicle-rental", 213 | "sourceType": "gbfs", 214 | "frequency": "30s", 215 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_turku/gbfs.json", 216 | "overloadingAllowed": true 217 | }, 218 | { 219 | "id": "liipi", 220 | "type": "vehicle-parking", 221 | "sourceType": "liipi", 222 | "feedId": "liipi", 223 | "timeZone": "Europe/Helsinki", 224 | "facilitiesFrequencySec": 3600, 225 | "facilitiesUrl": "https://parking.fintraffic.fi/api/v1/facilities.json?limit=-1", 226 | "utilizationsFrequencySec": 600, 227 | "utilizationsUrl": "https://parking.fintraffic.fi/api/v1/utilizations.json?limit=-1", 228 | "hubsUrl": "https://parking.fintraffic.fi/api/v1/hubs.json?limit=-1" 229 | }, 230 | { 231 | "id": "pori-trip-updates", 232 | "type": "stop-time-updater", 233 | "frequency": "60s", 234 | "url": "http://digitransit-proxy:8080/out/pjl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 235 | "feedId": "Pori", 236 | "fuzzyTripMatching": false, 237 | "backwardsDelayPropagationType": "ALWAYS" 238 | }, 239 | { 240 | "id": "pori-alerts", 241 | "type": "real-time-alerts", 242 | "frequency": "30s", 243 | "url": "http://digitransit-proxy:8080/out/pjl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 244 | "feedId": "Pori", 245 | "fuzzyTripMatching": false 246 | }, 247 | { 248 | "id": "oulu-trip-updates", 249 | "type": "stop-time-updater", 250 | "frequency": "60s", 251 | "url": "http://digitransit-proxy:8080/out/oulu.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 252 | "feedId": "OULU", 253 | "fuzzyTripMatching": false, 254 | "backwardsDelayPropagationType": "ALWAYS" 255 | }, 256 | { 257 | "id": "oulu-alerts", 258 | "type": "real-time-alerts", 259 | "frequency": "30s", 260 | "url": "http://digitransit-proxy:8080/out/oulu.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 261 | "feedId": "OULU", 262 | "fuzzyTripMatching": false 263 | }, 264 | { 265 | "id": "walttitest-trip-updates", 266 | "type": "stop-time-updater", 267 | "frequency": "60s", 268 | "url": "http://digitransit-proxy:8080/out/lmj.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 269 | "feedId": "WalttiTest", 270 | "fuzzyTripMatching": false, 271 | "backwardsDelayPropagationType": "ALWAYS" 272 | }, 273 | { 274 | "id": "walttitest-alerts", 275 | "type": "real-time-alerts", 276 | "frequency": "30s", 277 | "url": "http://digitransit-proxy:8080/out/lmj.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 278 | "feedId": "WalttiTest", 279 | "fuzzyTripMatching": false 280 | }, 281 | { 282 | "id": "tampere-trip-updates", 283 | "type": "stop-time-updater", 284 | "frequency": "60s", 285 | "url": "http://digitransit-proxy:8080/out/nysse.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 286 | "feedId": "tampere", 287 | "fuzzyTripMatching": false, 288 | "backwardsDelayPropagationType": "ALWAYS" 289 | }, 290 | { 291 | "id": "tampere-alerts", 292 | "type": "real-time-alerts", 293 | "frequency": "30s", 294 | "url": "http://digitransit-proxy:8080/out/nysse.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 295 | "feedId": "tampere", 296 | "fuzzyTripMatching": false 297 | } 298 | ] 299 | } 300 | -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | const gulp = require('gulp'); 2 | const rename = require('gulp-rename'); 3 | const dl = require('./task/Download'); 4 | const dlBlob = require('./task/DownloadDEMBlob'); 5 | const { setFeedIdTask } = require('./task/SetFeedId'); 6 | const { OBAFilterTask } = require('./task/OBAFilter'); 7 | const prepareFit = require('./task/PrepareFit'); 8 | const mapFit = require('./task/MapFit'); 9 | const { validateBlobSize } = require('./task/BlobValidation'); 10 | const { testOTPFile } = require('./task/OTPTest'); 11 | const { runOSMPreprocessing } = require('./task/OSMPreprocessing'); 12 | const seed = require('./task/Seed'); 13 | const { 14 | prepareRouterData, 15 | prepareRouterDataForStreetOnlyGraphBuild, 16 | prepareRouterDataForPrebuiltStreetGraphBuild, 17 | } = require('./task/PrepareRouterData'); 18 | const del = require('del'); 19 | const config = require('./config'); 20 | const { 21 | buildOTPGraphTask, 22 | buildOTPStreetOnlyGraphTask, 23 | } = require('./task/BuildOTPGraph'); 24 | const { renameGTFSFile } = require('./task/GTFSRename'); 25 | const { replaceGTFSFilesTask } = require('./task/GTFSReplace'); 26 | const { extractFilesTask, addFilesTask } = require('./task/ZipTask'); 27 | const { createDir } = require('./util'); 28 | const storageCleanup = require('./task/StorageCleanup'); 29 | 30 | const seedSourceDir = `${config.dataDir}/router-${config.router.id}`; // e.g. data/router-hsl 31 | 32 | const osmDlDir = `${config.dataDir}/downloads/osm`; 33 | const demDlDir = `${config.dataDir}/downloads/dem`; 34 | const gtfsDlDir = `${config.dataDir}/downloads/gtfs`; 35 | 36 | const osmDir = `${config.dataDir}/ready/osm`; 37 | const demDir = `${config.dataDir}/ready/dem`; 38 | const gtfsDir = `${config.dataDir}/ready/gtfs`; 39 | 40 | const gtfsSeedDir = `${config.dataDir}/seed`; 41 | const fitDir = `${config.dataDir}/fit`; 42 | const filterDir = `${config.dataDir}/filter`; 43 | const idDir = `${config.dataDir}/id`; 44 | const tmpIdDir = `${config.dataDir}/tmp-id`; 45 | const testGtfsDir = `${config.dataDir}/test/gtfs`; 46 | const tmpDir = `${config.dataDir}/tmp`; 47 | const tmpRenameDir = `${config.dataDir}/tmp-rename`; 48 | const renamedDir = `${config.dataDir}/renamed`; 49 | 50 | const noBuf = { buffer: false }; // options for gulp src 51 | 52 | /** 53 | * Download osm data 54 | */ 55 | gulp.task('osm:download', async cb => { 56 | if (!config.osm) { 57 | return Promise.resolve(); 58 | } 59 | createDir(osmDlDir); 60 | createDir(osmDir); 61 | await dl(config.osm, osmDlDir); 62 | cb(); 63 | }); 64 | 65 | gulp.task('osm:copyPreprocessingFiles', () => 66 | gulp 67 | .src(`${config.router.id}/osm-preprocessing/*.sh`, noBuf) 68 | .pipe(gulp.dest(`${config.dataDir}/${config.router.id}/osm-preprocessing`)), 69 | ); 70 | 71 | gulp.task( 72 | 'osm:update', 73 | gulp.series( 74 | 'osm:copyPreprocessingFiles', 75 | 'osm:download', 76 | () => 77 | gulp 78 | .src(`${osmDlDir}/*`, noBuf) 79 | .pipe(validateBlobSize()) 80 | .pipe( 81 | runOSMPreprocessing( 82 | `${config.dataDir}/${config.router.id}/osm-preprocessing`, 83 | ), 84 | ) 85 | .pipe(testOTPFile()) 86 | .pipe(gulp.dest(osmDir)), 87 | () => del(tmpDir), 88 | ), 89 | ); 90 | 91 | /** 92 | * Download and test new dem data 93 | */ 94 | gulp.task('dem:update', () => { 95 | if (!config.dem) { 96 | return Promise.resolve(); 97 | } 98 | createDir(demDlDir); 99 | createDir(demDir); 100 | return Promise.all(dlBlob(config.dem)).catch(() => { 101 | global.hasFailures = true; 102 | }); 103 | }); 104 | 105 | gulp.task('del:filter', () => del(filterDir)); 106 | gulp.task('del:fit', () => del(fitDir)); 107 | gulp.task('del:id', () => del(idDir)); 108 | 109 | /** 110 | * 1. download 111 | * 2. name zip as -gtfs.zip (in dir 'download') 112 | * 3. test zip with OpenTripPlanner 113 | * 4. copy to fit dir if test is succesful 114 | */ 115 | gulp.task( 116 | 'gtfs:dl', 117 | gulp.series( 118 | 'del:fit', 119 | cb => { 120 | dl(config.router.src, gtfsDlDir).then(() => { 121 | cb(); 122 | }); 123 | }, 124 | () => 125 | gulp 126 | .src(`${gtfsDlDir}/*`, noBuf) 127 | .pipe(renameGTFSFile()) 128 | .pipe(gulp.dest(renamedDir)) 129 | .pipe(replaceGTFSFilesTask(config.gtfsMap)) 130 | .pipe(gulp.dest(fitDir)), 131 | () => del([tmpRenameDir]), 132 | ), 133 | ); 134 | 135 | // Add feedId to gtfs files in id dir, and moves files to directory 'test/gtfs' 136 | gulp.task( 137 | 'gtfs:id', 138 | gulp.series( 139 | () => 140 | gulp 141 | .src(`${idDir}/*`, noBuf) 142 | .pipe(extractFilesTask(['feed_info.txt'])) 143 | .pipe(setFeedIdTask()) 144 | .pipe(addFilesTask(['feed_info.txt'])) 145 | .pipe(gulp.dest(testGtfsDir)), 146 | () => del(tmpDir), 147 | ), 148 | ); 149 | 150 | // Runs mapFit on gtfs files if fit is enabled, or just moves files to directory 'filter' 151 | gulp.task( 152 | 'gtfs:fit', 153 | config.router.src.some(src => src.fit) 154 | ? gulp.series( 155 | 'del:filter', 156 | () => prepareFit(config), 157 | () => 158 | gulp 159 | .src(`${fitDir}/*`, noBuf) 160 | .pipe(extractFilesTask(['stops.txt'])) 161 | .pipe(mapFit(config)) // modify backup of stops.txt 162 | .pipe(addFilesTask(['stops.txt'])) 163 | .pipe(gulp.dest(filterDir)), 164 | () => del(tmpDir), 165 | ) 166 | : () => gulp.src(`${fitDir}/*`, noBuf).pipe(gulp.dest(filterDir)), 167 | ); 168 | 169 | gulp.task('copyRules', () => 170 | gulp 171 | .src(`${config.router.id}/gtfs-rules/*`, noBuf) 172 | .pipe(gulp.dest(`${config.dataDir}/${config.router.id}/gtfs-rules`)), 173 | ); 174 | 175 | // Filter gtfs files and move result to directory 'id' 176 | gulp.task( 177 | 'gtfs:filter', 178 | config.router.src.some(src => src.rules) 179 | ? gulp.series( 180 | 'copyRules', 181 | 'del:id', 182 | () => 183 | gulp 184 | .src(`${filterDir}/*.zip`, noBuf) 185 | .pipe(extractFilesTask(config.passOBAfilter)) 186 | .pipe(OBAFilterTask(config.gtfsMap)) 187 | .pipe(addFilesTask(config.passOBAfilter)) 188 | .pipe(gulp.dest(idDir)), 189 | () => del(tmpDir), 190 | ) 191 | : () => gulp.src(`${filterDir}/*`, noBuf).pipe(gulp.dest(idDir)), 192 | ); 193 | 194 | // Test gtfs files and move result to directory 'ready/gtfs' 195 | gulp.task('gtfs:test', () => 196 | gulp 197 | .src(`${testGtfsDir}/*`, noBuf) 198 | .pipe(testOTPFile()) 199 | .pipe(gulp.dest(gtfsDir)), 200 | ); 201 | 202 | gulp.task( 203 | 'gtfs:update', 204 | gulp.series( 205 | 'gtfs:dl', 206 | 'gtfs:fit', 207 | 'gtfs:filter', 208 | 'gtfs:id', 209 | 'gtfs:test', 210 | () => del(tmpIdDir), 211 | ), 212 | ); 213 | 214 | // move listed packages from seed to ready 215 | gulp.task('gtfs:fallback', () => { 216 | const sources = global.failedFeeds 217 | .split(',') 218 | .map(feed => `${gtfsSeedDir}/${feed}-gtfs.zip`); 219 | return gulp.src(sources, noBuf).pipe(gulp.dest(gtfsDir)); 220 | }); 221 | 222 | gulp.task('gtfs:del', () => del([gtfsSeedDir, gtfsDir])); 223 | 224 | gulp.task( 225 | 'gtfs:seed', 226 | gulp.series('gtfs:del', () => 227 | gulp 228 | .src(`${seedSourceDir}/*-gtfs.zip`, noBuf) 229 | .pipe(gulp.dest(gtfsSeedDir)) 230 | .pipe(gulp.dest(gtfsDir)), 231 | ), 232 | ); 233 | 234 | gulp.task('osm:del', () => del(osmDir)); 235 | 236 | gulp.task( 237 | 'osm:seed', 238 | gulp.series('osm:del', () => 239 | gulp.src(`${seedSourceDir}/*.pbf`, noBuf).pipe(gulp.dest(osmDir)), 240 | ), 241 | ); 242 | 243 | gulp.task('dem:del', () => del(demDir)); 244 | 245 | gulp.task( 246 | 'dem:seed', 247 | gulp.series('dem:del', () => 248 | gulp.src(`${seedSourceDir}/*.tif`, noBuf).pipe(gulp.dest(demDir)), 249 | ), 250 | ); 251 | 252 | gulp.task('seed:cleanup', () => 253 | del([seedSourceDir, `${config.dataDir}/*.zip`]), 254 | ); 255 | 256 | /** 257 | * Seed DEM, GTFS & OSM data with data from a previous build to allow 258 | * continuous flow of data into production when one or more updated data files 259 | * are broken. The data is loaded from a storage that should persist between builds. 260 | */ 261 | gulp.task( 262 | 'seed', 263 | gulp.series( 264 | () => 265 | seed( 266 | config.storageDir, 267 | config.dataDir, 268 | config.router.id, 269 | process.env.SEED_TAG, 270 | ), 271 | 'dem:seed', 272 | 'osm:seed', 273 | 'gtfs:seed', 274 | 'seed:cleanup', 275 | ), 276 | ); 277 | 278 | gulp.task('router:del', () => del(`${config.dataDir}/build`)); 279 | 280 | gulp.task( 281 | 'router:copy', 282 | gulp.series('router:del', () => 283 | prepareRouterData(config.router).pipe( 284 | gulp.dest(`${config.dataDir}/build/${config.router.id}`), 285 | ), 286 | ), 287 | ); 288 | 289 | gulp.task( 290 | 'router:buildGraph', 291 | gulp.series('router:copy', () => buildOTPGraphTask(config.router)), 292 | ); 293 | 294 | gulp.task( 295 | 'router:copyForPrebuiltStreetGraphDataBuild', 296 | gulp.series('router:del', () => 297 | prepareRouterDataForPrebuiltStreetGraphBuild(config.router).pipe( 298 | gulp.dest(`${config.dataDir}/build/${config.router.id}`), 299 | ), 300 | ), 301 | ); 302 | 303 | gulp.task( 304 | 'router:buildWithPrebuiltStreetGraph', 305 | gulp.series('router:copyForPrebuiltStreetGraphDataBuild', () => 306 | buildOTPGraphTask(config.router), 307 | ), 308 | ); 309 | 310 | gulp.task( 311 | 'router:copyStreetOnlyGraphData', 312 | gulp.series('router:del', () => 313 | prepareRouterDataForStreetOnlyGraphBuild(config.router).pipe( 314 | gulp.dest(`${config.dataDir}/build/${config.router.id}`), 315 | ), 316 | ), 317 | ); 318 | 319 | gulp.task( 320 | 'router:buildStreetOnlyGraph', 321 | gulp.series('router:copyStreetOnlyGraphData', () => 322 | buildOTPStreetOnlyGraphTask(config.router), 323 | ), 324 | ); 325 | 326 | gulp.task('router:store', () => 327 | gulp 328 | .src(`${config.dataDir}/build/${config.router.id}/**/*`, noBuf) 329 | .pipe(gulp.dest(`${config.storageDir}/${global.storageDirName}/`)), 330 | ); 331 | 332 | gulp.task( 333 | 'router:storeForPrebuiltStreetGraphDataBuild', 334 | gulp.series( 335 | 'router:store', 336 | () => 337 | gulp 338 | .src(`${global.osmPrebuildDir}/report/*`, noBuf) 339 | .pipe( 340 | gulp.dest( 341 | `${config.storageDir}/${global.storageDirName}/street-report/`, 342 | ), 343 | ), 344 | () => 345 | gulp 346 | .src(`${global.osmPrebuildDir}/build.log`, noBuf) 347 | .pipe(rename('street-build.log')) 348 | .pipe(gulp.dest(`${config.storageDir}/${global.storageDirName}`)), 349 | ), 350 | ); 351 | 352 | gulp.task('storage:cleanup', () => 353 | storageCleanup(config.storageDir, config.router.id, process.env.SEED_TAG), 354 | ); 355 | 356 | gulp.task('storage:cleanupStreetOnlyGraphData', () => 357 | storageCleanup( 358 | config.storageDir, 359 | config.router.id, 360 | `osm-builds/${process.env.SEED_TAG}`, 361 | ), 362 | ); 363 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Build process for OpenTripPlanner-data-server and router specific OpenTripPlanner images 2 | 3 | [![Build](https://github.com/hsldevcom/OpenTripPlanner-data-container/workflows/Process%20v3%20push%20or%20pr/badge.svg?branch=v3)](https://github.com/HSLdevcom/OpenTripPlanner-data-container/actions) 4 | 5 | ## This project: 6 | 7 | Contains tools for fetching, building and deploying fresh opentripplanner data server and opentripplanner images 8 | for consumption by Digitransit maintained OTP version 2.x instances. 9 | 10 | ## Main components 11 | 12 | ### otp-data-builder 13 | 14 | The actual data builder application. This is a node.js application that fetches 15 | and processes new gtfs/osm data. It's build around gulp and all separate steps of 16 | databuilding process can also be called directly from the source tree. The only 17 | required external dependency is docker. Docker is used for launching external 18 | commands that do for example data manipulation. 19 | 20 | install gulp cli: 21 | `yarn global add gulp-cli` 22 | 23 | install app deps: 24 | `yarn` 25 | 26 | update osm data: 27 | `ROUTER_NAME=hsl gulp osm:update` 28 | 29 | download new gtfs data for waltti: 30 | `ROUTER_NAME=waltti gulp gtfs:dl` 31 | 32 | #### Configuration 33 | 34 | It is possible to change the behaviour of the data builder by defining environment variables. 35 | 36 | - `ROUTER_NAME` defines for which router the data gets updated for. 37 | - `DOCKER_USER` defines username for authenticating to docker hub. 38 | - `DOCKER_AUTH` defines password for authenticating to docker hub. 39 | - (Optional, default v3 and tag based on date) `DOCKER_TAG` defines what will be the updated docker tag of the data server images in the remote container registry. 40 | - (Optional, default hsldevcom) `ORG` defines what organization images belong to in the remote container registry. 41 | - (Optional, default v3) `SEED_TAG` defines what version of the data storage should be used for seeding. 42 | - (Optional, default v2) `OTP_TAG` defines what version of OTP is used for testing, building graphs and deploying a new OTP image (postfixed with router name). 43 | - (Optional, default v3) `TOOLS_TAG` defines what version of otp-data-tools image is used for testing. 44 | - (Optional, default dev) `BUILDER_TYPE` used as a postfix to slack bot name 45 | - (Optional) `SLACK_CHANNEL_ID` defines to which slack channel the messages are sent to 46 | - (Optional) `SLACK_ACCESS_TOKEN` bearer token for slack messaging 47 | - (Optional, default {}) `EXTRA_SRC` defines gtfs src values that should be overridden or completely new src that should be added with unique id. Example format: 48 | - `{"FOLI": {"url": "https://data.foli.fi/gtfs/gtfs.zip", "fit": false, "rules": ["router-waltti/gtfs-rules/waltti.rule"]}}` 49 | - You can remove a src by including `"remove": true`, `{"FOLI": {"remove": true}}` 50 | - (Optional, default {}) `EXTRA_UPDATERS` defines router-config.json updater values that should be overridden or completely new updater that should be added with unique id. Example format: 51 | - `{"turku-alerts": {"type": "real-time-alerts", "frequencySec": 30, "url": "https://foli-beta.nanona.fi/gtfs-rt/reittiopas", "feedId": "FOLI", "fuzzyTripMatching": true}}` 52 | - You can remove a src by including `"remove": true`, `{"turku-alerts": {"remove": true}}` 53 | - (Optional, default {}) `EXTRA_OSM` can redefine OSM source URLs. For example: `{"hsl": "https://tempserver.com/newhsl.pbf"}` 54 | - (Optional) `VERSION_CHECK` is a comma-separated list of feedIds from which the GTFS data's `feed_info.txt`'s file's `feed_version` field is parsed into a date object and it's checked if the data has been updated within the last 8 hours. If not, a message is sent to stdout (and slack, only monday-friday) to inform about usage of "old" data. 55 | - (Optional) `SKIPPED_SITES` defines a comma-separated list of sites from OTPQA tests that should be skipped. Example format: 56 | - `"turku.digitransit.fi,reittiopas.hsl.fi"` 57 | - (Optional) `DISABLE_BLOB_VALIDATION` should be included if blob (OSM) validation should be disabled temporarily. 58 | - (Optional) `NOSEED` should be included (together with DISABLE_BLOB_VALIDATION) when data loading for a new configuration is run first time and no seed image is available. 59 | - (Optional) `NOCLEANUP` can be used to disable removal of historical data in storage 60 | - (Optional) `JAVA_OPTS` Java parameters for running OTP 61 | - (Optional) `SPLIT_BUILD_TYPE` is an enum used to configure if the build should be split. The values are: 62 | - `ONLY_BUILD_STREET_GRAPH` to only build the street graph 63 | - `USE_PREBUILT_STREET_GRAPH` to use the prebuilt street graph to finish a complete graph build 64 | - All other values default to `NO_SPLIT_BUILD` which indicates that the build is run as normal 65 | - (Optional) `USE_SEEDED_OSM` skips OSM updating and uses existing seed version 66 | - (Optional) `SKIP_OSM_PREPROCESSING` skips OSM preprocessing even if an instruction file is defined 67 | - (Optional) `SKIP_OTP_TESTS` skips OTP tests 68 | 69 | ### Data processing steps 70 | 71 | - `seed` downloads previous data from storage (env variable SEED_TAG can be used to customize which storage location is used) 72 | and then extracts osm, dem, and gtfs data and places it in the `data/seed` and `data/ready` directories. 73 | The old data acts as backup in case fetching/validating new data fails. The command uses the zipped contents of the latest build that built a complete graph (from prebuilt data or from a normal build). 74 | 75 | - `dem:update` downloads required DEM information, after which data is copied to the `data/downloads/dem` directory. 76 | 77 | - `osm:update` downloads required OSM packages from configured locations, tests the files with OTP, 78 | and if the tests pass, data is copied to the `data/downloads/osm` directory. 79 | 80 | - `gtfs:update` 81 | 82 | - `gtfs:dl` downloads a GTFS package from a configured location and data is copied to the `data/fit` directory. The resulting zip file is named `.zip`. 83 | 84 | - `gtfs:fit` runs configured map fits. Copies data to the `data/filter` directory. 85 | 86 | - `gtfs:filter` runs configured filters. Copies data to the `data/id` directory. 87 | 88 | - `gtfs:id` sets the gtfs feed id to `` and copies data to the `data/test/gtfs` directory. 89 | 90 | - `gtfs:test` tests the file with OTP and if the test passes, data is copied to the `data/ready/gtfs` directory. 91 | 92 | - `router:buildGraph` 93 | 94 | - `router:copy` copies files needed for the build. 95 | - `buildOTPGraphTask(config.router)` builds a new graph with all the new data sets (and maybe seeded data sets if there were issues with new data). 96 | 97 | - `router:buildStreetOnlyGraph` 98 | 99 | - `router:copyStreetOnlyGraphData` copies files needed for the street only build. 100 | - `buildOTPStreetOnlyGraphTask(config.router)` builds a new street only graph with all the new data sets (and maybe seeded data sets if there were issues with new data). 101 | 102 | - `router:buildWithPrebuiltStreetGraph` 103 | 104 | - `router:copyForPrebuiltStreetGraphDataBuild` copies files needed for the build from prebuilt data. 105 | - `buildOTPGraphTask(config.router)` builds a new graph from prebuilt street only data with new gtfs data sets (and maybe seeded data sets if there were issues with new data). 106 | 107 | - `test.sh` runs the routing quality test bench defined in the `hsldevcom/OTPQA` repository. OTPQA test sets are associated with GTFS packages. 108 | If there are quality regressions, a comma separated list of failed GTFS feed identifiers is written to the local file `failed_feeds.txt`. 109 | 110 | - `router:store` stores the new data in storage (which can be a mounted storage volume). 111 | 112 | - `router:storeForPrebuiltStreetGraphDataBuild` stores the new data in storage (which can be a mounted storage volume). Also copies the `report` directory from the street only build to the output directory under the name `street-report`. 113 | 114 | - `deploy.sh` deploys a new opentripplanner-data-server image with the `DOCKER_TAG` env variable (default `v3`) postfixed with the router name, and 115 | pushes the image to Dockerhub.

116 | Normally, when the application is running as a container, the script `index.js` is run to execute all steps. 117 | The end result of the build is a data server image uploaded to dockerhub.

118 | Each data server image runs an http server listening to port `8080`. It serves a data bundle required for building a graph and a prebuilt graph. For example, in the HSL case: http://localhost:8080/router-hsl.zip and `graph-hsl-$OTPVERSION.zip`. The image 119 | does not include the data, the data needs to be mounted while running the container. 120 | 121 | - `deploy-otp.sh` tags an OTP image using the `OTP_TAG` env variable (default `v2`) postfixed with the router name and pushes the image to Dockerhub. 122 | This new OTP image will automatically use the graph and configuration from the storage location where the build's end result was stored at. 123 | 124 | - `storage:cleanup` keeps the 10 latest versions of the data in storage and removes the rest. 125 | 126 | - `storage:cleanupStreetOnlyGraphData` keeps the 10 latest versions of the street only build data in storage and removes the rest. 127 | 128 | #### Normal build 129 | 130 | 1. `seed` 131 | 2. `dem:update` 132 | 3. `osm:update` 133 | 4. `gtfs:update` 134 | - `gtfs:dl` 135 | - `gtfs:fit` 136 | - `gtfs:filter` 137 | - `gtfs:id` 138 | 5. `router:buildGraph` 139 | - `router:copy` 140 | - `buildOTPGraphTask(config.router)` 141 | 6. `test.sh` 142 | 7. `router:store` 143 | 8. `deploy.sh` 144 | 9. `deploy-otp.sh` 145 | 10. `storage:cleanup` 146 | 147 | #### Street only build 148 | 149 | 1. `seed` 150 | 2. `dem:update` 151 | 3. `osm:update` 152 | 4. `router:buildStreetOnlyGraph` 153 | - `router:copyStreetOnlyGraphData` 154 | - `buildOTPStreetOnlyGraphTask(config.router)` 155 | 5. `router:store` 156 | 6. `storage:cleanupStreetOnlyGraphData` 157 | 158 | #### Build from prebuilt street data 159 | 160 | 1. `seed` 161 | 2. `gtfs:update` 162 | - `gtfs:dl` 163 | - `gtfs:fit` 164 | - `gtfs:filter` 165 | - `gtfs:id` 166 | 3. `router:buildWithPrebuiltStreetGraph` 167 | - `router:copyForPrebuiltStreetGraphDataBuild` 168 | - `buildOTPGraphTask(config.router)` 169 | 4. `test.sh` 170 | 5. `router:storeForPrebuiltStreetGraphDataBuild` 171 | 6. `deploy.sh` 172 | 7. `deploy-otp.sh` 173 | 8. `storage:cleanup` 174 | 175 | ### otp-data-tools 176 | 177 | Contains tools, such as the OneBusAway gtfs filter, for gtfs manipulation. 178 | It uses the [opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli](https://registry.hub.docker.com/r/opentransitsoftwarefoundation/onebusaway-gtfs-transformer-cli) as the base image. 179 | These tools are packaged inside a docker container and are used during the data build process. 180 | 181 | #### OSM preprocessing 182 | 183 | OSM preprocessing is done if a bash script is defined for a specific config and a specific OSM file. 184 | See [hsl.sh](hsl/osm-preprocessing/hsl.sh) for an example. 185 | 186 | When creating OSM preprocessing instructions you should: 187 | 1. Name the bash file as follows: `.sh`. Valid file names can be e.g. `hsl.sh` or `southFinland.sh`. 188 | 2. Place the file in the `osm-preprocessing` directory of the config you want to use. 189 | 3. Make sure that the name of the output file is the same as the input file. The file has to be named `.pbf`, e.g. `hsl.pbf` or `southFinland.pbf`. 190 | 4. Make sure that you do not reuse input and output filenames in commands: 191 | - INCORRECT `osmfilter hsl.o5m -o=hsl.o5m ...` 192 | - CORRECT `osmfilter hsl.o5m -o=hsl2.o5m ...` 193 | 5. Test the script by running it locally and verifying that the output makes sense. 194 | -------------------------------------------------------------------------------- /waltti/router-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "routingDefaults": { 3 | "transferSlack": "1m30s", 4 | "waitReluctance": 0.95, 5 | "elevatorBoardTime": 60, 6 | "bicycle": { 7 | "boardCost": 120, 8 | "reluctance": 1.7, 9 | "optimization": "safest-streets" 10 | }, 11 | "car": { 12 | "reluctance": 10.0 13 | }, 14 | "walk": { 15 | "speed": 1.3, 16 | "reluctance": 1.75, 17 | "stairsReluctance": 1.2, 18 | "stairsTimeFactor": 2, 19 | "escalator": { 20 | "speed": 0.65 21 | }, 22 | "boardCost": 120 23 | }, 24 | "accessEgress": { 25 | "maxDuration": "1h" 26 | }, 27 | "maxDirectStreetDuration": "2h", 28 | "maxDirectStreetDurationForMode": { 29 | "walk": "90m" 30 | }, 31 | "maxJourneyDuration": "12h", 32 | "streetRoutingTimeout": "8s", 33 | "wheelchairAccessibility": { 34 | "stop": { 35 | "onlyConsiderAccessible": false, 36 | "unknownCost": 0, 37 | "inaccessibleCost": 100000 38 | }, 39 | "maxSlope": 0.125 40 | }, 41 | "itineraryFilters": { 42 | "transitGeneralizedCostLimit": { 43 | "costLimitFunction": "600 + 1.5x" 44 | }, 45 | "nonTransitGeneralizedCostLimit": "400 + 1.5x" 46 | }, 47 | "unpreferredCost": "1 + 0.4x", 48 | "unpreferred": { 49 | "routes": ["Lappeenranta:100", "Lappeenranta:101", "Lappeenranta:110"] 50 | } 51 | }, 52 | "gtfsApi": { 53 | "tracingTags": ["digitransit-subscription-id"] 54 | }, 55 | "transit": { 56 | "pagingSearchWindowAdjustments": ["8h", "4h", "4h", "4h", "4h"], 57 | "dynamicSearchWindow": { 58 | "minWindow": "1h" 59 | }, 60 | "maxNumberOfTransfers": 12, 61 | "transferCacheRequests": [ 62 | { 63 | "modes": "WALK", 64 | "walk": { 65 | "speed": 1.2, 66 | "reluctance": 1.8 67 | } 68 | }, 69 | { 70 | "modes": "WALK", 71 | "walk": { 72 | "speed": 1.2, 73 | "reluctance": 1.8 74 | }, 75 | "wheelchairAccessibility": { 76 | "enabled": true 77 | } 78 | }, 79 | { 80 | "modes": "WALK", 81 | "walk": { 82 | "speed": 1.67, 83 | "reluctance": 1.8 84 | } 85 | }, 86 | { 87 | "modes": "BICYCLE", 88 | "walk": { 89 | "speed": 1.2, 90 | "reluctance": 1.8 91 | }, 92 | "bicycle": { 93 | "speed": 5.55, 94 | "rental": { 95 | "useAvailabilityInformation": true 96 | } 97 | } 98 | }, 99 | { 100 | "modes": "BICYCLE", 101 | "walk": { 102 | "speed": 1.67, 103 | "reluctance": 1.8 104 | }, 105 | "bicycle": { 106 | "speed": 5.55, 107 | "rental": { 108 | "useAvailabilityInformation": true 109 | } 110 | } 111 | } 112 | ] 113 | }, 114 | "vectorTiles": { 115 | "attribution": "Digitransit data is licensed under CC BY 4.0.", 116 | "layers": [ 117 | { 118 | "name": "stops", 119 | "type": "Stop", 120 | "mapper": "Digitransit", 121 | "maxZoom": 20, 122 | "minZoom": 5, 123 | "cacheMaxSeconds": 43200 124 | }, 125 | { 126 | "name": "realtimeStops", 127 | "type": "Stop", 128 | "mapper": "DigitransitRealtime", 129 | "maxZoom": 20, 130 | "minZoom": 5, 131 | "cacheMaxSeconds": 60 132 | }, 133 | { 134 | "name": "stations", 135 | "type": "Station", 136 | "mapper": "Digitransit", 137 | "maxZoom": 20, 138 | "minZoom": 5, 139 | "cacheMaxSeconds": 43200 140 | }, 141 | { 142 | "name": "rentalStations", 143 | "type": "VehicleRentalStation", 144 | "mapper": "Digitransit", 145 | "maxZoom": 20, 146 | "minZoom": 5, 147 | "cacheMaxSeconds": 43200, 148 | "expansionFactor": 0.25 149 | }, 150 | { 151 | "name": "realtimeRentalStations", 152 | "type": "VehicleRentalStation", 153 | "mapper": "DigitransitRealtime", 154 | "maxZoom": 20, 155 | "minZoom": 5, 156 | "cacheMaxSeconds": 45, 157 | "expansionFactor": 0.25 158 | }, 159 | { 160 | "name": "vehicleParking", 161 | "type": "VehicleParking", 162 | "mapper": "Digitransit", 163 | "maxZoom": 20, 164 | "minZoom": 5, 165 | "cacheMaxSeconds": 43200, 166 | "expansionFactor": 0.25 167 | }, 168 | { 169 | "name": "vehicleParkingGroups", 170 | "type": "VehicleParkingGroup", 171 | "mapper": "Digitransit", 172 | "maxZoom": 20, 173 | "minZoom": 5, 174 | "cacheMaxSeconds": 43200, 175 | "expansionFactor": 0.25 176 | } 177 | ] 178 | }, 179 | "updaters": [ 180 | { 181 | "id": "digitraffic-trip-updates", 182 | "type": "stop-time-updater", 183 | "frequency": "60s", 184 | "url": "https://rata.digitraffic.fi/api/v1/trains/gtfs-rt-updates", 185 | "feedId": "digitraffic", 186 | "fuzzyTripMatching": false, 187 | "backwardsDelayPropagationType": "ALWAYS", 188 | "headers": { 189 | "digitraffic-user": "Digitransit/OTP" 190 | } 191 | }, 192 | { 193 | "id": "foli-trip-updates", 194 | "type": "stop-time-updater", 195 | "frequency": "60s", 196 | "url": "http://siri2gtfsrt:8080/FOLI", 197 | "feedId": "FOLI", 198 | "fuzzyTripMatching": true, 199 | "backwardsDelayPropagationType": "ALWAYS" 200 | }, 201 | { 202 | "id": "foli-alerts", 203 | "type": "real-time-alerts", 204 | "frequency": "30s", 205 | "url": "http://digitransit-proxy:8080/out/data.foli.fi/gtfs-rt/reittiopas", 206 | "feedId": "FOLI", 207 | "fuzzyTripMatching": false 208 | }, 209 | { 210 | "id": "oulu-trip-updates", 211 | "type": "stop-time-updater", 212 | "frequency": "60s", 213 | "url": "http://digitransit-proxy:8080/out/oulu.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 214 | "feedId": "OULU", 215 | "fuzzyTripMatching": false, 216 | "backwardsDelayPropagationType": "ALWAYS" 217 | }, 218 | { 219 | "id": "oulu-alerts", 220 | "type": "real-time-alerts", 221 | "frequency": "30s", 222 | "url": "http://digitransit-proxy:8080/out/oulu.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 223 | "feedId": "OULU", 224 | "fuzzyTripMatching": false 225 | }, 226 | { 227 | "id": "kuopio-trip-updates", 228 | "type": "stop-time-updater", 229 | "frequency": "60s", 230 | "url": "http://digitransit-proxy:8080/out/vilkku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 231 | "feedId": "Kuopio", 232 | "fuzzyTripMatching": false, 233 | "backwardsDelayPropagationType": "ALWAYS" 234 | }, 235 | { 236 | "id": "joensuu-trip-updates", 237 | "type": "stop-time-updater", 238 | "frequency": "60s", 239 | "url": "http://digitransit-proxy:8080/out/jojo.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 240 | "feedId": "Joensuu", 241 | "fuzzyTripMatching": false, 242 | "backwardsDelayPropagationType": "ALWAYS" 243 | }, 244 | { 245 | "id": "joensuu-alerts", 246 | "type": "real-time-alerts", 247 | "frequency": "30s", 248 | "url": "http://digitransit-proxy:8080/out/jojo.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 249 | "feedId": "Joensuu", 250 | "fuzzyTripMatching": false 251 | }, 252 | { 253 | "id": "lappeenranta-trip-updates", 254 | "type": "stop-time-updater", 255 | "frequency": "60s", 256 | "url": "http://digitransit-proxy:8080/out/lappeenranta.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 257 | "feedId": "Lappeenranta", 258 | "fuzzyTripMatching": false, 259 | "backwardsDelayPropagationType": "ALWAYS" 260 | }, 261 | { 262 | "id": "lappeenranta-alerts", 263 | "type": "real-time-alerts", 264 | "frequency": "30s", 265 | "url": "http://digitransit-proxy:8080/out/lappeenranta.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 266 | "feedId": "Lappeenranta", 267 | "fuzzyTripMatching": false 268 | }, 269 | { 270 | "id": "tampere-trip-updates", 271 | "type": "stop-time-updater", 272 | "frequency": "60s", 273 | "url": "https://gtfsrt.blob.core.windows.net/tampere/tripupdate", 274 | "feedId": "tampere", 275 | "fuzzyTripMatching": false, 276 | "backwardsDelayPropagationType": "ALWAYS" 277 | }, 278 | { 279 | "id": "tampere-alerts", 280 | "type": "real-time-alerts", 281 | "frequency": "30s", 282 | "url": "http://digitransit-proxy:8080/out/tre.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 283 | "feedId": "tampere", 284 | "fuzzyTripMatching": false 285 | }, 286 | { 287 | "id": "linkki-trip-updates", 288 | "type": "stop-time-updater", 289 | "frequency": "60s", 290 | "url": "http://digitransit-proxy:8080/out/linkki.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 291 | "feedId": "LINKKI", 292 | "fuzzyTripMatching": false, 293 | "backwardsDelayPropagationType": "ALWAYS" 294 | }, 295 | { 296 | "id": "linkki-alerts", 297 | "type": "real-time-alerts", 298 | "frequency": "30s", 299 | "url": "http://digitransit-proxy:8080/out/linkki.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 300 | "feedId": "LINKKI", 301 | "fuzzyTripMatching": false 302 | }, 303 | { 304 | "id": "lahti-alerts", 305 | "type": "real-time-alerts", 306 | "frequency": "30s", 307 | "url": "http://digitransit-proxy:8080/out/lsl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 308 | "feedId": "Lahti", 309 | "fuzzyTripMatching": false 310 | }, 311 | { 312 | "id": "lahti-trip-updates", 313 | "type": "stop-time-updater", 314 | "frequency": "60s", 315 | "url": "http://digitransit-proxy:8080/out/lsl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 316 | "feedId": "Lahti", 317 | "fuzzyTripMatching": false, 318 | "backwardsDelayPropagationType": "ALWAYS" 319 | }, 320 | { 321 | "id": "kuopio-alerts", 322 | "type": "real-time-alerts", 323 | "frequency": "30s", 324 | "url": "http://digitransit-proxy:8080/out/vilkku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 325 | "feedId": "Kuopio", 326 | "fuzzyTripMatching": false 327 | }, 328 | { 329 | "id": "hameenlinna-trip-updates", 330 | "type": "stop-time-updater", 331 | "frequency": "60s", 332 | "url": "http://digitransit-proxy:8080/out/hameenlinna.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 333 | "feedId": "Hameenlinna", 334 | "fuzzyTripMatching": false, 335 | "backwardsDelayPropagationType": "ALWAYS" 336 | }, 337 | { 338 | "id": "hameenlinna-alerts", 339 | "type": "real-time-alerts", 340 | "frequency": "30s", 341 | "url": "http://digitransit-proxy:8080/out/hameenlinna.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 342 | "feedId": "Hameenlinna", 343 | "fuzzyTripMatching": false 344 | }, 345 | { 346 | "id": "mikkeli-trip-updates", 347 | "type": "stop-time-updater", 348 | "frequency": "60s", 349 | "url": "http://digitransit-proxy:8080/out/mikkeli.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 350 | "feedId": "Mikkeli", 351 | "fuzzyTripMatching": false, 352 | "backwardsDelayPropagationType": "ALWAYS" 353 | }, 354 | { 355 | "id": "mikkeli-alerts", 356 | "type": "real-time-alerts", 357 | "frequency": "30s", 358 | "url": "http://digitransit-proxy:8080/out/mikkeli.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 359 | "feedId": "Mikkeli", 360 | "fuzzyTripMatching": false 361 | }, 362 | { 363 | "id": "vaasa-trip-updates", 364 | "type": "stop-time-updater", 365 | "frequency": "60s", 366 | "url": "http://digitransit-proxy:8080/out/lifti.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 367 | "feedId": "Vaasa", 368 | "fuzzyTripMatching": false, 369 | "backwardsDelayPropagationType": "ALWAYS" 370 | }, 371 | { 372 | "id": "vaasa-alerts", 373 | "type": "real-time-alerts", 374 | "frequency": "30s", 375 | "url": "http://digitransit-proxy:8080/out/lifti.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 376 | "feedId": "Vaasa", 377 | "fuzzyTripMatching": false 378 | }, 379 | { 380 | "id": "kouvola-trip-updates", 381 | "type": "stop-time-updater", 382 | "frequency": "60s", 383 | "url": "http://digitransit-proxy:8080/out/koutsi.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 384 | "feedId": "Kouvola", 385 | "fuzzyTripMatching": false, 386 | "backwardsDelayPropagationType": "ALWAYS" 387 | }, 388 | { 389 | "id": "kouvola-alerts", 390 | "type": "real-time-alerts", 391 | "frequency": "30s", 392 | "url": "http://digitransit-proxy:8080/out/koutsi.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 393 | "feedId": "Kouvola", 394 | "fuzzyTripMatching": false 395 | }, 396 | { 397 | "id": "kotka-trip-updates", 398 | "type": "stop-time-updater", 399 | "frequency": "60s", 400 | "url": "http://digitransit-proxy:8080/out/jonnejaminne.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 401 | "feedId": "Kotka", 402 | "fuzzyTripMatching": false, 403 | "backwardsDelayPropagationType": "ALWAYS" 404 | }, 405 | { 406 | "id": "kotka-alerts", 407 | "type": "real-time-alerts", 408 | "frequency": "30s", 409 | "url": "http://digitransit-proxy:8080/out/jonnejaminne.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 410 | "feedId": "Kotka", 411 | "fuzzyTripMatching": false 412 | }, 413 | { 414 | "id": "rovaniemi-trip-updates", 415 | "type": "stop-time-updater", 416 | "frequency": "60s", 417 | "url": "http://digitransit-proxy:8080/out/linkkari.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 418 | "feedId": "Rovaniemi", 419 | "fuzzyTripMatching": false, 420 | "backwardsDelayPropagationType": "ALWAYS" 421 | }, 422 | { 423 | "id": "rovaniemi-alerts", 424 | "type": "real-time-alerts", 425 | "frequency": "30s", 426 | "url": "http://digitransit-proxy:8080/out/linkkari.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 427 | "feedId": "Rovaniemi", 428 | "fuzzyTripMatching": false 429 | }, 430 | { 431 | "id": "pori-trip-updates", 432 | "type": "stop-time-updater", 433 | "frequency": "60s", 434 | "url": "http://digitransit-proxy:8080/out/pjl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 435 | "feedId": "Pori", 436 | "fuzzyTripMatching": false, 437 | "backwardsDelayPropagationType": "ALWAYS" 438 | }, 439 | { 440 | "id": "pori-alerts", 441 | "type": "real-time-alerts", 442 | "frequency": "30s", 443 | "url": "http://digitransit-proxy:8080/out/pjl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 444 | "feedId": "Pori", 445 | "fuzzyTripMatching": false 446 | }, 447 | { 448 | "id": "lappeenranta-bike-rental", 449 | "type": "vehicle-rental", 450 | "sourceType": "gbfs", 451 | "frequency": "30s", 452 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_lappeenranta/gbfs.json", 453 | "overloadingAllowed": true, 454 | "rentalPickupTypes": ["station"] 455 | }, 456 | { 457 | "id": "kotka-bike-rental", 458 | "type": "vehicle-rental", 459 | "sourceType": "gbfs", 460 | "frequency": "30s", 461 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_kotka/gbfs.json", 462 | "overloadingAllowed": true, 463 | "rentalPickupTypes": ["station"] 464 | }, 465 | { 466 | "id": "kouvola-bike-rental", 467 | "type": "vehicle-rental", 468 | "sourceType": "gbfs", 469 | "frequency": "30s", 470 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_kouvola/gbfs.json", 471 | "overloadingAllowed": true 472 | }, 473 | { 474 | "id": "turku-bike-rental", 475 | "type": "vehicle-rental", 476 | "sourceType": "gbfs", 477 | "frequency": "30s", 478 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_turku/gbfs.json", 479 | "overloadingAllowed": true 480 | }, 481 | { 482 | "id": "vilkku-bike-rental", 483 | "network": "freebike_kuopio", 484 | "type": "vehicle-rental", 485 | "sourceType": "gbfs", 486 | "frequency": "30s", 487 | "url": "http://digitransit-proxy:8080/out/tkhskuopiostrg.blob.core.windows.net/gbfs/gbfs.json", 488 | "overloadingAllowed": true 489 | }, 490 | { 491 | "id": "mankeli-bike-rental", 492 | "network": "freebike_lahti", 493 | "type": "vehicle-rental", 494 | "sourceType": "gbfs", 495 | "frequency": "60s", 496 | "url": "http://digitransit-proxy:8080/out/tkhslahtistorage.blob.core.windows.net/gbfs/gbfs.json", 497 | "overloadingAllowed": true, 498 | "rentalPickupTypes": ["station"] 499 | }, 500 | { 501 | "id": "liipi", 502 | "type": "vehicle-parking", 503 | "sourceType": "liipi", 504 | "feedId": "liipi", 505 | "timeZone": "Europe/Helsinki", 506 | "facilitiesFrequencySec": 3600, 507 | "facilitiesUrl": "https://parking.fintraffic.fi/api/v1/facilities.json?limit=-1", 508 | "utilizationsFrequencySec": 600, 509 | "utilizationsUrl": "https://parking.fintraffic.fi/api/v1/utilizations.json?limit=-1", 510 | "hubsUrl": "https://parking.fintraffic.fi/api/v1/hubs.json?limit=-1" 511 | }, 512 | { 513 | "id": "raasepori-alerts", 514 | "type": "real-time-alerts", 515 | "frequency": "30s", 516 | "url": "http://digitransit-proxy:8080/out/bosse.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 517 | "feedId": "Raasepori", 518 | "fuzzyTripMatching": false 519 | }, 520 | { 521 | "id": "tampere-bike-rental", 522 | "network": "inurba_tampere", 523 | "type": "vehicle-rental", 524 | "sourceType": "gbfs", 525 | "frequency": "30s", 526 | "url": "https://gbfs.urbansharing.com/tampere.onurbansharing.com/gbfs.json", 527 | "overloadingAllowed": true 528 | }, 529 | { 530 | "id": "salo-trip-updates", 531 | "type": "stop-time-updater", 532 | "frequency": "60s", 533 | "url": "http://digitransit-proxy:8080/out/paikku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 534 | "feedId": "Salo", 535 | "fuzzyTripMatching": false, 536 | "backwardsDelayPropagationType": "ALWAYS" 537 | }, 538 | { 539 | "id": "salo-alerts", 540 | "type": "real-time-alerts", 541 | "frequency": "30s", 542 | "url": "http://digitransit-proxy:8080/out/paikku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 543 | "feedId": "Salo", 544 | "fuzzyTripMatching": false 545 | }, 546 | { 547 | "id": "kajaani-trip-updates", 548 | "type": "stop-time-updater", 549 | "frequency": "60s", 550 | "url": "http://digitransit-proxy:8080/out/kajaani.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 551 | "feedId": "Kajaani", 552 | "fuzzyTripMatching": false, 553 | "backwardsDelayPropagationType": "ALWAYS" 554 | }, 555 | { 556 | "id": "kajaani-alerts", 557 | "type": "real-time-alerts", 558 | "frequency": "30s", 559 | "url": "http://digitransit-proxy:8080/out/kajaani.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 560 | "feedId": "Kajaani", 561 | "fuzzyTripMatching": false 562 | } 563 | ] 564 | } 565 | -------------------------------------------------------------------------------- /finland/router-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "routingDefaults": { 3 | "transferSlack": "1m30s", 4 | "waitReluctance": 0.99, 5 | "elevatorBoardTime": 60, 6 | "bicycle": { 7 | "boardCost": 120, 8 | "reluctance": 1.7, 9 | "optimization": "safest-streets" 10 | }, 11 | "car": { 12 | "reluctance": 10.0, 13 | "boardCost": 600 14 | }, 15 | "walk": { 16 | "speed": 1.3, 17 | "reluctance": 1.75, 18 | "stairsReluctance": 1.2, 19 | "stairsTimeFactor": 2, 20 | "escalator": { 21 | "speed": 0.65 22 | }, 23 | "boardCost": 120 24 | }, 25 | "accessEgress": { 26 | "maxDuration": "1h", 27 | "maxDurationForMode": { 28 | "CAR_TO_PARK": "2h", 29 | "BIKE": "2h", 30 | "CAR": "3h" 31 | }, 32 | "maxStopCountForMode": { 33 | "BIKE": 0, 34 | "CAR": 0 35 | } 36 | }, 37 | "maxDirectStreetDuration": "100h", 38 | "maxDirectStreetDurationForMode": { 39 | "walk": "90m" 40 | }, 41 | "maxJourneyDuration": "24h", 42 | "streetRoutingTimeout": "9s", 43 | "wheelchairAccessibility": { 44 | "stop": { 45 | "onlyConsiderAccessible": false, 46 | "unknownCost": 0, 47 | "inaccessibleCost": 100000 48 | }, 49 | "maxSlope": 0.125 50 | }, 51 | "itineraryFilters": { 52 | "transitGeneralizedCostLimit": { 53 | "costLimitFunction": "600 + 1.5x" 54 | }, 55 | "nonTransitGeneralizedCostLimit": "400 + 1.5x" 56 | }, 57 | "boardSlackForMode": { 58 | "AIRPLANE": "2700s" 59 | }, 60 | "alightSlackForMode": { 61 | "AIRPLANE": "1200s" 62 | } 63 | }, 64 | "gtfsApi": { 65 | "tracingTags": ["digitransit-subscription-id"] 66 | }, 67 | "flex": { 68 | "maxTransferDuration": "1m", 69 | "maxFlexTripDuration": "3h", 70 | "maxAccessWalkDuration": "5m", 71 | "maxEgressWalkDuration": "5m" 72 | }, 73 | "transit": { 74 | "pagingSearchWindowAdjustments": ["8h", "4h", "4h", "4h", "4h"], 75 | "dynamicSearchWindow": { 76 | "minWindow": "3h" 77 | }, 78 | "transferCacheRequests": [ 79 | { 80 | "modes": "WALK", 81 | "walk": { 82 | "speed": 1.2, 83 | "reluctance": 1.8 84 | } 85 | }, 86 | { 87 | "modes": "WALK", 88 | "walk": { 89 | "speed": 1.2, 90 | "reluctance": 1.8 91 | }, 92 | "wheelchairAccessibility": { 93 | "enabled": true 94 | } 95 | }, 96 | { 97 | "modes": "WALK", 98 | "walk": { 99 | "speed": 1.67, 100 | "reluctance": 1.8 101 | } 102 | }, 103 | { 104 | "modes": "BICYCLE", 105 | "walk": { 106 | "speed": 1.2, 107 | "reluctance": 1.8 108 | }, 109 | "bicycle": { 110 | "speed": 5.55, 111 | "rental": { 112 | "useAvailabilityInformation": true 113 | } 114 | } 115 | }, 116 | { 117 | "modes": "BICYCLE", 118 | "walk": { 119 | "speed": 1.67, 120 | "reluctance": 1.8 121 | }, 122 | "bicycle": { 123 | "speed": 5.55, 124 | "rental": { 125 | "useAvailabilityInformation": true 126 | } 127 | } 128 | }, 129 | { 130 | "modes": "CAR", 131 | "walk": { 132 | "speed": 1.3, 133 | "reluctance": 1.75 134 | } 135 | } 136 | ] 137 | }, 138 | "vectorTiles": { 139 | "attribution": "Digitransit data is licensed under CC BY 4.0.", 140 | "layers": [ 141 | { 142 | "name": "stops", 143 | "type": "Stop", 144 | "mapper": "Digitransit", 145 | "maxZoom": 20, 146 | "minZoom": 5, 147 | "cacheMaxSeconds": 43200 148 | }, 149 | { 150 | "name": "realtimeStops", 151 | "type": "Stop", 152 | "mapper": "DigitransitRealtime", 153 | "maxZoom": 20, 154 | "minZoom": 5, 155 | "cacheMaxSeconds": 60 156 | }, 157 | { 158 | "name": "stations", 159 | "type": "Station", 160 | "mapper": "Digitransit", 161 | "maxZoom": 20, 162 | "minZoom": 5, 163 | "cacheMaxSeconds": 43200 164 | }, 165 | { 166 | "name": "rentalStations", 167 | "type": "VehicleRentalStation", 168 | "mapper": "Digitransit", 169 | "maxZoom": 20, 170 | "minZoom": 5, 171 | "cacheMaxSeconds": 43200, 172 | "expansionFactor": 0.25 173 | }, 174 | { 175 | "name": "realtimeRentalStations", 176 | "type": "VehicleRentalStation", 177 | "mapper": "DigitransitRealtime", 178 | "maxZoom": 20, 179 | "minZoom": 5, 180 | "cacheMaxSeconds": 45, 181 | "expansionFactor": 0.25 182 | }, 183 | { 184 | "name": "realtimeRentalVehicles", 185 | "type": "VehicleRentalVehicle", 186 | "mapper": "DigitransitRealtime", 187 | "maxZoom": 20, 188 | "minZoom": 5, 189 | "cacheMaxSeconds": 45, 190 | "expansionFactor": 0.25 191 | }, 192 | { 193 | "name": "vehicleParking", 194 | "type": "VehicleParking", 195 | "mapper": "Digitransit", 196 | "maxZoom": 20, 197 | "minZoom": 5, 198 | "cacheMaxSeconds": 43200, 199 | "expansionFactor": 0.25 200 | }, 201 | { 202 | "name": "vehicleParkingGroups", 203 | "type": "VehicleParkingGroup", 204 | "mapper": "Digitransit", 205 | "maxZoom": 20, 206 | "minZoom": 5, 207 | "cacheMaxSeconds": 43200, 208 | "expansionFactor": 0.25 209 | } 210 | ] 211 | }, 212 | "updaters": [ 213 | { 214 | "id": "hsl-trip-updates", 215 | "type": "mqtt-gtfs-rt-updater", 216 | "url": "tcp://pred.rt.hsl.fi", 217 | "topic": "gtfsrt/v2/fi/hsl/tu", 218 | "feedId": "HSL", 219 | "fuzzyTripMatching": true, 220 | "backwardsDelayPropagationType": "ALWAYS" 221 | }, 222 | { 223 | "id": "hsl-alerts", 224 | "type": "real-time-alerts", 225 | "frequency": "30s", 226 | "url": "https://realtime.hsl.fi/realtime/service-alerts/v2/hsl", 227 | "feedId": "HSL", 228 | "fuzzyTripMatching": true 229 | }, 230 | { 231 | "type": "vehicle-positions", 232 | "url": "https://realtime.hsl.fi/realtime/vehicle-positions/v2/hsl", 233 | "feedId": "HSL", 234 | "frequency": "30s", 235 | "fuzzyTripMatching": true, 236 | "features": ["stop-position", "occupancy"] 237 | }, 238 | { 239 | "id": "digitraffic-trip-updates", 240 | "type": "stop-time-updater", 241 | "frequency": "60s", 242 | "url": "https://rata.digitraffic.fi/api/v1/trains/gtfs-rt-updates", 243 | "feedId": "digitraffic", 244 | "fuzzyTripMatching": false, 245 | "backwardsDelayPropagationType": "ALWAYS", 246 | "headers": { 247 | "digitraffic-user": "Digitransit/OTP" 248 | } 249 | }, 250 | { 251 | "id": "oulu-trip-updates", 252 | "type": "stop-time-updater", 253 | "frequency": "60s", 254 | "url": "http://digitransit-proxy:8080/out/oulu.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 255 | "feedId": "OULU", 256 | "fuzzyTripMatching": false, 257 | "backwardsDelayPropagationType": "ALWAYS" 258 | }, 259 | { 260 | "id": "oulu-alerts", 261 | "type": "real-time-alerts", 262 | "frequency": "30s", 263 | "url": "http://digitransit-proxy:8080/out/oulu.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 264 | "feedId": "OULU", 265 | "fuzzyTripMatching": false 266 | }, 267 | { 268 | "id": "tampere-trip-updates", 269 | "type": "stop-time-updater", 270 | "frequency": "60s", 271 | "url": "https://gtfsrt.blob.core.windows.net/tampere/tripupdate", 272 | "feedId": "tampere", 273 | "fuzzyTripMatching": false, 274 | "backwardsDelayPropagationType": "ALWAYS" 275 | }, 276 | { 277 | "id": "tampere-alerts", 278 | "type": "real-time-alerts", 279 | "frequency": "30s", 280 | "url": "http://digitransit-proxy:8080/out/tre.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 281 | "feedId": "tampere", 282 | "fuzzyTripMatching": false 283 | }, 284 | { 285 | "id": "linkki-trip-updates", 286 | "type": "stop-time-updater", 287 | "frequency": "60s", 288 | "url": "http://digitransit-proxy:8080/out/linkki.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 289 | "feedId": "LINKKI", 290 | "fuzzyTripMatching": false, 291 | "backwardsDelayPropagationType": "ALWAYS" 292 | }, 293 | { 294 | "id": "linkki-alerts", 295 | "type": "real-time-alerts", 296 | "frequency": "30s", 297 | "url": "http://digitransit-proxy:8080/out/linkki.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 298 | "feedId": "LINKKI", 299 | "fuzzyTripMatching": false 300 | }, 301 | { 302 | "id": "foli-trip-updates", 303 | "type": "stop-time-updater", 304 | "frequency": "60s", 305 | "url": "http://siri2gtfsrt:8080/FOLI", 306 | "feedId": "FOLI", 307 | "fuzzyTripMatching": true, 308 | "backwardsDelayPropagationType": "ALWAYS" 309 | }, 310 | { 311 | "id": "foli-alerts", 312 | "type": "real-time-alerts", 313 | "frequency": "30s", 314 | "url": "http://digitransit-proxy:8080/out/data.foli.fi/gtfs-rt/reittiopas", 315 | "feedId": "FOLI", 316 | "fuzzyTripMatching": false 317 | }, 318 | { 319 | "id": "kuopio-trip-updates", 320 | "type": "stop-time-updater", 321 | "frequency": "60s", 322 | "url": "http://digitransit-proxy:8080/out/vilkku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 323 | "feedId": "Kuopio", 324 | "fuzzyTripMatching": false, 325 | "backwardsDelayPropagationType": "ALWAYS" 326 | }, 327 | { 328 | "id": "joensuu-trip-updates", 329 | "type": "stop-time-updater", 330 | "frequency": "60s", 331 | "url": "http://digitransit-proxy:8080/out/jojo.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 332 | "feedId": "Joensuu", 333 | "fuzzyTripMatching": false, 334 | "backwardsDelayPropagationType": "ALWAYS" 335 | }, 336 | { 337 | "id": "joensuu-alerts", 338 | "type": "real-time-alerts", 339 | "frequency": "30s", 340 | "url": "http://digitransit-proxy:8080/out/jojo.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 341 | "feedId": "Joensuu", 342 | "fuzzyTripMatching": false 343 | }, 344 | { 345 | "id": "lappeenranta-trip-updates", 346 | "type": "stop-time-updater", 347 | "frequency": "60s", 348 | "url": "http://digitransit-proxy:8080/out/lappeenranta.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 349 | "feedId": "Lappeenranta", 350 | "fuzzyTripMatching": false, 351 | "backwardsDelayPropagationType": "ALWAYS" 352 | }, 353 | { 354 | "id": "lappeenranta-alerts", 355 | "type": "real-time-alerts", 356 | "frequency": "30s", 357 | "url": "http://digitransit-proxy:8080/out/lappeenranta.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 358 | "feedId": "Lappeenranta", 359 | "fuzzyTripMatching": false 360 | }, 361 | { 362 | "id": "lahti-alerts", 363 | "type": "real-time-alerts", 364 | "frequency": "30s", 365 | "url": "http://digitransit-proxy:8080/out/lsl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 366 | "feedId": "Lahti", 367 | "fuzzyTripMatching": false 368 | }, 369 | { 370 | "id": "lahti-trip-updates", 371 | "type": "stop-time-updater", 372 | "frequency": "60s", 373 | "url": "http://digitransit-proxy:8080/out/lsl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 374 | "feedId": "Lahti", 375 | "fuzzyTripMatching": false, 376 | "backwardsDelayPropagationType": "ALWAYS" 377 | }, 378 | { 379 | "id": "kuopio-alerts", 380 | "type": "real-time-alerts", 381 | "frequency": "30s", 382 | "url": "http://digitransit-proxy:8080/out/vilkku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 383 | "feedId": "Kuopio", 384 | "fuzzyTripMatching": false 385 | }, 386 | { 387 | "id": "hameenlinna-trip-updates", 388 | "type": "stop-time-updater", 389 | "frequency": "60s", 390 | "url": "http://digitransit-proxy:8080/out/hameenlinna.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 391 | "feedId": "Hameenlinna", 392 | "fuzzyTripMatching": false, 393 | "backwardsDelayPropagationType": "ALWAYS" 394 | }, 395 | { 396 | "id": "hameenlinna-alerts", 397 | "type": "real-time-alerts", 398 | "frequency": "30s", 399 | "url": "http://digitransit-proxy:8080/out/hameenlinna.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 400 | "feedId": "Hameenlinna", 401 | "fuzzyTripMatching": false 402 | }, 403 | { 404 | "id": "mikkeli-trip-updates", 405 | "type": "stop-time-updater", 406 | "frequency": "60s", 407 | "url": "http://digitransit-proxy:8080/out/mikkeli.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 408 | "feedId": "Mikkeli", 409 | "fuzzyTripMatching": false, 410 | "backwardsDelayPropagationType": "ALWAYS" 411 | }, 412 | { 413 | "id": "mikkeli-alerts", 414 | "type": "real-time-alerts", 415 | "frequency": "30s", 416 | "url": "http://digitransit-proxy:8080/out/mikkeli.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 417 | "feedId": "Mikkeli", 418 | "fuzzyTripMatching": false 419 | }, 420 | { 421 | "id": "vaasa-trip-updates", 422 | "type": "stop-time-updater", 423 | "frequency": "60s", 424 | "url": "http://digitransit-proxy:8080/out/lifti.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 425 | "feedId": "Vaasa", 426 | "fuzzyTripMatching": false, 427 | "backwardsDelayPropagationType": "ALWAYS" 428 | }, 429 | { 430 | "id": "vaasa-alerts", 431 | "type": "real-time-alerts", 432 | "frequency": "30s", 433 | "url": "http://digitransit-proxy:8080/out/lifti.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 434 | "feedId": "Vaasa", 435 | "fuzzyTripMatching": false 436 | }, 437 | { 438 | "id": "kouvola-trip-updates", 439 | "type": "stop-time-updater", 440 | "frequency": "60s", 441 | "url": "http://digitransit-proxy:8080/out/koutsi.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 442 | "feedId": "Kouvola", 443 | "fuzzyTripMatching": false, 444 | "backwardsDelayPropagationType": "ALWAYS" 445 | }, 446 | { 447 | "id": "kouvola-alerts", 448 | "type": "real-time-alerts", 449 | "frequency": "30s", 450 | "url": "http://digitransit-proxy:8080/out/koutsi.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 451 | "feedId": "Kouvola", 452 | "fuzzyTripMatching": false 453 | }, 454 | { 455 | "id": "kotka-trip-updates", 456 | "type": "stop-time-updater", 457 | "frequency": "60s", 458 | "url": "http://digitransit-proxy:8080/out/jonnejaminne.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 459 | "feedId": "Kotka", 460 | "fuzzyTripMatching": false, 461 | "backwardsDelayPropagationType": "ALWAYS" 462 | }, 463 | { 464 | "id": "kotka-alerts", 465 | "type": "real-time-alerts", 466 | "frequency": "30s", 467 | "url": "http://digitransit-proxy:8080/out/jonnejaminne.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 468 | "feedId": "Kotka", 469 | "fuzzyTripMatching": false 470 | }, 471 | { 472 | "id": "rovaniemi-trip-updates", 473 | "type": "stop-time-updater", 474 | "frequency": "60s", 475 | "url": "http://digitransit-proxy:8080/out/linkkari.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 476 | "feedId": "Rovaniemi", 477 | "fuzzyTripMatching": false, 478 | "backwardsDelayPropagationType": "ALWAYS" 479 | }, 480 | { 481 | "id": "rovaniemi-alerts", 482 | "type": "real-time-alerts", 483 | "frequency": "30s", 484 | "url": "http://digitransit-proxy:8080/out/linkkari.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 485 | "feedId": "Rovaniemi", 486 | "fuzzyTripMatching": false 487 | }, 488 | { 489 | "id": "kajaani-trip-updates", 490 | "type": "stop-time-updater", 491 | "frequency": "60s", 492 | "url": "http://digitransit-proxy:8080/out/kajaani.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 493 | "feedId": "Kajaani", 494 | "fuzzyTripMatching": false, 495 | "backwardsDelayPropagationType": "ALWAYS" 496 | }, 497 | { 498 | "id": "kajaani-alerts", 499 | "type": "real-time-alerts", 500 | "frequency": "30s", 501 | "url": "http://digitransit-proxy:8080/out/kajaani.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 502 | "feedId": "Kajaani", 503 | "fuzzyTripMatching": false 504 | }, 505 | { 506 | "id": "rauma-trip-updates", 507 | "type": "stop-time-updater", 508 | "frequency": "60s", 509 | "url": "http://digitransit-proxy:8080/out/rauma.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 510 | "feedId": "Rauma", 511 | "fuzzyTripMatching": false, 512 | "backwardsDelayPropagationType": "ALWAYS" 513 | }, 514 | { 515 | "id": "rauma-alerts", 516 | "type": "real-time-alerts", 517 | "frequency": "30s", 518 | "url": "http://digitransit-proxy:8080/out/rauma.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 519 | "feedId": "Rauma", 520 | "fuzzyTripMatching": false 521 | }, 522 | { 523 | "id": "pori-trip-updates", 524 | "type": "stop-time-updater", 525 | "frequency": "60s", 526 | "url": "http://digitransit-proxy:8080/out/pjl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 527 | "feedId": "Pori", 528 | "fuzzyTripMatching": false, 529 | "backwardsDelayPropagationType": "ALWAYS" 530 | }, 531 | { 532 | "id": "pori-alerts", 533 | "type": "real-time-alerts", 534 | "frequency": "30s", 535 | "url": "http://digitransit-proxy:8080/out/pjl.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 536 | "feedId": "Pori", 537 | "fuzzyTripMatching": false 538 | }, 539 | { 540 | "id": "salo-trip-updates", 541 | "type": "stop-time-updater", 542 | "frequency": "60s", 543 | "url": "http://digitransit-proxy:8080/out/paikku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 544 | "feedId": "Salo", 545 | "fuzzyTripMatching": false, 546 | "backwardsDelayPropagationType": "ALWAYS" 547 | }, 548 | { 549 | "id": "salo-alerts", 550 | "type": "real-time-alerts", 551 | "frequency": "30s", 552 | "url": "http://digitransit-proxy:8080/out/paikku.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 553 | "feedId": "Salo", 554 | "fuzzyTripMatching": false 555 | }, 556 | { 557 | "id": "lappeenranta-bike-rental", 558 | "type": "vehicle-rental", 559 | "sourceType": "gbfs", 560 | "frequency": "30s", 561 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_lappeenranta/gbfs.json", 562 | "overloadingAllowed": true, 563 | "rentalPickupTypes": ["station"] 564 | }, 565 | { 566 | "id": "kotka-bike-rental", 567 | "type": "vehicle-rental", 568 | "sourceType": "gbfs", 569 | "frequency": "30s", 570 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_kotka/gbfs.json", 571 | "overloadingAllowed": true, 572 | "rentalPickupTypes": ["station"] 573 | }, 574 | { 575 | "id": "kouvola-bike-rental", 576 | "type": "vehicle-rental", 577 | "sourceType": "gbfs", 578 | "frequency": "30s", 579 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_kouvola/gbfs.json", 580 | "overloadingAllowed": true 581 | }, 582 | { 583 | "id": "lahti-bike-rental", 584 | "type": "vehicle-rental", 585 | "sourceType": "gbfs", 586 | "frequency": "60s", 587 | "network": "freebike_lahti", 588 | "url": "http://digitransit-proxy:8080/out/tkhslahtistorage.blob.core.windows.net/gbfs/gbfs.json", 589 | "overloadingAllowed": true, 590 | "rentalPickupTypes": ["station"] 591 | }, 592 | { 593 | "id": "vantaa-bike-rental", 594 | "type": "vehicle-rental", 595 | "sourceType": "gbfs", 596 | "frequency": "30s", 597 | "network": "vantaa", 598 | "url": "http://digitransit-proxy:8080/out/vantaa-api.giravolta.io/api-opendata/gbfs/2_3/gbfs", 599 | "overloadingAllowed": true, 600 | "rentalPickupTypes": ["station"] 601 | }, 602 | { 603 | "id": "hsl-bike-rental", 604 | "type": "vehicle-rental", 605 | "sourceType": "smoove", 606 | "frequency": "30s", 607 | "network": "smoove", 608 | "url": "http://digitransit-proxy:8080/out/helsinki-fi.smoove.pro/api-public/stations", 609 | "overloadingAllowed": true 610 | }, 611 | { 612 | "id": "turku-bike-rental", 613 | "type": "vehicle-rental", 614 | "sourceType": "gbfs", 615 | "frequency": "30s", 616 | "url": "http://digitransit-proxy:8080/out/stables.donkey.bike/api/public/gbfs/2/donkey_turku/gbfs.json", 617 | "overloadingAllowed": true 618 | }, 619 | { 620 | "id": "vilkku-bike-rental", 621 | "network": "freebike_kuopio", 622 | "type": "vehicle-rental", 623 | "sourceType": "gbfs", 624 | "frequency": "30s", 625 | "url": "http://digitransit-proxy:8080/out/tkhskuopiostrg.blob.core.windows.net/gbfs/gbfs.json", 626 | "overloadingAllowed": true 627 | }, 628 | { 629 | "id": "tampere-bike-rental", 630 | "network": "inurba_tampere", 631 | "type": "vehicle-rental", 632 | "sourceType": "gbfs", 633 | "frequency": "30s", 634 | "url": "https://gbfs.urbansharing.com/tampere.onurbansharing.com/gbfs.json", 635 | "overloadingAllowed": true 636 | }, 637 | { 638 | "id": "liipi", 639 | "type": "vehicle-parking", 640 | "sourceType": "liipi", 641 | "feedId": "liipi", 642 | "timeZone": "Europe/Helsinki", 643 | "facilitiesFrequencySec": 3600, 644 | "facilitiesUrl": "https://parking.fintraffic.fi/api/v1/facilities.json?limit=-1", 645 | "utilizationsFrequencySec": 600, 646 | "utilizationsUrl": "https://parking.fintraffic.fi/api/v1/utilizations.json?limit=-1", 647 | "hubsUrl": "https://parking.fintraffic.fi/api/v1/hubs.json?limit=-1" 648 | }, 649 | { 650 | "id": "raasepori-alerts", 651 | "type": "real-time-alerts", 652 | "frequency": "30s", 653 | "url": "http://digitransit-proxy:8080/out/bosse.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 654 | "feedId": "Raasepori", 655 | "fuzzyTripMatching": false 656 | }, 657 | { 658 | "id": "varely-trip-updates", 659 | "type": "stop-time-updater", 660 | "frequency": "60s", 661 | "url": "http://digitransit-proxy:8080/out/varely.mattersoft.fi/api/gtfsrealtime/v1.0/feed/tripupdate", 662 | "feedId": "VARELY", 663 | "fuzzyTripMatching": false, 664 | "backwardsDelayPropagationType": "ALWAYS" 665 | }, 666 | { 667 | "id": "varely-alerts", 668 | "type": "real-time-alerts", 669 | "frequency": "30s", 670 | "url": "http://digitransit-proxy:8080/out/varely.mattersoft.fi/api/gtfsrealtime/v1.0/feed/servicealert", 671 | "feedId": "VARELY", 672 | "fuzzyTripMatching": false 673 | } 674 | ] 675 | } 676 | --------------------------------------------------------------------------------