├── tests ├── feeds │ ├── invalid_feed_1 │ │ └── bazin.ga │ ├── invalid_feed_2 │ │ ├── transfers.txt │ │ ├── frequencies.txt │ │ ├── routes.txt │ │ ├── feed_info.txt │ │ ├── agency.txt │ │ ├── fare_rules.txt │ │ ├── fare_attributes.txt │ │ ├── trips.txt │ │ ├── shapes.txt │ │ ├── stop_times.txt │ │ └── stops.txt │ ├── mock_agency │ │ ├── transfers.txt │ │ ├── calendar_dates.txt │ │ ├── frequencies.txt │ │ ├── routes.txt │ │ ├── feed_info.txt │ │ ├── agency.txt │ │ ├── fare_rules.txt │ │ ├── calendar.txt │ │ ├── fare_attributes.txt │ │ ├── trips.txt │ │ ├── shapes.txt │ │ ├── stop_times.txt │ │ └── stops.txt │ ├── interpolated_no_shapes │ │ ├── trips.txt │ │ ├── routes.txt │ │ ├── agency.txt │ │ ├── calendar.txt │ │ ├── stop_times.txt │ │ └── stops.txt │ ├── interpolated_with_shapes │ │ ├── trips.txt │ │ ├── routes.txt │ │ ├── agency.txt │ │ ├── calendar.txt │ │ ├── shapes.txt │ │ ├── stop_times.txt │ │ └── stops.txt │ ├── wide_range_in_calendar_dates │ │ ├── calendar_dates.txt │ │ ├── routes.txt │ │ ├── agency.txt │ │ ├── calendar.txt │ │ ├── trips.txt │ │ ├── shapes.txt │ │ ├── stop_times.txt │ │ └── stops.txt │ ├── only_calendar_dates │ │ ├── calendar_dates.txt │ │ ├── routes.txt │ │ ├── agency.txt │ │ ├── trips.txt │ │ ├── stop_times.txt │ │ ├── shapes.txt │ │ └── stops.txt │ └── only_calendar │ │ ├── routes.txt │ │ ├── agency.txt │ │ ├── calendar.txt │ │ ├── stops.txt │ │ ├── trips.txt │ │ └── stop_times.txt ├── util.js ├── download.test.js ├── db.operations.test.js ├── db.load.test.js └── db.query.test.js ├── scripts ├── renameCoverageFolder.js └── preMerge.js ├── models ├── shape.js ├── feed_info.js ├── spatial │ ├── shape_gis.js │ ├── stop.js │ └── trip.js ├── fare_attribute.js ├── agency.js ├── calendar_date.js ├── frequency.js ├── calendar.js ├── transfer.js ├── index.js ├── route.js ├── stop_time.js ├── fare_rule.js ├── trip.js └── stop.js ├── .gitignore ├── .eslintrc.json ├── LICENSE ├── index.js ├── .travis.yml ├── lib ├── util.js ├── download.js ├── operations.js └── gtfsLoader.js ├── README.md └── package.json /tests/feeds/invalid_feed_1/bazin.ga: -------------------------------------------------------------------------------- 1 | asldkjf 2 | -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/transfers.txt: -------------------------------------------------------------------------------- 1 | from_stop_id,to_stop_id,transfer_type 2 | SAC1,SAC2,3 -------------------------------------------------------------------------------- /tests/feeds/mock_agency/transfers.txt: -------------------------------------------------------------------------------- 1 | from_stop_id,to_stop_id,transfer_type 2 | SAC1,SAC2,3 -------------------------------------------------------------------------------- /tests/feeds/mock_agency/calendar_dates.txt: -------------------------------------------------------------------------------- 1 | service_id,date,exception_type 2 | weekend,20151225,2 3 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_no_shapes/trips.txt: -------------------------------------------------------------------------------- 1 | route_id,trip_id,direction_id,service_id 2 | 1,1,0,weekday 3 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_with_shapes/trips.txt: -------------------------------------------------------------------------------- 1 | route_id,trip_id,direction_id,service_id,shape_id 2 | 1,1,0,weekday,1 3 | -------------------------------------------------------------------------------- /tests/feeds/wide_range_in_calendar_dates/calendar_dates.txt: -------------------------------------------------------------------------------- 1 | service_id,date,exception_type 2 | christmas,20151225,2 3 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar_dates/calendar_dates.txt: -------------------------------------------------------------------------------- 1 | service_id,date,exception_type 2 | day1,20150801,1 3 | day2,20150802,1 4 | -------------------------------------------------------------------------------- /tests/feeds/mock_agency/frequencies.txt: -------------------------------------------------------------------------------- 1 | trip_id,start_time,end_time,headway_secs,exact_times 2 | weekday_trips,8:00:00,16:00:01,7200,1 -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/frequencies.txt: -------------------------------------------------------------------------------- 1 | trip_id,start_time,end_time,headway_secs,exact_times 2 | weekday_trips,8:00:00,16:00:01,7200,1 -------------------------------------------------------------------------------- /tests/feeds/interpolated_no_shapes/routes.txt: -------------------------------------------------------------------------------- 1 | route_id,route_short_name,route_long_name,route_desc,route_type,route_url 2 | 1,1,Route 1,,3, 3 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_with_shapes/routes.txt: -------------------------------------------------------------------------------- 1 | route_id,route_short_name,route_long_name,route_desc,route_type,route_url 2 | 1,1,Route 1,,3, 3 | -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/routes.txt: -------------------------------------------------------------------------------- 1 | route_id,route_short_name,route_long_name,route_desc,route_type,route_url 2 | LA-Seattle,LA-SEA,Los Angeles - Seattle,,2, -------------------------------------------------------------------------------- /tests/feeds/mock_agency/routes.txt: -------------------------------------------------------------------------------- 1 | route_id,route_short_name,route_long_name,route_desc,route_type,route_url 2 | LA-Seattle,LA-SEA,Los Angeles - Seattle,,2, -------------------------------------------------------------------------------- /tests/feeds/only_calendar/routes.txt: -------------------------------------------------------------------------------- 1 | route_id,route_short_name,route_long_name,route_desc,route_type,route_url 2 | good-route,G,Route with 15 min frequency,,3, 3 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar_dates/routes.txt: -------------------------------------------------------------------------------- 1 | route_id,route_short_name,route_long_name,route_desc,route_type,route_url 2 | LA-Seattle,LA-SEA,Los Angeles - Seattle,,2, -------------------------------------------------------------------------------- /scripts/renameCoverageFolder.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | 3 | var dialect = process.env.DIALECT 4 | 5 | fs.rename('coverage', 'coverage-' + dialect, function(){}); -------------------------------------------------------------------------------- /tests/feeds/mock_agency/feed_info.txt: -------------------------------------------------------------------------------- 1 | feed_publisher_name,feed_publisher_url,feed_lang,feed_version 2 | mock factory,https://github.com/evansiroky/gtfs-sequelize,en,1.0 3 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar/agency.txt: -------------------------------------------------------------------------------- 1 | agency_name,agency_url,agency_timezone,agency_lang,agency_phone 2 | test,http://www.example.org,America/Los_Angeles,en,5555555555 3 | -------------------------------------------------------------------------------- /tests/feeds/wide_range_in_calendar_dates/routes.txt: -------------------------------------------------------------------------------- 1 | route_id,route_short_name,route_long_name,route_desc,route_type,route_url 2 | LA-Seattle,LA-SEA,Los Angeles - Seattle,,2, -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/feed_info.txt: -------------------------------------------------------------------------------- 1 | feed_publisher_name,feed_publisher_url,feed_lang,feed_version 2 | mock factory,https://github.com/evansiroky/gtfs-sequelize,en,1.0 3 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar/calendar.txt: -------------------------------------------------------------------------------- 1 | service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date 2 | weekday,1,1,1,1,1,0,0,20000101,21001231 3 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_no_shapes/agency.txt: -------------------------------------------------------------------------------- 1 | agency_name,agency_url,agency_timezone,agency_lang,agency_phone 2 | test,http://www.example.org,America/Los_Angeles,en,5555555555 3 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_no_shapes/calendar.txt: -------------------------------------------------------------------------------- 1 | service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date 2 | weekday,1,1,1,1,1,0,0,20000101,21001231 3 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_with_shapes/agency.txt: -------------------------------------------------------------------------------- 1 | agency_name,agency_url,agency_timezone,agency_lang,agency_phone 2 | test,http://www.example.org,America/Los_Angeles,en,5555555555 3 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_with_shapes/calendar.txt: -------------------------------------------------------------------------------- 1 | service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date 2 | weekday,1,1,1,1,1,0,0,20000101,21001231 3 | -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/agency.txt: -------------------------------------------------------------------------------- 1 | agency_name,agency_url,agency_timezone,agency_lang,agency_phone 2 | West Coast Maglev,http://www.example.com,America/Los_Angeles,en,(555) 555-5555 3 | -------------------------------------------------------------------------------- /tests/feeds/mock_agency/agency.txt: -------------------------------------------------------------------------------- 1 | agency_name,agency_url,agency_timezone,agency_lang,agency_phone 2 | West Coast Maglev,http://www.example.com,America/Los_Angeles,en,(555) 555-5555 3 | -------------------------------------------------------------------------------- /tests/feeds/mock_agency/fare_rules.txt: -------------------------------------------------------------------------------- 1 | fare_id,route_id,origin_id,destination_id,contains_id 2 | route_based_fare,LA-Seattle,,, 3 | origin_destination_fare,,Z1,Z2, 4 | contains_fare,,,,Z3 -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/fare_rules.txt: -------------------------------------------------------------------------------- 1 | fare_id,route_id,origin_id,destination_id,contains_id 2 | route_based_fare,LA-Seattle,,, 3 | origin_destination_fare,,Z1,Z2, 4 | contains_fare,,,,Z3 -------------------------------------------------------------------------------- /tests/feeds/only_calendar_dates/agency.txt: -------------------------------------------------------------------------------- 1 | agency_name,agency_url,agency_timezone,agency_lang,agency_phone 2 | West Coast Maglev,http://www.example.com,America/Los_Angeles,en,(555) 555-5555 3 | -------------------------------------------------------------------------------- /tests/feeds/wide_range_in_calendar_dates/agency.txt: -------------------------------------------------------------------------------- 1 | agency_name,agency_url,agency_timezone,agency_lang,agency_phone 2 | West Coast Maglev,http://www.example.com,America/Los_Angeles,en,(555) 555-5555 3 | -------------------------------------------------------------------------------- /scripts/preMerge.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'), 2 | rimraf = require('rimraf'); 3 | 4 | rimraf('coverage', function(err) { 5 | if(err) throw new Error(err); 6 | fs.mkdir('coverage', function(){}); 7 | }) -------------------------------------------------------------------------------- /tests/feeds/mock_agency/calendar.txt: -------------------------------------------------------------------------------- 1 | service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date 2 | weekday,1,1,1,1,1,0,0,20000101,21001231 3 | weekend,0,0,0,0,0,1,1,20000101,21001231 4 | -------------------------------------------------------------------------------- /tests/feeds/mock_agency/fare_attributes.txt: -------------------------------------------------------------------------------- 1 | fare_id,price,currency_type,payment_method,transfers 2 | global_fare,10,USD,0,0 3 | route_based_fare,20,USD,0,0 4 | origin_destination_fare,30,USD,0,0 5 | contains_fare,40,USD,0,0 6 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_no_shapes/stop_times.txt: -------------------------------------------------------------------------------- 1 | trip_id,arrival_time,departure_time,stop_id,stop_sequence 2 | 1,08:00:00,08:00:00,1,1 3 | 1,,,2,2 4 | 1,,,3,3 5 | 1,08:04:00,08:04:00,4,4 6 | 1,,,5,5 7 | 1,08:08:00,08:08:00,6,6 8 | -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/fare_attributes.txt: -------------------------------------------------------------------------------- 1 | fare_id,price,currency_type,payment_method,transfers 2 | global_fare,10,USD,0,0 3 | route_based_fare,20,USD,0,0 4 | origin_destination_fare,30,USD,0,0 5 | contains_fare,40,USD,0,0 6 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar_dates/trips.txt: -------------------------------------------------------------------------------- 1 | route_id,service_id,trip_id,trip_headsign,block_id,shape_id 2 | LA-Seattle,day1,trip1,Seattle Weekend Express,1,la-sea-shp 3 | LA-Seattle,day2,trip2,Seattle Weekday Express,1,la-sea-shp 4 | -------------------------------------------------------------------------------- /tests/feeds/wide_range_in_calendar_dates/calendar.txt: -------------------------------------------------------------------------------- 1 | service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,start_date,end_date 2 | weekday,1,1,1,1,1,0,0,20000101,20101231 3 | weekend,0,0,0,0,0,1,1,20000101,20101231 4 | -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/trips.txt: -------------------------------------------------------------------------------- 1 | route_id,service_id,trip_id,trip_headsign,block_id,shape_id 2 | LA-Seattle,weekend,weekend_trip,Seattle Weekend Express,1,la-sea-shp 3 | LA-Seattle,weekday,weekday_trips,Seattle Weekday Express,1,la-sea-shp -------------------------------------------------------------------------------- /tests/feeds/mock_agency/trips.txt: -------------------------------------------------------------------------------- 1 | route_id,service_id,trip_id,trip_headsign,block_id,shape_id 2 | LA-Seattle,weekend,weekend_trip,Seattle Weekend Express,1,la-sea-shp 3 | LA-Seattle,weekday,weekday_trips,Seattle Weekday Express,1,la-sea-shp -------------------------------------------------------------------------------- /tests/feeds/wide_range_in_calendar_dates/trips.txt: -------------------------------------------------------------------------------- 1 | route_id,service_id,trip_id,trip_headsign,block_id,shape_id 2 | LA-Seattle,weekend,weekend_trip,Seattle Weekend Express,1,la-sea-shp 3 | LA-Seattle,weekday,weekday_trips,Seattle Weekday Express,1,la-sea-shp -------------------------------------------------------------------------------- /tests/feeds/interpolated_with_shapes/shapes.txt: -------------------------------------------------------------------------------- 1 | shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence 2 | 1,37.04469,-122.02294,1 3 | 1,37.04692,-122.01877,2 4 | 1,37.04825,-122.01693,3 5 | 1,37.05109,-122.0147,4 6 | 1,37.05541,-122.01199,5 7 | 1,37.05904,-122.01002,6 8 | -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/shapes.txt: -------------------------------------------------------------------------------- 1 | shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence 2 | la-sea-shp,34.056313,-118.234014,1 3 | la-sea-shp,36.970318,-118.705662,2 4 | la-sea-shp,42.939383,-122.099206,3 5 | la-sea-shp,46.662348,-122.101693,4 6 | la-sea-shp,47.598398,-122.329480,5 -------------------------------------------------------------------------------- /tests/feeds/mock_agency/shapes.txt: -------------------------------------------------------------------------------- 1 | shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence 2 | la-sea-shp,34.056313,-118.234014,1 3 | la-sea-shp,36.970318,-118.705662,2 4 | la-sea-shp,42.939383,-122.099206,3 5 | la-sea-shp,46.662348,-122.101693,4 6 | la-sea-shp,47.598398,-122.329480,5 7 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar_dates/stop_times.txt: -------------------------------------------------------------------------------- 1 | trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type 2 | trip1,12:00:00,12:00:00,LA,1,0,0 3 | trip1,14:00:00,14:00:00,SEA,2,0,0 4 | trip2,00:00:00,00:00:00,LA,1,0,0 5 | trip2,02:05:00,02:05:00,SEA,2,0,0 6 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar/stops.txt: -------------------------------------------------------------------------------- 1 | stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding 2 | 1,,Butler Ln,,37.0612132,-122.0074332,,,0,,, 3 | 2,,Scotts Valley Dr & Victor Sq,,37.0590172,-122.0096058,,,0,,, 4 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar_dates/shapes.txt: -------------------------------------------------------------------------------- 1 | shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence 2 | la-sea-shp,34.056313,-118.234014,1 3 | la-sea-shp,36.970318,-118.705662,2 4 | la-sea-shp,42.939383,-122.099206,3 5 | la-sea-shp,46.662348,-122.101693,4 6 | la-sea-shp,47.598398,-122.329480,5 -------------------------------------------------------------------------------- /tests/feeds/wide_range_in_calendar_dates/shapes.txt: -------------------------------------------------------------------------------- 1 | shape_id,shape_pt_lat,shape_pt_lon,shape_pt_sequence 2 | la-sea-shp,34.056313,-118.234014,1 3 | la-sea-shp,36.970318,-118.705662,2 4 | la-sea-shp,42.939383,-122.099206,3 5 | la-sea-shp,46.662348,-122.101693,4 6 | la-sea-shp,47.598398,-122.329480,5 -------------------------------------------------------------------------------- /tests/feeds/interpolated_with_shapes/stop_times.txt: -------------------------------------------------------------------------------- 1 | trip_id,arrival_time,departure_time,stop_id,stop_sequence,shape_dist_traveled 2 | 1,08:00:00,08:00:00,1,1,0 3 | 1,,,2,2,0.445481 4 | 1,,,3,3,0.665792 5 | 1,08:04:00,08:04:00,4,4,1.038474 6 | 1,,,5,5,1.575674 7 | 1,08:08:00,08:08:00,6,6,2.015540 8 | -------------------------------------------------------------------------------- /tests/feeds/mock_agency/stop_times.txt: -------------------------------------------------------------------------------- 1 | trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type 2 | weekend_trip,12:00:00,12:00:00,LA,1,0,0 3 | weekend_trip,14:00:00,14:00:00,SEA,2,0,0 4 | weekday_trips,00:00:00,00:00:00,LA,1,0,0 5 | weekday_trips,02:05:00,02:05:00,SEA,2,0,0 6 | -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/stop_times.txt: -------------------------------------------------------------------------------- 1 | trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type 2 | weekend_trip,12:00:00,12:00:00,LA,1,0,0 3 | weekend_trip,14:00:00,14:00:00,SEA,2,0,0 4 | weekday_trips,00:00:00,00:00:00,LA,1,0,0 5 | weekday_trips,02:05:00,02:05:00,SEA,2,0,0 6 | -------------------------------------------------------------------------------- /tests/feeds/wide_range_in_calendar_dates/stop_times.txt: -------------------------------------------------------------------------------- 1 | trip_id,arrival_time,departure_time,stop_id,stop_sequence,pickup_type,drop_off_type 2 | weekend_trip,12:00:00,12:00:00,LA,1,0,0 3 | weekend_trip,14:00:00,14:00:00,SEA,2,0,0 4 | weekday_trips,00:00:00,00:00:00,LA,1,0,0 5 | weekday_trips,02:05:00,02:05:00,SEA,2,0,0 6 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_no_shapes/stops.txt: -------------------------------------------------------------------------------- 1 | stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding 2 | 1,,Bean Creek,,37.04469,-122.02294,,,0,,, 3 | 2,,Erba,,37.04692,-122.01877,,,0,,, 4 | 3,,Disc,,37.04825,-122.01693,,,0,,, 5 | 4,,Carbonero,,37.05109,-122.0147,,,0,,, 6 | 5,,El Pueblo,,37.05541,-122.01199,,,0,,, 7 | 6,,Victor Sq,,37.05904,-122.01002,,,0,,, 8 | -------------------------------------------------------------------------------- /tests/feeds/interpolated_with_shapes/stops.txt: -------------------------------------------------------------------------------- 1 | stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding 2 | 1,,Bean Creek,,37.04469,-122.02294,,,0,,, 3 | 2,,Erba,,37.04692,-122.01877,,,0,,, 4 | 3,,Disc,,37.04825,-122.01693,,,0,,, 5 | 4,,Carbonero,,37.05109,-122.0147,,,0,,, 6 | 5,,El Pueblo,,37.05541,-122.01199,,,0,,, 7 | 6,,Victor Sq,,37.05904,-122.01002,,,0,,, 8 | -------------------------------------------------------------------------------- /tests/feeds/invalid_feed_2/stops.txt: -------------------------------------------------------------------------------- 1 | stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding 2 | LA,1,Los Angeles,,34.056313,-118.234014,,,,,, 3 | SEA,2,Seattle,,47.598398,-122.329480,,,,,, 4 | SF,3,San Francisco,,37.789475,-122.396447,Z1,,,,, 5 | PDX,4,Portland,,45.529091,-122.676772,Z2,,,,, 6 | SAC1,5,Sacramento,,38.586337,-121.499188,Z3,,,,, 7 | SAC2,6,Sacramento-2,,38.591070,-121.500376,,,,,, -------------------------------------------------------------------------------- /tests/feeds/mock_agency/stops.txt: -------------------------------------------------------------------------------- 1 | stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding 2 | LA,1,Los Angeles,,34.056313,-118.234014,,,,,, 3 | SEA,2,Seattle,,47.598398,-122.329480,,,,,, 4 | SF,3,San Francisco,,37.789475,-122.396447,Z1,,,,, 5 | PDX,4,Portland,,45.529091,-122.676772,Z2,,,,, 6 | SAC1,5,Sacramento,,38.586337,-121.499188,Z3,,,,, 7 | SAC2,6,Sacramento-2,,38.591070,-121.500376,,,,,, -------------------------------------------------------------------------------- /tests/feeds/only_calendar_dates/stops.txt: -------------------------------------------------------------------------------- 1 | stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding 2 | LA,1,Los Angeles,,34.056313,-118.234014,,,,,, 3 | SEA,2,Seattle,,47.598398,-122.329480,,,,,, 4 | SF,3,San Francisco,,37.789475,-122.396447,Z1,,,,, 5 | PDX,4,Portland,,45.529091,-122.676772,Z2,,,,, 6 | SAC1,5,Sacramento,,38.586337,-121.499188,Z3,,,,, 7 | SAC2,6,Sacramento-2,,38.591070,-121.500376,,,,,, -------------------------------------------------------------------------------- /tests/feeds/wide_range_in_calendar_dates/stops.txt: -------------------------------------------------------------------------------- 1 | stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon,zone_id,stop_url,location_type,parent_station,stop_timezone,wheelchair_boarding 2 | LA,1,Los Angeles,,34.056313,-118.234014,,,,,, 3 | SEA,2,Seattle,,47.598398,-122.329480,,,,,, 4 | SF,3,San Francisco,,37.789475,-122.396447,Z1,,,,, 5 | PDX,4,Portland,,45.529091,-122.676772,Z2,,,,, 6 | SAC1,5,Sacramento,,38.586337,-121.499188,Z3,,,,, 7 | SAC2,6,Sacramento-2,,38.591070,-121.500376,,,,,, -------------------------------------------------------------------------------- /models/shape.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | return sequelize.define("shape", { 5 | shape_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | shape_pt_lat: DataTypes.FLOAT(7), 10 | shape_pt_lon: DataTypes.FLOAT(7), 11 | shape_pt_sequence: { 12 | type: DataTypes.INTEGER, 13 | primaryKey: true 14 | }, 15 | shape_dist_traveled: DataTypes.FLOAT 16 | }, util.makeTableOptions(sequelize, { 17 | freezeTableName: true 18 | })); 19 | } 20 | -------------------------------------------------------------------------------- /models/feed_info.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | return sequelize.define("feed_info", { 5 | feed_publisher_name: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | feed_publisher_url: DataTypes.STRING(255), 10 | feed_lang: DataTypes.STRING(255), 11 | feed_start_date: DataTypes.DATE, 12 | feed_end_date: DataTypes.DATE, 13 | feed_version: DataTypes.STRING(255) 14 | }, util.makeTableOptions(sequelize, { 15 | freezeTableName: true 16 | })); 17 | } 18 | -------------------------------------------------------------------------------- /models/spatial/shape_gis.js: -------------------------------------------------------------------------------- 1 | var util = require('../../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var ShapeGIS = sequelize.define("shape_gis", { 5 | shape_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | geom: DataTypes.GEOMETRY('LINESTRING', 4326) 10 | }, util.makeTableOptions(sequelize, { 11 | freezeTableName: true, 12 | classMethods: { 13 | associate: function (models) { 14 | 15 | ShapeGIS.hasMany(models.trip, { 16 | foreignKey: 'shape_id' 17 | }); 18 | 19 | } 20 | } 21 | })); 22 | 23 | return ShapeGIS; 24 | } 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage* 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 27 | node_modules 28 | 29 | # Project level exclusions 30 | downloads 31 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true, 4 | "node": true 5 | }, 6 | "extends": "standard", 7 | "rules": { 8 | "complexity": ["warn", 10], 9 | "import/order": ["warn", { 10 | "newlines-between": "always" 11 | }], 12 | "no-magic-numbers": ["warn", { 13 | "ignore": [-1, 0, 1, 2] 14 | }], 15 | "prefer-const": ["warn", { 16 | "destructuring": "all", 17 | "ignoreReadBeforeAssign": false 18 | }], 19 | "require-jsdoc": ["warn", { 20 | "require": { 21 | "FunctionDeclaration": true, 22 | "MethodDefinition": true, 23 | "ClassDeclaration": true 24 | } 25 | }], 26 | "sort-keys": "warn", 27 | "sort-vars": "warn" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /models/fare_attribute.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var FareAttribute = sequelize.define("fare_attribute", { 5 | fare_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | price: DataTypes.FLOAT, 10 | currency_type: DataTypes.STRING(3), 11 | payment_method: DataTypes.INTEGER, 12 | transfers: DataTypes.INTEGER, 13 | transfer_duration: DataTypes.INTEGER 14 | }, util.makeTableOptions(sequelize, { 15 | freezeTableName: true, 16 | classMethods: { 17 | associate: function (models) { 18 | 19 | FareAttribute.hasMany(models.fare_rule, { 20 | foreignKey: 'fare_id' 21 | }); 22 | } 23 | } 24 | })); 25 | 26 | return FareAttribute; 27 | } 28 | -------------------------------------------------------------------------------- /models/agency.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Agency = sequelize.define("agency", { 5 | agency_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | agency_name: DataTypes.STRING(255), 10 | agency_url: DataTypes.STRING(255), 11 | agency_timezone: DataTypes.STRING(100), 12 | agency_lang: DataTypes.STRING(2), 13 | agency_phone: DataTypes.STRING(50), 14 | agency_fare_url: DataTypes.STRING(255) 15 | }, util.makeTableOptions(sequelize, { 16 | freezeTableName: true, 17 | classMethods: { 18 | associate: function (models) { 19 | Agency.hasMany(models.route, { 20 | foreignKey: 'agency_id' 21 | }); 22 | } 23 | } 24 | })); 25 | 26 | return Agency; 27 | } 28 | -------------------------------------------------------------------------------- /models/calendar_date.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var CalendarDate = sequelize.define("calendar_date", { 5 | service_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true, 8 | references: { 9 | model: util.makeModelReference(sequelize, "calendar"), 10 | key: "service_id" 11 | } 12 | }, 13 | date: { 14 | type: DataTypes.STRING(8), 15 | primaryKey: true 16 | }, 17 | exception_type: DataTypes.INTEGER 18 | }, util.makeTableOptions(sequelize, { 19 | freezeTableName: true, 20 | classMethods: { 21 | associate: function (models) { 22 | CalendarDate.belongsTo(models.calendar, { 23 | foreignKeyContraint: true, 24 | foreignKey: "service_id" 25 | }); 26 | } 27 | } 28 | })); 29 | 30 | return CalendarDate; 31 | } 32 | -------------------------------------------------------------------------------- /models/frequency.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Frequency = sequelize.define("frequency", { 5 | trip_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true, 8 | references: { 9 | model: util.makeModelReference(sequelize, "trip"), 10 | key: "trip_id" 11 | } 12 | }, 13 | start_time: { 14 | type: DataTypes.INTEGER, 15 | primaryKey: true 16 | }, 17 | end_time: { 18 | type: DataTypes.INTEGER, 19 | primaryKey: true 20 | }, 21 | headway_secs: DataTypes.INTEGER, 22 | exact_times: DataTypes.INTEGER 23 | }, util.makeTableOptions(sequelize, { 24 | freezeTableName: true, 25 | classMethods: { 26 | associate: function (models) { 27 | 28 | Frequency.belongsTo(models.trip, { 29 | foreignKeyContraint: true, 30 | foreignKey: "trip_id" 31 | }); 32 | 33 | } 34 | } 35 | })); 36 | 37 | return Frequency; 38 | } 39 | -------------------------------------------------------------------------------- /models/calendar.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Calendar = sequelize.define("calendar", { 5 | service_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | monday: DataTypes.INTEGER, 10 | tuesday: DataTypes.INTEGER, 11 | wednesday: DataTypes.INTEGER, 12 | thursday: DataTypes.INTEGER, 13 | friday: DataTypes.INTEGER, 14 | saturday: DataTypes.INTEGER, 15 | sunday: DataTypes.INTEGER, 16 | start_date: DataTypes.STRING(8), 17 | end_date: DataTypes.STRING(8) 18 | }, util.makeTableOptions(sequelize, { 19 | freezeTableName: true, 20 | classMethods: { 21 | associate: function (models) { 22 | 23 | Calendar.hasMany(models.calendar_date, { 24 | foreignKey: 'service_id' 25 | }); 26 | 27 | Calendar.hasMany(models.trip, { 28 | foreignKey: 'service_id' 29 | }); 30 | 31 | } 32 | } 33 | })); 34 | 35 | return Calendar; 36 | } 37 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar/trips.txt: -------------------------------------------------------------------------------- 1 | route_id,trip_id,direction_id,service_id 2 | good-route,am-1,0,weekday 3 | good-route,am-2,0,weekday 4 | good-route,am-3,0,weekday 5 | good-route,am-4,0,weekday 6 | good-route,am-5,0,weekday 7 | good-route,am-6,0,weekday 8 | good-route,am-7,0,weekday 9 | good-route,am-8,0,weekday 10 | good-route,am-9,0,weekday 11 | good-route,am-10,0,weekday 12 | good-route,am-11,0,weekday 13 | good-route,am-12,0,weekday 14 | good-route,am-13,0,weekday 15 | good-route,am-14,0,weekday 16 | good-route,am-15,0,weekday 17 | good-route,am-16,0,weekday 18 | good-route,pm-1,0,weekday 19 | good-route,pm-2,0,weekday 20 | good-route,pm-3,0,weekday 21 | good-route,pm-4,0,weekday 22 | good-route,pm-5,0,weekday 23 | good-route,pm-6,0,weekday 24 | good-route,pm-7,0,weekday 25 | good-route,pm-8,0,weekday 26 | good-route,pm-9,0,weekday 27 | good-route,pm-10,0,weekday 28 | good-route,pm-11,0,weekday 29 | good-route,pm-12,0,weekday 30 | good-route,pm-13,0,weekday 31 | good-route,pm-14,0,weekday 32 | good-route,pm-15,0,weekday 33 | good-route,pm-16,0,weekday 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Evan Siroky 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /models/spatial/stop.js: -------------------------------------------------------------------------------- 1 | var util = require('../../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Stop = sequelize.define("stop", { 5 | stop_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | stop_code: DataTypes.STRING(20), 10 | stop_name: DataTypes.STRING(255), 11 | stop_desc: DataTypes.TEXT, 12 | stop_lat: DataTypes.FLOAT(7), 13 | stop_lon: DataTypes.FLOAT(7), 14 | zone_id: DataTypes.STRING(255), 15 | stop_url: DataTypes.STRING(255), 16 | location_type: DataTypes.INTEGER, 17 | parent_station: DataTypes.INTEGER, 18 | stop_timezone: DataTypes.STRING(100), 19 | wheelchair_boarding: DataTypes.INTEGER, 20 | geom: DataTypes.GEOMETRY('POINT', 4326) 21 | }, util.makeTableOptions(sequelize, { 22 | freezeTableName: true, 23 | classMethods: { 24 | associate: function (models) { 25 | 26 | Stop.hasMany(models.stop_time, { 27 | foreignKey: 'stop_id' 28 | }); 29 | 30 | Stop.hasMany(models.transfer, { 31 | foreignKey: 'from_stop_id' 32 | }); 33 | 34 | Stop.hasMany(models.transfer, { 35 | foreignKey: 'to_stop_id' 36 | }); 37 | 38 | } 39 | } 40 | })); 41 | 42 | return Stop; 43 | } 44 | -------------------------------------------------------------------------------- /models/transfer.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Transfer = sequelize.define("transfer", { 5 | from_stop_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true, 8 | references: { 9 | model: util.makeModelReference(sequelize, "stop"), 10 | key: "stop_id" 11 | } 12 | }, 13 | to_stop_id: { 14 | type: DataTypes.STRING(255), 15 | primaryKey: true, 16 | references: { 17 | model: util.makeModelReference(sequelize, "stop"), 18 | key: "stop_id" 19 | } 20 | }, 21 | transfer_type: DataTypes.INTEGER, 22 | min_transfer_time: DataTypes.INTEGER 23 | }, util.makeTableOptions(sequelize, { 24 | freezeTableName: true, 25 | classMethods: { 26 | associate: function (models) { 27 | 28 | Transfer.belongsTo(models.stop, { 29 | as: 'from_stop', 30 | foreignKeyContraint: true, 31 | foreignKey: 'from_stop_id', 32 | targetKey: 'stop_id' 33 | }); 34 | 35 | Transfer.belongsTo(models.stop, { 36 | as: 'to_stop', 37 | foreignKeyContraint: true, 38 | foreignKey: 'to_stop_id', 39 | targetKey: 'stop_id' 40 | }); 41 | 42 | } 43 | } 44 | })); 45 | 46 | return Transfer; 47 | } 48 | -------------------------------------------------------------------------------- /models/index.js: -------------------------------------------------------------------------------- 1 | // copied from https://github.com/sequelize/express-example 2 | var fs = require("fs"), 3 | path = require("path"), 4 | Sequelize = require("sequelize"); 5 | 6 | module.exports = function(config, options) { 7 | options = options || {} 8 | if(typeof config === 'string' || config instanceof String) { 9 | var sequelize = new Sequelize(config, options); 10 | } else if(!config && options) { 11 | var sequelize = new Sequelize(options); 12 | } else if(config.database) { 13 | var sequelize = new Sequelize(config.database, config.username, config.password, options); 14 | } else { 15 | console.log(typeof config); 16 | console.log(config instanceof String); 17 | var err = Error('invalid database config'); 18 | throw err; 19 | } 20 | 21 | var db = {}; 22 | 23 | fs.readdirSync(__dirname) 24 | .filter(function(file) { 25 | return (file.indexOf(".") > 0) && (file !== "index.js"); 26 | }) 27 | .forEach(function(file) { 28 | var model = sequelize.import(path.join(__dirname, file)); 29 | db[model.name] = model; 30 | }); 31 | 32 | Object.keys(db).forEach(function(modelName) { 33 | if ("associate" in db[modelName]) { 34 | db[modelName].associate(db); 35 | } 36 | }); 37 | 38 | db.sequelize = sequelize; 39 | db.Sequelize = Sequelize; 40 | 41 | return db; 42 | } 43 | -------------------------------------------------------------------------------- /models/route.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Route = sequelize.define("route", { 5 | route_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | agency_id: { 10 | type: DataTypes.STRING(255), 11 | references: { 12 | model: util.makeModelReference(sequelize, 'agency'), 13 | key: "agency_id" 14 | } 15 | }, 16 | route_short_name: DataTypes.STRING(50), 17 | route_long_name: DataTypes.STRING(255), 18 | route_desc: DataTypes.TEXT, 19 | route_type: DataTypes.INTEGER, 20 | route_url: DataTypes.STRING(255), 21 | route_color: DataTypes.STRING(255), 22 | route_text_color: DataTypes.STRING(255) 23 | }, util.makeTableOptions(sequelize, { 24 | freezeTableName: true, 25 | classMethods: { 26 | associate: function (models) { 27 | 28 | Route.belongsTo(models.agency, { 29 | foreignKeyContraint: true, 30 | foreignKey: "agency_id" 31 | }); 32 | 33 | Route.hasMany(models.trip, { 34 | foreignKey: 'route_id' 35 | }); 36 | 37 | /* Don't fully understand how to get these working with sequelize yet 38 | Route.hasMany(models.fare_rule, { 39 | foreignKey: 'route_id' 40 | }); 41 | */ 42 | 43 | } 44 | } 45 | })); 46 | 47 | return Route; 48 | } 49 | -------------------------------------------------------------------------------- /models/stop_time.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var StopTime = sequelize.define("stop_time", { 5 | trip_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true, 8 | references: { 9 | model: util.makeModelReference(sequelize, 'trip'), 10 | key: 'trip_id' 11 | } 12 | }, 13 | arrival_time: DataTypes.INTEGER, 14 | departure_time: DataTypes.INTEGER, 15 | stop_id: { 16 | type: DataTypes.STRING(255), 17 | primaryKey: true, 18 | references: { 19 | model: util.makeModelReference(sequelize, 'stop'), 20 | key: 'stop_id' 21 | } 22 | }, 23 | stop_sequence: { 24 | type: DataTypes.INTEGER, 25 | primaryKey: true 26 | }, 27 | stop_headsign: DataTypes.STRING(255), 28 | pickup_type: DataTypes.INTEGER, 29 | drop_off_type: DataTypes.INTEGER, 30 | shape_dist_traveled: DataTypes.FLOAT, 31 | timepoint: DataTypes.INTEGER 32 | }, util.makeTableOptions(sequelize, { 33 | freezeTableName: true, 34 | classMethods: { 35 | associate: function (models) { 36 | StopTime.belongsTo(models.trip, { 37 | foreignKeyContraint: true, 38 | foreignKey: "trip_id" 39 | }); 40 | StopTime.belongsTo(models.stop, { 41 | foreignKeyContraint: true, 42 | foreignKey: "stop_id" 43 | }); 44 | } 45 | } 46 | })); 47 | 48 | return StopTime; 49 | } 50 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 2 | const downloadGtfs = require('./lib/download.js') 3 | const loadgtfs = require('./lib/gtfsLoader.js') 4 | const operations = require('./lib/operations') 5 | const Database = require('./models') 6 | 7 | module.exports = function (config) { 8 | const connectToDatabase = function (rawModels) { 9 | const db = Database(config.database, config.sequelizeOptions ? config.sequelizeOptions : {}) 10 | if (!rawModels && config.spatial) { 11 | db.stop = db.sequelize.import('models/spatial/stop.js') 12 | db.shape_gis = db.sequelize.import('models/spatial/shape_gis.js') 13 | db.trip = db.sequelize.import('models/spatial/trip.js') 14 | // reassociate spatially-enable models 15 | db.stop.associate(db) 16 | db.shape_gis.associate(db) 17 | db.trip.associate(db) 18 | } 19 | return db 20 | } 21 | 22 | const download = function (callback) { 23 | downloadGtfs(config.gtfsUrl, config.downloadsDir, callback) 24 | } 25 | 26 | const interpolateStopTimes = function (callback) { 27 | const db = connectToDatabase() 28 | operations.interpolateStopTimes(db, callback) 29 | } 30 | 31 | const loadGtfs = function (callback) { 32 | loadgtfs(config.downloadsDir, 33 | config.gtfsFileOrFolder, 34 | connectToDatabase(true), 35 | config.spatial, 36 | config.interpolateStopTimes, 37 | callback) 38 | } 39 | 40 | return { 41 | config: config, 42 | connectToDatabase: connectToDatabase, 43 | downloadGtfs: download, 44 | interpolateStopTimes: interpolateStopTimes, 45 | loadGtfs: loadGtfs 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /models/fare_rule.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var FareRule = sequelize.define("fare_rule", { 5 | fare_id: { 6 | type: DataTypes.STRING(255), 7 | references: { 8 | model: util.makeModelReference(sequelize, "fare_attribute"), 9 | key: "fare_id" 10 | } 11 | }, 12 | route_id: DataTypes.STRING(255), 13 | origin_id: DataTypes.STRING(255), 14 | destination_id: DataTypes.STRING(255), 15 | contains_id: DataTypes.STRING(255) 16 | }, util.makeTableOptions(sequelize, { 17 | freezeTableName: true, 18 | classMethods: { 19 | associate: function (models) { 20 | 21 | FareRule.belongsTo(models.fare_attribute, { 22 | foreignKeyContraint: true, 23 | foreignKey: "fare_id" 24 | }); 25 | 26 | /* Don't fully understand how to get these working with sequelize yet 27 | FareRule.belongsTo(models.route, { 28 | foreignKey: 'route_id', 29 | constraints: false 30 | }); 31 | 32 | FareRule.belongsTo(models.stop, { 33 | as: 'origin_stop', 34 | foreignKey: 'origin_id', 35 | targetKey: 'zone_id', 36 | constraints: false 37 | }); 38 | 39 | FareRule.belongsTo(models.stop, { 40 | as: 'destination_stop', 41 | foreignKey: 'destination_id', 42 | targetKey: 'zone_id', 43 | constraints: false 44 | }); 45 | 46 | FareRule.belongsTo(models.stop, { 47 | as: 'contains_stop', 48 | foreignKey: 'contains_id', 49 | targetKey: 'zone_id', 50 | constraints: false 51 | });*/ 52 | 53 | } 54 | } 55 | })); 56 | 57 | return FareRule; 58 | } 59 | -------------------------------------------------------------------------------- /models/trip.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Trip = sequelize.define("trip", { 5 | route_id: { 6 | type: DataTypes.STRING(255), 7 | references: { 8 | model: util.makeModelReference(sequelize, "route"), 9 | key: "route_id" 10 | } 11 | }, 12 | service_id: { 13 | type: DataTypes.STRING(255), 14 | references: { 15 | model: util.makeModelReference(sequelize, "calendar"), 16 | key: "service_id" 17 | } 18 | }, 19 | trip_id: { 20 | type: DataTypes.STRING(255), 21 | primaryKey: true 22 | }, 23 | trip_headsign: DataTypes.STRING(255), 24 | trip_short_name: DataTypes.STRING(100), 25 | direction_id: { 26 | defaultValue: 0, 27 | type: DataTypes.INTEGER 28 | }, 29 | block_id: DataTypes.STRING(255), 30 | shape_id: DataTypes.STRING(255), // association omitted. See spatial trip model for relation. 31 | wheelchair_accessible: DataTypes.INTEGER, 32 | bikes_allowed: DataTypes.INTEGER 33 | }, util.makeTableOptions(sequelize, { 34 | freezeTableName: true, 35 | classMethods: { 36 | associate: function (models) { 37 | 38 | Trip.belongsTo(models.route, { 39 | foreignKeyContraint: true, 40 | foreignKey: "route_id" 41 | }); 42 | 43 | Trip.belongsTo(models.calendar, { 44 | foreignKeyContraint: true, 45 | foreignKey: "service_id" 46 | }); 47 | 48 | Trip.hasMany(models.stop_time, { 49 | foreignKey: 'trip_id' 50 | }); 51 | 52 | Trip.hasMany(models.frequency, { 53 | foreignKey: 'trip_id' 54 | }); 55 | 56 | } 57 | } 58 | })); 59 | 60 | return Trip; 61 | } 62 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | language: node_js 3 | cache: 4 | directories: 5 | - node_modules 6 | 7 | addons: 8 | postgresql: "9.5" 9 | apt: 10 | packages: 11 | - postgresql-9.5-postgis-2.3 12 | 13 | notifications: 14 | email: false 15 | 16 | node_js: 17 | - '6' 18 | 19 | env: 20 | - COVERAGE=false 21 | 22 | before_install: 23 | - cd ~ 24 | - wget https://sqlite.org/2017/sqlite-autoconf-3160200.tar.gz 25 | - tar xvfz sqlite-autoconf-3160200.tar.gz 26 | - cd sqlite-autoconf-3160200 27 | - ./configure 28 | - sudo make 29 | - sudo make install 30 | - cd /home/travis/build/evansiroky/gtfs-sequelize 31 | - sqlite3 -version 32 | - mysql -uroot -e 'CREATE DATABASE gtfs_sequelize_test;' 33 | - mysql -uroot -e "GRANT ALL PRIVILEGES ON gtfs_sequelize_test.* TO 'gtfs_sequelize'@'localhost' IDENTIFIED BY 'gtfs_sequelize';" 34 | - mysql -uroot -e "FLUSH PRIVILEGES;" 35 | - psql -U postgres -c 'CREATE DATABASE gtfs_sequelize_test;' 36 | - psql gtfs_sequelize_test -U postgres -c 'CREATE EXTENSION postgis;' 37 | - psql -U postgres -c "CREATE USER gtfs_sequelize PASSWORD 'gtfs_sequelize';" 38 | - psql -U postgres -c 'CREATE ROLE gtfs_sequelize_role;' 39 | - psql -U postgres -c 'GRANT gtfs_sequelize_role TO gtfs_sequelize;' 40 | - psql -U postgres -c 'GRANT ALL ON DATABASE gtfs_sequelize_test TO gtfs_sequelize_role;' 41 | - psql gtfs_sequelize_test -U postgres -c 'CREATE SCHEMA test_schema;' 42 | - psql gtfs_sequelize_test -U postgres -c 'GRANT ALL ON SCHEMA test_schema TO PUBLIC;' 43 | 44 | matrix: 45 | fast_finish: true 46 | include: 47 | - node_js: "6" 48 | env: COVERAGE=true 49 | script: "npm run codeclimate" 50 | allow_failures: 51 | - node_js: "6" 52 | env: COVERAGE=true 53 | script: "npm run codeclimate" 54 | 55 | before_script: 56 | - npm prune 57 | 58 | after_success: 59 | - npm run semantic-release 60 | 61 | branches: 62 | except: 63 | - /^v\d+\.\d+\.\d+$/ 64 | -------------------------------------------------------------------------------- /models/spatial/trip.js: -------------------------------------------------------------------------------- 1 | var util = require('../../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Trip = sequelize.define("trip", { 5 | route_id: { 6 | type: DataTypes.STRING(255), 7 | references: { 8 | model: util.makeModelReference(sequelize, "route"), 9 | key: "route_id" 10 | } 11 | }, 12 | service_id: { 13 | type: DataTypes.STRING(255), 14 | references: { 15 | model: util.makeModelReference(sequelize, "calendar"), 16 | key: "service_id" 17 | } 18 | }, 19 | trip_id: { 20 | type: DataTypes.STRING(255), 21 | primaryKey: true 22 | }, 23 | trip_headsign: DataTypes.STRING(255), 24 | trip_short_name: DataTypes.STRING(100), 25 | direction_id: DataTypes.INTEGER, 26 | block_id: DataTypes.STRING(255), 27 | shape_id: { 28 | type: DataTypes.STRING(255), 29 | references: { 30 | model: util.makeModelReference(sequelize, "shape_gis"), 31 | key: "shape_id" 32 | } 33 | }, 34 | wheelchair_accessible: DataTypes.INTEGER, 35 | bikes_allowed: DataTypes.INTEGER 36 | }, util.makeTableOptions(sequelize, { 37 | freezeTableName: true, 38 | classMethods: { 39 | associate: function (models) { 40 | 41 | Trip.belongsTo(models.route, { 42 | foreignKeyContraint: true, 43 | foreignKey: "route_id" 44 | }); 45 | 46 | Trip.belongsTo(models.calendar, { 47 | foreignKeyContraint: true, 48 | foreignKey: "service_id" 49 | }); 50 | 51 | Trip.belongsTo(models.shape_gis, { 52 | foreignKeyContraint: true, 53 | foreignKey: "shape_id" 54 | }); 55 | 56 | Trip.hasMany(models.stop_time, { 57 | foreignKey: 'trip_id' 58 | }); 59 | 60 | Trip.hasMany(models.frequency, { 61 | foreignKey: 'trip_id' 62 | }); 63 | 64 | } 65 | } 66 | })); 67 | 68 | return Trip; 69 | } 70 | -------------------------------------------------------------------------------- /models/stop.js: -------------------------------------------------------------------------------- 1 | var util = require('../lib/util') 2 | 3 | module.exports = function(sequelize, DataTypes) { 4 | var Stop = sequelize.define("stop", { 5 | stop_id: { 6 | type: DataTypes.STRING(255), 7 | primaryKey: true 8 | }, 9 | stop_code: DataTypes.STRING(20), 10 | stop_name: DataTypes.STRING(255), 11 | stop_desc: DataTypes.TEXT, 12 | stop_lat: DataTypes.FLOAT(7), 13 | stop_lon: DataTypes.FLOAT(7), 14 | zone_id: DataTypes.STRING(255), 15 | stop_url: DataTypes.STRING(255), 16 | location_type: DataTypes.INTEGER, 17 | parent_station: DataTypes.STRING(255), 18 | stop_timezone: DataTypes.STRING(100), 19 | wheelchair_boarding: DataTypes.INTEGER 20 | }, util.makeTableOptions(sequelize, { 21 | freezeTableName: true, 22 | classMethods: { 23 | associate: function (models) { 24 | 25 | Stop.hasMany(models.stop_time, { 26 | foreignKey: 'stop_id' 27 | }); 28 | 29 | /* Don't fully understand how to get these working with sequelize yet 30 | Stop.hasMany(models.fare_rule, { 31 | as: 'fare_rule_origins', 32 | foreignKey: 'zone_id', 33 | targetKey: 'origin_id' 34 | }); 35 | 36 | Stop.hasMany(models.fare_rule, { 37 | as: 'fare_rule_destinations', 38 | foreignKey: 'zone_id', 39 | targetKey: 'destination_id' 40 | }); 41 | 42 | Stop.hasMany(models.fare_rule, { 43 | as: 'fare_rule_contains', 44 | foreignKey: 'zone_id', 45 | targetKey: 'contains_id' 46 | }); 47 | 48 | Stop.hasMany(models.transfer, { 49 | as: 'transfer_from_stops', 50 | foreignKey: 'stop_id', 51 | targetKey: 'from_stop_id' 52 | }); 53 | 54 | Stop.hasMany(models.transfer, { 55 | as: 'transfer_to_stops', 56 | foreignKey: 'stop_id', 57 | targetKey: 'to_stop_id' 58 | });*/ 59 | 60 | } 61 | } 62 | })); 63 | 64 | return Stop; 65 | } 66 | -------------------------------------------------------------------------------- /tests/feeds/only_calendar/stop_times.txt: -------------------------------------------------------------------------------- 1 | trip_id,arrival_time,departure_time,stop_id,stop_sequence 2 | am-1,06:00:00,06:00:00,1,1 3 | am-1,06:01:00,06:01:00,2,2 4 | am-2,06:15:00,06:15:00,1,1 5 | am-2,06:16:00,06:16:00,2,2 6 | am-3,06:30:00,06:30:00,1,1 7 | am-3,06:31:00,06:31:00,2,2 8 | am-4,06:45:00,06:45:00,1,1 9 | am-4,06:46:00,06:46:00,2,2 10 | am-5,07:00:00,07:00:00,1,1 11 | am-5,07:01:00,07:01:00,2,2 12 | am-6,07:15:00,07:15:00,1,1 13 | am-6,07:16:00,07:16:00,2,2 14 | am-7,07:30:00,07:30:00,1,1 15 | am-7,07:31:00,07:31:00,2,2 16 | am-8,07:45:00,07:45:00,1,1 17 | am-8,07:46:00,07:46:00,2,2 18 | am-9,08:00:00,08:00:00,1,1 19 | am-9,08:01:00,08:01:00,2,2 20 | am-10,08:15:00,08:15:00,1,1 21 | am-10,08:16:00,08:16:00,2,2 22 | am-11,08:30:00,08:30:00,1,1 23 | am-11,08:31:00,08:31:00,2,2 24 | am-12,08:45:00,08:45:00,1,1 25 | am-12,08:46:00,08:46:00,2,2 26 | am-13,09:00:00,09:00:00,1,1 27 | am-13,09:01:00,09:01:00,2,2 28 | am-14,09:15:00,09:15:00,1,1 29 | am-14,09:16:00,09:16:00,2,2 30 | am-15,09:30:00,09:30:00,1,1 31 | am-15,09:31:00,09:31:00,2,2 32 | am-16,09:45:00,09:45:00,1,1 33 | am-16,09:46:00,09:46:00,2,2 34 | pm-1,14:00:00,14:00:00,1,1 35 | pm-1,14:01:00,14:01:00,2,2 36 | pm-2,14:15:00,14:15:00,1,1 37 | pm-2,14:16:00,14:16:00,2,2 38 | pm-3,14:30:00,14:30:00,1,1 39 | pm-3,14:31:00,14:31:00,2,2 40 | pm-4,14:45:00,14:45:00,1,1 41 | pm-4,14:46:00,14:46:00,2,2 42 | pm-5,15:00:00,15:00:00,1,1 43 | pm-5,15:01:00,15:01:00,2,2 44 | pm-6,15:15:00,15:15:00,1,1 45 | pm-6,15:16:00,15:16:00,2,2 46 | pm-7,15:30:00,15:30:00,1,1 47 | pm-7,15:31:00,15:31:00,2,2 48 | pm-8,15:45:00,15:45:00,1,1 49 | pm-8,15:46:00,15:46:00,2,2 50 | pm-9,16:00:00,16:00:00,1,1 51 | pm-9,16:01:00,16:01:00,2,2 52 | pm-10,16:15:00,16:15:00,1,1 53 | pm-10,16:16:00,16:16:00,2,2 54 | pm-11,16:30:00,16:30:00,1,1 55 | pm-11,16:31:00,16:31:00,2,2 56 | pm-12,16:45:00,16:45:00,1,1 57 | pm-12,16:46:00,16:46:00,2,2 58 | pm-13,17:00:00,17:00:00,1,1 59 | pm-13,17:01:00,17:01:00,2,2 60 | pm-14,17:15:00,17:15:00,1,1 61 | pm-14,17:16:00,17:16:00,2,2 62 | pm-15,17:30:00,17:30:00,1,1 63 | pm-15,17:31:00,17:31:00,2,2 64 | pm-16,17:45:00,17:45:00,1,1 65 | pm-16,17:46:00,17:46:00,2,2 66 | -------------------------------------------------------------------------------- /lib/util.js: -------------------------------------------------------------------------------- 1 | var util = {}; 2 | 3 | util.getConnectionString = function(sequelize) { 4 | return sequelize.getDialect() + '://' + 5 | sequelize.config.username + ':' + 6 | sequelize.config.password + '@' + 7 | sequelize.config.host + ':' + 8 | sequelize.config.port + '/' + 9 | sequelize.config.database; 10 | } 11 | 12 | util.makeStreamerConfig = function(model) { 13 | var dialect = model.sequelize.getDialect() 14 | var schema = model.sequelize.options.schema 15 | var config = { 16 | tableName: model.tableName, 17 | columns: Object.keys(model.attributes), 18 | primaryKey: Object.keys(model.primaryKeys)[0] 19 | }; 20 | 21 | if (schema) { 22 | if (dialect === 'postgres') { 23 | config.tableName = '"' + schema + '"."' + model.tableName + '"' 24 | } else { 25 | config.tableName = '`' + schema + '.' + model.tableName + '`' 26 | } 27 | } 28 | 29 | if (dialect === 'sqlite') { 30 | config.sqliteStorage = model.sequelize.options.storage 31 | } else { 32 | config.dbConnString = util.getConnectionString(model.sequelize) 33 | } 34 | 35 | if(['mysql', 'sqlite'].indexOf(dialect) > -1) { 36 | config.deferUntilEnd = true; 37 | } 38 | 39 | // special case for fare rules 40 | // since I haven't found a way for sequelize to define tables with a composite primary key 41 | // that may have columns with null values, delete the auto-generated id column 42 | if(model.tableName === 'fare_rule') { 43 | config.columns.splice(config.columns.indexOf('id'), 1); 44 | if(dialect === 'sqlite' && !schema) { 45 | config.deferUntilEnd = false 46 | } 47 | } 48 | 49 | return config; 50 | } 51 | 52 | util.makeModelReference = function (sequelize, modelName) { 53 | if (sequelize.options.schema) { 54 | modelName = { 55 | schema: sequelize.options.schema, 56 | tableName: modelName 57 | } 58 | } 59 | return modelName 60 | } 61 | 62 | util.makeTableOptions = function(sequelize, options) { 63 | if (sequelize.options.schema) { 64 | options.schema = sequelize.options.schema 65 | } 66 | 67 | return options 68 | } 69 | 70 | module.exports = util; 71 | -------------------------------------------------------------------------------- /lib/download.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | 4 | var async = require('async') 5 | var download = require('download') 6 | var Client = require('ftp') 7 | var parse = require('url-parse') 8 | var rimraf = require('rimraf') 9 | 10 | module.exports = function (gtfsUrl, downloadsDir, callback) { 11 | var err 12 | if (!gtfsUrl) { 13 | err = new Error('GTFS download url not specified.') 14 | return callback(err) 15 | } 16 | 17 | if (!downloadsDir) { 18 | err = Error('GTFS download directory not specified.') 19 | return callback(err) 20 | } 21 | 22 | try { 23 | fs.mkdirSync(downloadsDir) 24 | } catch (e) { 25 | if (e.code !== 'EEXIST') { 26 | return callback(e) 27 | } 28 | } 29 | 30 | // determine download protocol 31 | var parsed = parse(gtfsUrl) 32 | 33 | if (['http:', 'https:'].indexOf(parsed.protocol) > -1) { 34 | // download using http(s) the gtfs and save to the downloads folder 35 | var dlFile = downloadsDir + '/google_transit.zip' 36 | 37 | console.log('Downloading', gtfsUrl) 38 | async.auto({ 39 | rm: function (cb) { 40 | rimraf(dlFile, cb) 41 | }, 42 | dl: ['rm', function (results, cb) { 43 | download(gtfsUrl, downloadsDir, { filename: 'google_transit.zip' }) 44 | .then(() => cb()) 45 | .catch(cb) 46 | }] 47 | }, callback) 48 | } else if (parsed.protocol === 'ftp:') { 49 | // download using ftp 50 | var c = new Client() 51 | c.on('ready', function () { 52 | console.log('downloading ' + parsed.pathname) 53 | c.get(parsed.pathname, function (err, stream) { 54 | if (err) { 55 | callback(err) 56 | return 57 | } 58 | stream.once('close', function () { 59 | c.end() 60 | callback() 61 | }) 62 | stream.pipe(fs.createWriteStream(path.join(downloadsDir, 'google_transit.zip'))) 63 | }) 64 | }) 65 | console.log('connecting to ftp') 66 | // connect to ftp 67 | c.connect({ 68 | host: parsed.host, 69 | user: parsed.username, 70 | password: parsed.password 71 | }) 72 | } else { 73 | var error = new Error('unsupported download protocol') 74 | return callback(error) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /tests/util.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | 4 | var yazl = require('yazl') 5 | 6 | if (typeof Promise === 'undefined') { 7 | global.Promise = require('promise-polyfill') 8 | } 9 | 10 | /** 11 | * Helper to drop the db before or after a test 12 | */ 13 | function dropDb (gtfs, done) { 14 | var db = gtfs.connectToDatabase() 15 | db.sequelize.drop() 16 | .then(() => { 17 | console.log('dropped') 18 | return db.sequelize.close() 19 | }) 20 | .then(() => { 21 | console.log('closed') 22 | done() 23 | }) 24 | .catch(done) 25 | } 26 | 27 | /** 28 | * Get gtfs config for a test suite 29 | */ 30 | function getConfig () { 31 | var config = { 32 | downloadsDir: 'tests/feeds', 33 | maxLoadTimeout: 60000, 34 | sequelizeOptions: { 35 | logging: false 36 | } 37 | } 38 | 39 | switch (process.env.DIALECT) { 40 | case 'mysql-spatial': 41 | config.spatial = true 42 | config.database = 'mysql://gtfs_sequelize:gtfs_sequelize@localhost:3306/gtfs_sequelize_test' 43 | break 44 | case 'mysql': 45 | config.database = 'mysql://gtfs_sequelize:gtfs_sequelize@localhost:3306/gtfs_sequelize_test' 46 | break 47 | case 'postgis': 48 | config.spatial = true 49 | config.database = 'postgres://gtfs_sequelize:gtfs_sequelize@localhost:5432/gtfs_sequelize_test' 50 | break 51 | case 'postgres': 52 | config.database = 'postgres://gtfs_sequelize:gtfs_sequelize@localhost:5432/gtfs_sequelize_test' 53 | break 54 | case 'sqlite': 55 | var sqliteStorage = path.resolve(__dirname) + '/temp.sqlite' 56 | config.sequelizeOptions.dialect = 'sqlite' 57 | config.sequelizeOptions.storage = sqliteStorage 58 | break 59 | default: 60 | throw new Error('Invalid DIALECT') 61 | } 62 | 63 | return config 64 | } 65 | 66 | var zipMockAgency = function (callback) { 67 | // make a gtfs zip file from the mock data and put it in the downloads directory 68 | var zipfile = new yazl.ZipFile() 69 | 70 | // add all files in mock agency folder 71 | var zipSourceDir = 'tests/feeds/mock_agency' 72 | fs.readdirSync(zipSourceDir) 73 | .forEach(function (file) { 74 | zipfile.addFile(path.join(zipSourceDir, file), file) 75 | }) 76 | 77 | try { 78 | fs.mkdirSync('downloads') 79 | } catch (e) { 80 | if (e.code !== 'EEXIST') { 81 | callback(e) 82 | } 83 | } 84 | 85 | zipfile.outputStream.pipe(fs.createWriteStream(path.join('downloads', 'mock_gtfs.zip'))).on('close', function () { 86 | callback() 87 | }) 88 | 89 | zipfile.end() 90 | } 91 | 92 | module.exports = { 93 | dropDb: dropDb, 94 | getConfig: getConfig, 95 | zipMockAgency: zipMockAgency 96 | } 97 | -------------------------------------------------------------------------------- /tests/download.test.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | 3 | var assert = require('chai').assert, 4 | nock = require('nock'), 5 | rimraf = require('rimraf'); 6 | 7 | var util = require('./util.js') 8 | 9 | var GTFS = require('../index.js') 10 | 11 | var makeFileIsDownloadedVerificationFn = function(callback) { 12 | 13 | var aWhileAgo = (new Date()).getTime() - 100000 14 | 15 | return function(err) { 16 | try { 17 | assert.isNotOk(err) 18 | } catch(e) { 19 | return callback(e) 20 | } 21 | 22 | fs.stat('./downloads/google_transit.zip', function(err, stats) { 23 | 24 | try { 25 | assert.isNotOk(err) 26 | assert.isAbove(stats.ctime.getTime(), aWhileAgo, 'file update time is before test!') 27 | } catch(e) { 28 | return callback(e) 29 | } 30 | 31 | callback() 32 | 33 | }) 34 | } 35 | } 36 | 37 | describe('download', function() { 38 | 39 | var BASE_URL = 'http://example.com/', 40 | NOCK_HOST = 'http://example.com' 41 | 42 | var gtfs, promise; 43 | 44 | before(function(done) { 45 | util.zipMockAgency(done) 46 | }) 47 | 48 | afterEach(function(done) { 49 | rimraf('./downloads', done) 50 | }) 51 | 52 | describe('data sanity checks', function() { 53 | 54 | it('should fail without url being provided', function(done) { 55 | gtfs = GTFS({}) 56 | gtfs.downloadGtfs(function(err) { 57 | assert.isOk(err); 58 | assert.property(err, 'message'); 59 | assert.equal(err.message, 'GTFS download url not specified.'); 60 | done(); 61 | }) 62 | }) 63 | 64 | it('should fail without downloads directory being provided', function(done) { 65 | gtfs = GTFS({ gtfsUrl: 'http://example.com/gtfs.zip' }) 66 | gtfs.downloadGtfs(function(err) { 67 | assert.isOk(err); 68 | assert.property(err, 'message'); 69 | assert.equal(err.message, 'GTFS download directory not specified.'); 70 | done(); 71 | }) 72 | }) 73 | 74 | it('should fail with invalid protocol url being provided', function(done) { 75 | gtfs = GTFS({ 76 | gtfsUrl: 'xyz://example.com/gtfs.zip', 77 | downloadsDir: 'downloads' 78 | }) 79 | gtfs.downloadGtfs(function(err) { 80 | assert.isOk(err); 81 | assert.property(err, 'message'); 82 | assert.equal(err.message, 'unsupported download protocol'); 83 | done(); 84 | }) 85 | }) 86 | }) 87 | 88 | it('gtfs should download via http', function(done) { 89 | 90 | var scope = nock(NOCK_HOST) 91 | .get('/google_transit.zip') 92 | .replyWithFile(200, './downloads/mock_gtfs.zip') 93 | 94 | var doneHelper = function(err) { 95 | scope.done() 96 | done(err) 97 | } 98 | 99 | gtfs = GTFS({ 100 | gtfsUrl: BASE_URL + 'google_transit.zip', 101 | downloadsDir: 'downloads' 102 | }); 103 | 104 | gtfs.downloadGtfs(makeFileIsDownloadedVerificationFn(doneHelper)) 105 | }) 106 | }) 107 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # gtfs-sequelize 2 | 3 | [![npm version](https://badge.fury.io/js/gtfs-sequelize.svg)](http://badge.fury.io/js/gtfs-sequelize) [![Build Status](https://travis-ci.org/evansiroky/gtfs-sequelize.svg?branch=master)](https://travis-ci.org/evansiroky/gtfs-sequelize) [![Dependency Status](https://david-dm.org/evansiroky/gtfs-sequelize.svg)](https://david-dm.org/evansiroky/gtfs-sequelize) [![Test Coverage](https://codeclimate.com/github/evansiroky/gtfs-sequelize/badges/coverage.svg)](https://codeclimate.com/github/evansiroky/gtfs-sequelize/coverage) 4 | 5 | A model of the static GTFS using [sequelize.js](http://sequelizejs.com/). 6 | 7 | Currently works only with PostgreSQL (including PostGIS), MySQL (with spatial capabilities) and sqlite (but NOT spatialite). 8 | 9 | ## Table of Contents 10 | 11 | * [Installation](#installation) 12 | * [API](#api) 13 | 14 | ## Installation 15 | 16 | In order to use this library, you must also install the additional libraries in your project depending on the database that you use. 17 | 18 | ### PostgreSQL 19 | 20 | npm install pg --save 21 | npm install pg-copy-streams --save 22 | npm install pg-query-stream --save 23 | npm install pg-hstore --save 24 | 25 | #### With pg and node v0.10.x 26 | 27 | You must also install the package `promise-polyfill` and write additional code. See [here](https://github.com/brianc/node-postgres/issues/1057) for more details. 28 | 29 | ### MySQL 30 | 31 | npm install mysql --save 32 | npm install streamsql --save 33 | 34 | ### SQLite 35 | 36 | npm install sqlite3 --save 37 | npm install streamsql --save 38 | 39 | Usage with SQLite requires that sqlite is installed and is available via a unix command line. 40 | 41 | ## API: 42 | 43 | ### GTFS(options) 44 | 45 | Create a new GTFS API. 46 | 47 | Example: 48 | 49 | ```js 50 | var GTFS = require('gtfs-sequelize'); 51 | 52 | var pgConfig = { 53 | database: 'postgres://gtfs_sequelize:gtfs_sequelize@localhost:5432/gtfs-sequelize-test', 54 | downloadsDir: 'downloads', 55 | gtfsFileOrFolder: 'google_transit.zip', 56 | spatial: true, 57 | sequelizeOptions: { 58 | logging: false 59 | } 60 | } 61 | 62 | var gtfs = GTFS(pgConfig); 63 | gtfs.loadGtfs(function() { 64 | //database loading has finished callback 65 | }); 66 | ``` 67 | 68 | #### options 69 | 70 | | Key | Value | 71 | | -- | -- | 72 | | database | A database connection string. You must specify a user and a database in your connection string. The database must already exist, but the tables within the db do not need to exist. | 73 | | downloadsDir | The directory where you want the feed zip fils downloaded to or where you're going to read the feed read from. | 74 | | gtfsFileOrFolder | The (zip) file or folder to load the gtfs from | 75 | | interpolateStopTimes | Default is undefined. If true, after loading the stop_times table, all stop_times with undefined arrival and departure times will be updated to include interpolated arrival and departure times. | 76 | | sequelizeOptions | Options to pass to sequelize. Note: to use a specific schema you'll want to pass something like this: `{ schema: 'your_schema' }` | 77 | | spatial | Default is undefined. If true, spatial tables for the shapes and stops will be created. | 78 | 79 | ### gtfs.connectToDatabase() 80 | 81 | Return a sequelize api of the database. 82 | 83 | Example: 84 | 85 | ```js 86 | var db = gtfs.connectToDatabase() 87 | 88 | db.stop.findAll() 89 | .then(stops => { 90 | console.log(stops) 91 | }) 92 | ``` 93 | 94 | ### gtfs.downloadGtfs(callback) 95 | 96 | If a url is provided, the feed will be attempted to be downloaded. Works with `http`, `https` and `ftp`. 97 | 98 | ### gtfs.interpolateStopTimes(callback) 99 | 100 | Interpolate stop_times with undefined arrival and departure times. If you load a gtfs with the `interpolateStopTimes` flag set to true, you don't need to call this. 101 | 102 | ### gtfs.loadGtfs(callback) 103 | 104 | Load the gtfs into the database. 105 | -------------------------------------------------------------------------------- /tests/db.operations.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('chai').assert 2 | const rimraf = require('rimraf') 3 | 4 | const GTFS = require('../index.js') 5 | 6 | const util = require('./util.js') 7 | 8 | // prepare config for tests 9 | describe(process.env.DIALECT, function () { 10 | describe('operations', function () { 11 | describe('interpolated stop times', () => { 12 | const expectedStopTimesNoShapes = [ 13 | { 14 | arrival_time: 28800, 15 | departure_time: 28800 16 | }, 17 | { 18 | arrival_time: 28880, 19 | departure_time: 28880 20 | }, 21 | { 22 | arrival_time: 28960, 23 | departure_time: 28960 24 | }, 25 | { 26 | arrival_time: 29040, 27 | departure_time: 29040 28 | }, 29 | { 30 | arrival_time: 29160, 31 | departure_time: 29160 32 | }, 33 | { 34 | arrival_time: 29280, 35 | departure_time: 29280 36 | } 37 | ] 38 | 39 | const expectedStopTimesWithShapes = [ 40 | { 41 | arrival_time: 28800, 42 | departure_time: 28800 43 | }, 44 | { 45 | arrival_time: 28903, 46 | departure_time: 28903 47 | }, 48 | { 49 | arrival_time: 28954, 50 | departure_time: 28954 51 | }, 52 | { 53 | arrival_time: 29040, 54 | departure_time: 29040 55 | }, 56 | { 57 | arrival_time: 29172, 58 | departure_time: 29172 59 | }, 60 | { 61 | arrival_time: 29280, 62 | departure_time: 29280 63 | } 64 | ] 65 | 66 | const testConfigs = [ 67 | { 68 | describe: 'no shapes', 69 | expectedStopTimes: expectedStopTimesNoShapes, 70 | gtfsFileOrFolder: 'interpolated_no_shapes' 71 | }, 72 | { 73 | describe: 'no shapes, with schema', 74 | expectedStopTimes: expectedStopTimesNoShapes, 75 | gtfsFileOrFolder: 'interpolated_no_shapes', 76 | schema: 'test_schema' 77 | }, 78 | { 79 | describe: 'with shapes', 80 | expectedStopTimes: expectedStopTimesWithShapes, 81 | gtfsFileOrFolder: 'interpolated_with_shapes' 82 | } 83 | ] 84 | 85 | testConfigs.forEach(testConfig => { 86 | describe(testConfig.describe, () => { 87 | const config = util.getConfig() 88 | 89 | config.gtfsFileOrFolder = testConfig.gtfsFileOrFolder 90 | if (testConfig.schema) { 91 | if (process.env.DIALECT === 'sqlite') { 92 | console.warn('skipping sqlite test w/ schema cause I dunno why it\'s not working') 93 | return 94 | } 95 | config.sequelizeOptions.schema = testConfig.schema 96 | } 97 | 98 | const gtfs = GTFS(config) 99 | 100 | after(function (done) { 101 | const sqliteStorage = config.sequelizeOptions.storage 102 | if (sqliteStorage) { 103 | console.log('remove sqlite storage') 104 | rimraf(sqliteStorage, done) 105 | } else { 106 | util.dropDb(gtfs, done) 107 | } 108 | }) 109 | 110 | before(done => { 111 | this.timeout(config.maxLoadTimeout) 112 | gtfs.loadGtfs(done) 113 | }) 114 | 115 | it('should correctly calculate interpolated stop times', (done) => { 116 | this.timeout(config.maxLoadTimeout) 117 | 118 | // interpolate the stop times 119 | gtfs.interpolateStopTimes(err => { 120 | if (err) return done(err) 121 | 122 | const db = gtfs.connectToDatabase() 123 | db.stop_time 124 | .findAll({ 125 | where: { 126 | trip_id: '1' 127 | }, 128 | order: [ 129 | ['stop_sequence', 'ASC'] 130 | ] 131 | }) 132 | .then(stopTimes => { 133 | for (let i = 0; i < stopTimes.length; i++) { 134 | const expectedStopTime = testConfig.expectedStopTimes[i] 135 | const actualStopTime = stopTimes[i] 136 | assert.strictEqual( 137 | Math.round(actualStopTime.arrival_time), 138 | expectedStopTime.arrival_time 139 | ) 140 | assert.strictEqual( 141 | Math.round(actualStopTime.departure_time), 142 | expectedStopTime.departure_time 143 | ) 144 | } 145 | done() 146 | }) 147 | .catch(done) 148 | }) 149 | }) 150 | }) 151 | }) 152 | }) 153 | }) 154 | }) 155 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "gtfs-sequelize", 3 | "version": "0.0.0-semantically-release", 4 | "description": "A model for the static GTFS using sequelize.", 5 | "main": "index.js", 6 | "engines": { 7 | "node": ">=6" 8 | }, 9 | "scripts": { 10 | "lint": "./node_modules/.bin/eslint lib", 11 | "test": "npm run test-download && npm run test-mysql && npm run test-mysql-spatial && npm run test-postgres && npm run test-postgis && npm run test-sqlite", 12 | "test-download": "mocha tests/download.test.js", 13 | "test-mysql": "cross-env DIALECT=mysql mocha tests/db.*.test.js", 14 | "test-mysql-spatial": "cross-env DIALECT=mysql-spatial mocha tests/db.*.test.js", 15 | "test-postgres": "cross-env DIALECT=postgres mocha tests/db.*.test.js", 16 | "test-postgis": "cross-env DIALECT=postgis mocha tests/db.*.test.js", 17 | "test-sqlite": "cross-env DIALECT=sqlite mocha tests/db.*.test.js", 18 | "cover-all": "npm run cover-download && npm run cover-mysql && npm run cover-mysql-spatial && npm run cover-postgres && npm run cover-postgis && npm run cover-sqlite && npm run merge-coverage", 19 | "precover-download": "rimraf coverage && rimraf coverage-download", 20 | "cover-download": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- tests/download.test.js", 21 | "postcover-download": "cross-env DIALECT=download node scripts/renameCoverageFolder.js", 22 | "precover-mysql": "rimraf coverage && rimraf coverage-mysql", 23 | "cover-mysql": "cross-env DIALECT=mysql ./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- tests/db.*.test.js", 24 | "postcover-mysql": "cross-env DIALECT=mysql node scripts/renameCoverageFolder.js", 25 | "precover-mysql-spatial": "rimraf coverage && rimraf coverage-mysql-spatial", 26 | "cover-mysql-spatial": "cross-env DIALECT=mysql-spatial ./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- tests/db.*.test.js", 27 | "postcover-mysql-spatial": "cross-env DIALECT=mysql-spatial node scripts/renameCoverageFolder.js", 28 | "precover-postgres": "rimraf coverage && rimraf coverage-postgres", 29 | "cover-postgres": "cross-env DIALECT=postgres ./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- tests/db.*.test.js", 30 | "postcover-postgres": "cross-env DIALECT=postgres node scripts/renameCoverageFolder.js", 31 | "precover-postgis": "rimraf coverage && rimraf coverage-postgis", 32 | "cover-postgis": "cross-env DIALECT=postgis ./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- tests/db.*.test.js", 33 | "postcover-postgis": "cross-env DIALECT=postgis node scripts/renameCoverageFolder.js", 34 | "precover-sqlite": "rimraf coverage && rimraf coverage-sqlite", 35 | "cover-sqlite": "cross-env DIALECT=sqlite ./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- tests/db.*.test.js", 36 | "postcover-sqlite": "cross-env DIALECT=sqlite node scripts/renameCoverageFolder.js", 37 | "premerge-coverage": "node scripts/preMerge.js", 38 | "merge-coverage": "./node_modules/.bin/lcov-result-merger \"coverage-*/lcov.info\" \"coverage/lcov.info\"", 39 | "codeclimate-send": "cross-env CODECLIMATE_REPO_TOKEN=783da970f89b4c64bcd7a9ac3240c8671dc2838688ccaa90bd642cf27ec359dc ./node_modules/.bin/codeclimate-test-reporter < coverage/lcov.info", 40 | "codeclimate": "npm run cover-all && npm run codeclimate-send", 41 | "semantic-release": "semantic-release pre && npm publish && semantic-release post" 42 | }, 43 | "repository": { 44 | "type": "git", 45 | "url": "https://github.com/evansiroky/gtfs-sequelize.git" 46 | }, 47 | "keywords": [ 48 | "gtfs", 49 | "transit", 50 | "database" 51 | ], 52 | "author": "Evan Siroky", 53 | "license": "MIT", 54 | "bugs": { 55 | "url": "https://github.com/evansiroky/gtfs-sequelize/issues" 56 | }, 57 | "homepage": "https://github.com/evansiroky/gtfs-sequelize#readme", 58 | "dependencies": { 59 | "async": "^2.0.0-rc.5", 60 | "csvtojson": "^1.1.9", 61 | "db-streamer": "^1.2.1", 62 | "download": "^6.2.5", 63 | "ftp": "^0.3.10", 64 | "moment": "^2.10.6", 65 | "nock": "^9.0.0", 66 | "rimraf": "^2.4.3", 67 | "sequelize": "3.27.0", 68 | "unzip2": "^0.2.5", 69 | "url-parse": "^1.0.2", 70 | "uuid": "^3.0.0" 71 | }, 72 | "devDependencies": { 73 | "chai": "^3.5.0", 74 | "codeclimate-test-reporter": "^0.4.0", 75 | "cross-env": "^3.1.3", 76 | "cz-conventional-changelog": "^1.1.6", 77 | "eslint": "^4.14.0", 78 | "eslint-config-standard": "^11.0.0-beta.0", 79 | "eslint-plugin-import": "^2.8.0", 80 | "eslint-plugin-node": "^5.2.1", 81 | "eslint-plugin-promise": "^3.6.0", 82 | "eslint-plugin-standard": "^3.0.1", 83 | "istanbul": "^0.4.2", 84 | "lcov-result-merger": "^1.0.2", 85 | "mocha": "^3.0.0", 86 | "mysql": "^2.11.1", 87 | "pg": "^6.0.0", 88 | "pg-copy-streams": "^1.0.0", 89 | "pg-hstore": "^2.3.2", 90 | "pg-query-stream": "^1.1.1", 91 | "promise-polyfill": "^6.0.0", 92 | "semantic-release": "^6.3.2", 93 | "sqlite3": "^3.1.8", 94 | "streamsql": "^0.8.5", 95 | "yazl": "^2.3.0" 96 | }, 97 | "config": { 98 | "commitizen": { 99 | "path": "./node_modules/cz-conventional-changelog" 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /tests/db.load.test.js: -------------------------------------------------------------------------------- 1 | var assert = require('chai').assert 2 | 3 | var GTFS = require('../index.js') 4 | 5 | var util = require('./util.js') 6 | 7 | // prepare config for tests 8 | describe(process.env.DIALECT, function () { 9 | describe('loading', function () { 10 | before(function (done) { 11 | // zip up the mock agency before tests 12 | util.zipMockAgency(done) 13 | }) 14 | 15 | afterEach(function (done) { 16 | // drop and create the database before each test 17 | var config = util.getConfig() 18 | var gtfs = GTFS(config) 19 | util.dropDb(gtfs, done) 20 | }) 21 | 22 | it('data should load from folder', function (done) { 23 | var config = util.getConfig() 24 | this.timeout(config.maxLoadTimeout) 25 | 26 | config.gtfsFileOrFolder = 'mock_agency' 27 | 28 | var gtfs = GTFS(config) 29 | gtfs.loadGtfs(done) 30 | }) 31 | 32 | it('data should load from zip folder', function (done) { 33 | var config = util.getConfig() 34 | this.timeout(config.maxLoadTimeout) 35 | 36 | config.downloadsDir = 'downloads' 37 | config.gtfsFileOrFolder = 'mock_gtfs.zip' 38 | 39 | var gtfs = GTFS(config) 40 | gtfs.loadGtfs(done) 41 | }) 42 | 43 | it('should fail gracefully with an invalid feed folder', function (done) { 44 | var config = util.getConfig() 45 | this.timeout(config.maxLoadTimeout) 46 | 47 | config.downloadsDir = 'tests/feeds' 48 | config.gtfsFileOrFolder = 'invalid_feed_1' 49 | 50 | var gtfs = GTFS(config) 51 | gtfs.loadGtfs(function (err) { 52 | assert.include(err.message, 'agency.txt <--- FILE NOT FOUND. THIS FILE IS REQUIRED. THIS FEED IS INVALID.') 53 | done() 54 | }) 55 | }) 56 | 57 | it('should fail gracefully when both calendar files are missing', function (done) { 58 | var config = util.getConfig() 59 | this.timeout(config.maxLoadTimeout) 60 | 61 | config.downloadsDir = 'tests/feeds' 62 | config.gtfsFileOrFolder = 'invalid_feed_2' 63 | 64 | var gtfs = GTFS(config) 65 | gtfs.loadGtfs(function (err) { 66 | assert.include(err.message, 'NEITHER calendar.txt OR calendar_dates.txt IS PRESENT IN THIS FEED. THIS FEED IS INVALID.') 67 | done() 68 | }) 69 | }) 70 | 71 | it('data should load from feed with wide range in calendar dates', function (done) { 72 | var config = util.getConfig() 73 | this.timeout(config.maxLoadTimeout) 74 | 75 | config.downloadsDir = 'tests/feeds' 76 | config.gtfsFileOrFolder = 'wide_range_in_calendar_dates' 77 | 78 | var gtfs = GTFS(config) 79 | gtfs.loadGtfs(function (err) { 80 | assert.isNotOk(err) 81 | 82 | // inspect the calendar table and expect to find christmas service 83 | var db = gtfs.connectToDatabase() 84 | db.calendar 85 | .findAll({ 86 | include: [db.calendar_date, db.trip], 87 | where: { 88 | service_id: 'christmas' 89 | } 90 | }) 91 | .then(function (data) { 92 | // existence of record 93 | assert.isAbove(data.length, 0) 94 | 95 | assert.strictEqual(data[0].end_date, '20151225') 96 | 97 | done() 98 | }) 99 | .catch(done) 100 | }) 101 | }) 102 | 103 | it('should load a gtfs with only calendar_dates.txt', function (done) { 104 | var config = util.getConfig() 105 | this.timeout(config.maxLoadTimeout) 106 | 107 | config.gtfsFileOrFolder = 'only_calendar_dates' 108 | 109 | var gtfs = GTFS(config) 110 | gtfs.loadGtfs(done) 111 | }) 112 | 113 | it('should load a gtfs without calendar_dates.txt', function (done) { 114 | var config = util.getConfig() 115 | this.timeout(config.maxLoadTimeout) 116 | 117 | config.gtfsFileOrFolder = 'only_calendar' 118 | 119 | var gtfs = GTFS(config) 120 | gtfs.loadGtfs(done) 121 | }) 122 | 123 | it('should load a gtfs and interpolate stop times', function (done) { 124 | var config = util.getConfig() 125 | this.timeout(config.maxLoadTimeout) 126 | 127 | config.gtfsFileOrFolder = 'interpolated_no_shapes' 128 | config.interpolateStopTimes = true 129 | 130 | var gtfs = GTFS(config) 131 | gtfs.loadGtfs(done) 132 | }) 133 | 134 | it('should load a gtfs and try to interpolate stop times that do not need interpolation', function (done) { 135 | var config = util.getConfig() 136 | this.timeout(config.maxLoadTimeout) 137 | 138 | config.gtfsFileOrFolder = 'only_calendar' 139 | config.interpolateStopTimes = true 140 | 141 | var gtfs = GTFS(config) 142 | gtfs.loadGtfs(done) 143 | }) 144 | 145 | describe('with schema', () => { 146 | afterEach(function (done) { 147 | // drop and create the database before each test 148 | var config = util.getConfig() 149 | config.sequelizeOptions.schema = 'test_schema' 150 | var gtfs = GTFS(config) 151 | util.dropDb(gtfs, done) 152 | }) 153 | 154 | it('should load into a specific schema', function (done) { 155 | var config = util.getConfig() 156 | this.timeout(config.maxLoadTimeout) 157 | 158 | config.gtfsFileOrFolder = 'mock_agency' 159 | config.sequelizeOptions.schema = 'test_schema' 160 | 161 | var gtfs = GTFS(config) 162 | 163 | gtfs.loadGtfs(done) 164 | }) 165 | }) 166 | }) 167 | }) 168 | -------------------------------------------------------------------------------- /lib/operations.js: -------------------------------------------------------------------------------- 1 | const async = require('async') 2 | const dbStreamer = require('db-streamer') 3 | 4 | const util = require('./util') 5 | 6 | /** 7 | * Make an update query to the db to set the interpolated times in 8 | * a particular range of a particular trip 9 | */ 10 | function updateInterpolatedTimes (cfg, callback) { 11 | const db = cfg.db 12 | const lastTimepoint = cfg.lastTimepoint 13 | const nextTimepoint = cfg.nextTimepoint 14 | const timeDiff = nextTimepoint.arrival_time - lastTimepoint.departure_time 15 | let literal 16 | // sqlite null is a string 17 | if (nextTimepoint.shape_dist_traveled && nextTimepoint.shape_dist_traveled !== 'NULL') { 18 | // calculate interpolation based off of distance ratios 19 | const distanceTraveled = nextTimepoint.shape_dist_traveled - lastTimepoint.shape_dist_traveled 20 | literal = `${lastTimepoint.departure_time} + 21 | ${timeDiff} * 22 | (shape_dist_traveled - ${lastTimepoint.shape_dist_traveled}) / 23 | ${distanceTraveled}` 24 | } else { 25 | // calculate interpolation based off of stop sequence ratios 26 | const numStopsPassed = nextTimepoint.stop_sequence - lastTimepoint.stop_sequence 27 | literal = `${lastTimepoint.departure_time} + 28 | ${timeDiff} * 29 | (stop_sequence - ${lastTimepoint.stop_sequence}) / 30 | ${numStopsPassed}` 31 | } 32 | const updateLiteral = db.sequelize.literal(literal) 33 | db.stop_time 34 | .update( 35 | { 36 | arrival_time: updateLiteral, 37 | departure_time: updateLiteral 38 | }, 39 | { 40 | where: { 41 | trip_id: lastTimepoint.trip_id, 42 | stop_sequence: { 43 | $gt: lastTimepoint.stop_sequence, 44 | $lt: nextTimepoint.stop_sequence 45 | } 46 | } 47 | } 48 | ) 49 | .then(() => { 50 | callback() 51 | }) 52 | .catch(callback) 53 | } 54 | 55 | /** 56 | * Calculate and assign an approximate arrival and departure time 57 | * at all stop_times that have an undefined arrival and departure time 58 | */ 59 | function interpolateStopTimes (db, callback) { 60 | console.log('interpolating stop times') 61 | const streamerConfig = util.makeStreamerConfig(db.trip) 62 | const querier = dbStreamer.getQuerier(streamerConfig) 63 | const maxUpdateConcurrency = db.trip.sequelize.getDialect() === 'sqlite' ? 1 : 100 64 | const updateQueue = async.queue(updateInterpolatedTimes, maxUpdateConcurrency) 65 | let isComplete = false 66 | let numUpdates = 0 67 | 68 | /** 69 | * Helper function to call upon completion of interpolation 70 | */ 71 | function onComplete (err) { 72 | if (err) { 73 | console.log('interpolation encountered an error: ', err) 74 | return callback(err) 75 | } 76 | // set is complete and create a queue drain function 77 | // however, a feed may not have any interpolated times, so 78 | // `isComplete` is set in case nothing is pushed to the queue 79 | isComplete = true 80 | updateQueue.drain = () => { 81 | console.log('interpolation completed successfully') 82 | callback(err) 83 | } 84 | } 85 | 86 | let rowTimeout 87 | 88 | /** 89 | * Helper function to account for stop_times that are completely interpolated 90 | */ 91 | function onRowComplete () { 92 | if (rowTimeout) { 93 | clearTimeout(rowTimeout) 94 | } 95 | if (isComplete && numUpdates === 0) { 96 | rowTimeout = setTimeout(() => { 97 | // check yet again, because interpolated times could've appeared since setting timeout 98 | if (numUpdates === 0) { 99 | console.log('interpolation completed successfully (no interpolations needed)') 100 | callback() 101 | } 102 | }, 10000) 103 | } 104 | } 105 | 106 | // TODO: fix this cause it doesn't work w/ sqlite with a schema for some reason 107 | const statement = `SELECT trip_id FROM ${streamerConfig.tableName}` 108 | querier.execute( 109 | statement, 110 | row => { 111 | // get all stop_times for trip 112 | db.stop_time 113 | .findAll({ 114 | where: { 115 | trip_id: row.trip_id 116 | } 117 | }) 118 | // iterate through stop times to determine null arrival or departure times 119 | .then(stopTimes => { 120 | let lastStopTime 121 | let lastTimepoint 122 | let lookingForNextTimepoint = false 123 | 124 | stopTimes.forEach(stopTime => { 125 | if (lookingForNextTimepoint) { 126 | // check if current stop time has a time 127 | // mysql null stop times are showin up as 0, which might be bug elsewhere 128 | // sqlite null shows up as 'NULL' 129 | if ( 130 | stopTime.arrival_time !== null && 131 | stopTime.arrival_time !== 'NULL' && 132 | stopTime.arrival_time >= lastTimepoint.departure_time 133 | ) { 134 | // found next timepoint 135 | // make update query to set interpolated times 136 | updateQueue.push({ 137 | db: db, 138 | lastTimepoint: lastTimepoint, 139 | nextTimepoint: stopTime 140 | }) 141 | numUpdates++ 142 | lookingForNextTimepoint = false 143 | } 144 | } else { 145 | // sqlite uninterpolated shows up ass 'NULL' 146 | if (stopTime.arrival_time === null || stopTime.arrival_time === 'NULL') { 147 | lastTimepoint = lastStopTime 148 | lookingForNextTimepoint = true 149 | } 150 | } 151 | lastStopTime = stopTime 152 | }) 153 | onRowComplete() 154 | }) 155 | .catch(onComplete) 156 | }, 157 | onComplete 158 | ) 159 | } 160 | 161 | module.exports = { 162 | interpolateStopTimes: interpolateStopTimes 163 | } 164 | -------------------------------------------------------------------------------- /tests/db.query.test.js: -------------------------------------------------------------------------------- 1 | var assert = require('chai').assert 2 | var moment = require('moment') 3 | var rimraf = require('rimraf') 4 | 5 | var util = require('./util.js') 6 | 7 | // prepare config for tests 8 | var config = util.getConfig() 9 | var gtfs 10 | var maxLoadTimeout = config.maxLoadTimeout 11 | 12 | describe(process.env.DIALECT, function () { 13 | describe('querying', function () { 14 | var db 15 | 16 | after(function (done) { 17 | var sqliteStorage = config.sequelizeOptions.storage 18 | if (sqliteStorage) { 19 | rimraf(sqliteStorage, done) 20 | } else { 21 | util.dropDb(gtfs, done) 22 | } 23 | }) 24 | 25 | before(function (done) { 26 | this.timeout(maxLoadTimeout) 27 | 28 | // load mock gtfs file before running querying tests 29 | config.downloadsDir = 'tests/feeds' 30 | config.gtfsFileOrFolder = 'mock_agency' 31 | config.sequelizeOptions.logging = false 32 | config.sequelizeOptions.schema = undefined 33 | 34 | gtfs = require('../index.js')(config) 35 | gtfs.loadGtfs(function (err) { 36 | if (err) return done(err) 37 | db = gtfs.connectToDatabase() 38 | done() 39 | }) 40 | }) 41 | 42 | it('agency query should work', function () { 43 | return db.agency 44 | .findAll({ include: [db.route] }) 45 | .then(function (data) { 46 | // existence of record 47 | assert.isAbove(data.length, 0) 48 | 49 | // uuid generated for agency_id when none provided 50 | assert.isAbove(data[0].agency_id.length, 0) 51 | 52 | // correct data 53 | assert.strictEqual(data[0].agency_name, 'West Coast Maglev') 54 | 55 | // associations 56 | assert.strictEqual(data[0].routes[0].route_long_name, 'Los Angeles - Seattle') 57 | }) 58 | }) 59 | 60 | it('calendar query should work', function () { 61 | return db.calendar 62 | .findAll({ 63 | include: [db.calendar_date, db.trip], 64 | where: { 65 | service_id: 'weekend' 66 | } 67 | }) 68 | .then(function (data) { 69 | // existence of record 70 | assert.isAbove(data.length, 0) 71 | 72 | // correct data 73 | // convert to utc because mysql has tz conversion issues 74 | assert.strictEqual(moment.utc(data[0].end_date).date(), 31) 75 | 76 | // associations 77 | // convert to utc because mysql has tz conversion issues 78 | assert.strictEqual(moment.utc(data[0].calendar_dates[0].date).date(), 25) 79 | assert.strictEqual(data[0].trips[0].trip_headsign, 'Seattle Weekend Express') 80 | }) 81 | }) 82 | 83 | it('calendar_date query should work', function () { 84 | return db.calendar_date 85 | .findAll({ 86 | include: [db.calendar], 87 | service_id: 'weekend' 88 | }) 89 | .then(function (data) { 90 | // existence of record 91 | assert.isAbove(data.length, 0) 92 | 93 | // correct data 94 | // convert to utc because mysql has tz conversion issues 95 | assert.strictEqual(moment.utc(data[0].date).date(), 25) 96 | 97 | // associations 98 | // convert to utc because mysql has tz conversion issues 99 | assert.strictEqual(moment.utc(data[0].calendar.end_date).date(), 31) 100 | }) 101 | }) 102 | 103 | it('fare_attribute query should work', function () { 104 | return db.fare_attribute 105 | .findAll({ 106 | include: [db.fare_rule], 107 | where: { 108 | fare_id: 'route_based_fare' 109 | } 110 | }) 111 | .then(function (data) { 112 | // existence of record 113 | assert.isAbove(data.length, 0) 114 | 115 | // correct data 116 | assert.strictEqual(data[0].fare_id, 'route_based_fare') 117 | 118 | // associations 119 | assert.strictEqual(data[0].fare_rules[0].fare_id, 'route_based_fare') 120 | }) 121 | }) 122 | 123 | describe('fare rules queries', function () { 124 | 125 | /* Don't fully understand how to get these working with sequelize yet 126 | it('route-based fare rule', function() { 127 | return db.fare_rule 128 | .findAll({ 129 | include: [db.fare_attribute, db.route], 130 | where: { 131 | fare_id: 'route_based_fare' 132 | } 133 | }) 134 | .then(function(data) { 135 | // existence of record 136 | assert.isAbove(data.length, 0); 137 | 138 | // correct data 139 | assert.strictEqual(data[0].fare_id, 'route_based_fare'); 140 | 141 | // associations 142 | assert.strictEqual(data[0].fare_attribute.price, 20); 143 | assert.strictEqual(data[0].route.route_id, 'LA-Seattle'); 144 | }); 145 | }); 146 | 147 | it('origin-destination-based fare rule', function() { 148 | return db.fare_rule 149 | .findAll({ 150 | include: [db.fare_attribute, { 151 | model: db.stop, 152 | as: 'origin_stop' 153 | }, { 154 | model: db.stop, 155 | as: 'destination_stop' 156 | }], 157 | where: { 158 | fare_id: 'origin_destination_fare' 159 | } 160 | }) 161 | .then(function(data) { 162 | // existence of record 163 | assert.isAbove(data.length, 0); 164 | 165 | // correct data 166 | assert.strictEqual(data[0].fare_id, 'origin_destination_fare'); 167 | 168 | // associations 169 | assert.strictEqual(data[0].fare_attribute.price, 30); 170 | assert.strictEqual(data[0].origin_stop.stop_name, 'San Francisco'); 171 | assert.strictEqual(data[0].destination_stop.stop_name, 'Portland'); 172 | }); 173 | }); 174 | 175 | it('contains-based fare rule', function() { 176 | return db.fare_rule 177 | .findAll({ 178 | include: [db.fare_attribute, { 179 | model: db.stop, 180 | as: 'contains_stop' 181 | }], 182 | where: { 183 | fare_id: 'contains_fare' 184 | } 185 | }) 186 | .then(function(data) { 187 | // existence of record 188 | assert.isAbove(data.length, 0); 189 | 190 | // correct data 191 | assert.strictEqual(data[0].fare_id, 'contains_fare'); 192 | 193 | // associations 194 | assert.strictEqual(data[0].fare_attribute.price, 40); 195 | assert.strictEqual(data[0].contains_stop.stop_name, 'Sacramento'); 196 | }); 197 | }); */ 198 | 199 | }) 200 | 201 | it('feed_info query should work', function () { 202 | return db.feed_info 203 | .findAll() 204 | .then(function (data) { 205 | // existence of record 206 | assert.isAbove(data.length, 0) 207 | 208 | // correct data 209 | assert.strictEqual(data[0].feed_publisher_name, 'mock factory') 210 | }) 211 | }) 212 | 213 | it('frequency query should work', function () { 214 | return db.frequency 215 | .findAll({ include: [db.trip] }) 216 | .then(function (data) { 217 | // existence of record 218 | assert.isAbove(data.length, 0) 219 | 220 | // correct data 221 | assert.strictEqual(data[0].headway_secs, 7200) 222 | 223 | // associations 224 | assert.strictEqual(data[0].trip.trip_headsign, 'Seattle Weekday Express') 225 | }) 226 | }) 227 | 228 | it('route query should work', function () { 229 | return db.route 230 | .findAll({ 231 | include: [db.trip, db.agency] 232 | }) 233 | .then(function (data) { 234 | // existence of record 235 | assert.isAbove(data.length, 0) 236 | 237 | // correct data 238 | assert.strictEqual(data[0].route_short_name, 'LA-SEA') 239 | 240 | // associations 241 | assert.strictEqual(data[0].agency.agency_name, 'West Coast Maglev') 242 | assert.isAbove(data[0].trips.length, 0) 243 | // assert.strictEqual(data[0].fare_rules[0].fare_id, 'route_based_fare'); 244 | }) 245 | }) 246 | 247 | it('shape query should work', function () { 248 | return db.shape 249 | .findAll({ 250 | where: { 251 | shape_pt_sequence: 1 252 | } 253 | }) 254 | .then(function (data) { 255 | // existence of record 256 | assert.isAbove(data.length, 0) 257 | 258 | // correct data 259 | assert.closeTo(data[0].shape_pt_lat, 34.056313, 0.001) 260 | }) 261 | }) 262 | 263 | describe('stop queries should work', function () { 264 | it('stop served by stop_time', function () { 265 | return db.stop 266 | .findAll({ 267 | include: [{ 268 | model: db.stop_time, 269 | where: { 270 | trip_id: 'weekend_trip' 271 | } 272 | }], 273 | where: { 274 | stop_id: 'LA' 275 | } 276 | }) 277 | .then(function (data) { 278 | // existence of record 279 | assert.isAbove(data.length, 0) 280 | 281 | // correct data 282 | assert.strictEqual(data[0].stop_name, 'Los Angeles') 283 | if (config.spatial) { 284 | assert.strictEqual(data[0].geom.type, 'Point') 285 | } 286 | 287 | // associations 288 | assert.strictEqual(data[0].stop_times[0].arrival_time, 43200) 289 | }) 290 | }) 291 | 292 | /* Don't fully understand how to get these working with sequelize yet 293 | it('stop with origin fare rule', function() { 294 | return db.stop 295 | .findAll({ 296 | include: [{ 297 | model: db.fare_rule, 298 | as: 'fare_rule_origins' 299 | }], 300 | where: { 301 | stop_id: 'SF' 302 | } 303 | }) 304 | .then(function(data) { 305 | // existence of record 306 | assert.isAbove(data.length, 0); 307 | 308 | // associations 309 | assert.strictEqual(data[0].fare_rule_origins[0].fare_id, 'origin_destination_fare'); 310 | }); 311 | }); 312 | 313 | it('stop with destination fare rule', function() { 314 | return db.stop 315 | .findAll({ 316 | include: [{ 317 | model: db.fare_rule, 318 | as: 'fare_rule_destinations' 319 | }], 320 | where: { 321 | stop_id: 'PDX' 322 | } 323 | }) 324 | .then(function(data) { 325 | // existence of record 326 | assert.isAbove(data.length, 0); 327 | 328 | // associations 329 | assert.strictEqual(data[0].fare_rule_destinations[0].fare_id, 'origin_destination_fare'); 330 | }); 331 | }); 332 | 333 | it('stop with contains fare rule', function() { 334 | return db.stop 335 | .findAll({ 336 | include: [{ 337 | model: db.fare_rule, 338 | as: 'fare_rule_contains' 339 | }], 340 | where: { 341 | stop_id: 'SAC1' 342 | } 343 | }) 344 | .then(function(data) { 345 | // existence of record 346 | assert.isAbove(data.length, 0); 347 | 348 | // associations 349 | assert.strictEqual(data[0].fare_rule_contains[0].fare_id, 'contains_fare'); 350 | }); 351 | }); 352 | 353 | it('stop with transfer from_stop', function() { 354 | return db.stop 355 | .findAll({ 356 | include: [{ 357 | model: db.transfer, 358 | as: 'transfer_from_stops' 359 | }], 360 | where: { 361 | stop_id: 'SAC1' 362 | } 363 | }) 364 | .then(function(data) { 365 | // existence of record 366 | assert.isAbove(data.length, 0); 367 | 368 | // associations 369 | assert.strictEqual(data[0].transfer_from_stops[0].transfer_type, 3); 370 | }); 371 | }); 372 | 373 | it('stop with transfer to_stop', function() { 374 | return db.stop 375 | .findAll({ 376 | include: [{ 377 | model: db.transfer, 378 | as: 'transfer_to_stops' 379 | }], 380 | where: { 381 | stop_id: 'SAC2' 382 | } 383 | }) 384 | .then(function(data) { 385 | // existence of record 386 | assert.isAbove(data.length, 0); 387 | 388 | // associations 389 | assert.strictEqual(data[0].transfer_to_stops[0].transfer_type, 3); 390 | }); 391 | }); 392 | */ 393 | }) 394 | 395 | it('stop_time query should work', function () { 396 | return db.stop_time 397 | .findAll({ 398 | include: [db.trip, db.stop], 399 | where: { 400 | trip_id: 'weekend_trip', 401 | stop_sequence: 1 402 | } 403 | }) 404 | .then(function (data) { 405 | // existence of record 406 | assert.isAbove(data.length, 0) 407 | 408 | // correct data 409 | assert.strictEqual(data[0].arrival_time, 43200) 410 | 411 | // associations 412 | assert.strictEqual(data[0].trip.trip_headsign, 'Seattle Weekend Express') 413 | assert.strictEqual(data[0].stop.stop_name, 'Los Angeles') 414 | }) 415 | }) 416 | 417 | it('transfer query should work', function () { 418 | return db.transfer 419 | .findAll({ 420 | include: [{ 421 | model: db.stop, 422 | as: 'from_stop' 423 | }, { 424 | model: db.stop, 425 | as: 'to_stop' 426 | }] 427 | }) 428 | .then(function (data) { 429 | // existence of record 430 | assert.isAbove(data.length, 0) 431 | 432 | // correct data 433 | assert.strictEqual(data[0].transfer_type, 3) 434 | 435 | // associations 436 | assert.strictEqual(data[0].from_stop.stop_name, 'Sacramento') 437 | assert.strictEqual(data[0].to_stop.stop_name, 'Sacramento-2') 438 | }) 439 | }) 440 | 441 | it('trip query should work', function () { 442 | var includes = [db.route, db.stop_time, db.calendar, db.frequency] 443 | if (config.spatial) { 444 | includes.push(db.shape_gis) 445 | } 446 | return db.trip 447 | .findAll({ 448 | include: includes, 449 | where: { 450 | trip_id: 'weekday_trips' 451 | } 452 | }) 453 | .then(function (data) { 454 | // existence of record 455 | assert.isAbove(data.length, 0) 456 | 457 | // correct data 458 | assert.strictEqual(data[0].trip_headsign, 'Seattle Weekday Express') 459 | 460 | // associations 461 | assert.strictEqual(data[0].route.route_long_name, 'Los Angeles - Seattle') 462 | assert.isAbove(data[0].stop_times.length, 0) 463 | // convert to utc because mysql has tz conversion issues 464 | assert.strictEqual(moment.utc(data[0].calendar.end_date).date(), 31) 465 | assert.strictEqual(data[0].frequencies[0].headway_secs, 7200) 466 | if (config.spatial) { 467 | assert.strictEqual(data[0].shape_gi.geom.type, 'LineString') 468 | } 469 | }) 470 | }) 471 | 472 | if (config.spatial) { 473 | it('shape_gis query should work', function () { 474 | return db.shape_gis 475 | .findAll({ 476 | include: [db.trip] 477 | }) 478 | .then(function (data) { 479 | // existence of record 480 | assert.isAbove(data.length, 0) 481 | 482 | // correct data 483 | assert.strictEqual(data[0].geom.type, 'LineString') 484 | 485 | // associations 486 | assert.isAbove(data[0].trips.length, 0) 487 | }) 488 | }) 489 | } 490 | }) 491 | }) 492 | -------------------------------------------------------------------------------- /lib/gtfsLoader.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | var path = require('path') 3 | 4 | var async = require('async') 5 | var csv = require('csvtojson') 6 | var dbStreamer = require('db-streamer') 7 | var moment = require('moment') 8 | var rimraf = require('rimraf') 9 | var unzip = require('unzip2') 10 | var uuid = require('uuid') 11 | 12 | const operations = require('./operations') 13 | var util = require('./util.js') 14 | 15 | var DATE_FORMAT = 'YYYYMMDD' 16 | var lastAgencyId, numAgencies 17 | let hasShapeTable = false 18 | 19 | // convert dateString to moment 20 | var toMoment = function (dateString) { 21 | return moment(dateString, DATE_FORMAT) 22 | } 23 | 24 | // convert timeString to int of seconds past midnight 25 | var toSecondsAfterMidnight = function (timeString) { 26 | if (!timeString) { 27 | return null 28 | } 29 | var timeArr = timeString.split(':') 30 | return parseInt(timeArr[0], 10) * 3600 + 31 | parseInt(timeArr[1], 10) * 60 + 32 | parseInt(timeArr[2]) 33 | } 34 | 35 | var loadGtfs = function (extractedFolder, db, isSpatial, interpolateStopTimes, callback) { 36 | numAgencies = 0 37 | lastAgencyId = null 38 | 39 | var dropAllTables = function (dropCallback) { 40 | var models = [db.feed_info, 41 | db.transfer, 42 | db.frequency, 43 | db.stop_time, 44 | db.calendar_date, 45 | db.fare_rule, 46 | db.fare_attribute, 47 | db.shape, 48 | db.trip, 49 | db.calendar, 50 | db.stop, 51 | db.route, 52 | db.agency] 53 | 54 | if (isSpatial) { 55 | models.splice(9, 0, db.sequelize.import('../models/spatial/shape_gis.js')) 56 | } 57 | 58 | console.log('dropping all gtfs tables') 59 | 60 | // iterate syncrhonously through each table 61 | async.eachSeries(models, 62 | function (model, modelCallback) { 63 | model.drop() 64 | .then(modelCallback) 65 | .catch(function (err) { 66 | console.error('error with model: ', model) 67 | modelCallback(err) 68 | }) 69 | }, 70 | function (err) { 71 | if (!err) console.log('tables dropped') 72 | else console.error('error: ', err.message) 73 | dropCallback(err) 74 | } 75 | ) 76 | } 77 | 78 | var postProcess = function (postProcessCallback) { 79 | const postprocesses = [] 80 | if (isSpatial) { 81 | var dialect = db.sequelize.options.dialect 82 | if (['postgres', 'mysql'].indexOf(dialect) === -1) { 83 | var err = Error('Spatial columns not supported for dialect ' + dialect + '.') 84 | return postProcessCallback(err) 85 | } 86 | postprocesses.push(makeStopGeom) 87 | postprocesses.push(makeShapeTable) 88 | } 89 | 90 | if (interpolateStopTimes) { 91 | postprocesses.push(doInterpolation) 92 | } 93 | 94 | async.series( 95 | postprocesses, 96 | function (err, results) { 97 | postProcessCallback(err) 98 | } 99 | ) 100 | } 101 | 102 | function doInterpolation (interpolationCallback) { 103 | operations.interpolateStopTimes(db, interpolationCallback) 104 | } 105 | 106 | var makeStopGeom = function (seriesCallback) { 107 | console.log('adding stop geometry') 108 | var model = db.sequelize.import('../models/spatial/stop.js') 109 | var dialect = db.sequelize.getDialect() 110 | var alterStopTableQuery, geomUpdateQuery 111 | 112 | db.stop = model 113 | 114 | if (dialect === 'mysql') { 115 | alterStopTableQuery = 'ALTER TABLE ' 116 | if (db.sequelize.options.schema) { 117 | alterStopTableQuery += '`' + db.sequelize.options.schema + '.stop`' 118 | } else { 119 | alterStopTableQuery += 'stop' 120 | } 121 | alterStopTableQuery += ' ADD geom Point;' 122 | geomUpdateQuery = db.sequelize.fn('Point', 123 | db.sequelize.col('stop_lon'), 124 | db.sequelize.col('stop_lat')) 125 | } else if (dialect === 'postgres') { 126 | alterStopTableQuery = 'SELECT AddGeometryColumn (' 127 | if (db.sequelize.options.schema) { 128 | alterStopTableQuery += "'" + db.sequelize.options.schema + "', 'stop'" 129 | } else { 130 | alterStopTableQuery += "'stop'" 131 | } 132 | alterStopTableQuery += ", 'geom', 4326, 'POINT', 2);" 133 | geomUpdateQuery = db.sequelize.literal('ST_SetSRID(ST_MakePoint(stop_lon, stop_lat), 4326)') 134 | } 135 | 136 | db.sequelize.query(alterStopTableQuery).then(function (results) { 137 | db.stop.update({ 138 | geom: geomUpdateQuery 139 | }, { 140 | where: { 141 | stop_lat: { 142 | gt: 0 143 | } 144 | } 145 | }).then(function () { 146 | console.log('stop table altered') 147 | seriesCallback() 148 | }) 149 | }) 150 | } 151 | 152 | var makeShapeTable = function (seriesCallback) { 153 | if (!hasShapeTable) { 154 | console.log('shape table does not exist, skipping creation of shape_gis table') 155 | return seriesCallback() 156 | } 157 | console.log('creating shape_gis table') 158 | var processShape = function (shapePoint, shapeCallback) { 159 | db.shape.findAll({ 160 | where: { 161 | shape_id: shapePoint.shape_id 162 | }, 163 | order: [['shape_pt_sequence', 'ASC']], 164 | attributes: ['shape_pt_lat', 'shape_pt_lon'] 165 | }).then(function (shapePoints) { 166 | var shapeGeom = { 167 | type: 'LineString', 168 | crs: { type: 'name', properties: { name: 'EPSG:4326' } }, 169 | coordinates: [] 170 | } 171 | 172 | var addPointToShape = function (pt) { 173 | shapeGeom.coordinates.push([pt.shape_pt_lon, pt.shape_pt_lat]) 174 | } 175 | 176 | for (var i = 0; i < shapePoints.length; i++) { 177 | addPointToShape(shapePoints[i]) 178 | } 179 | 180 | db.shape_gis.create({ 181 | shape_id: shapePoint.shape_id, 182 | geom: shapeGeom 183 | }).then(function () { 184 | shapeCallback() 185 | }) 186 | }) 187 | } 188 | 189 | var model = db.sequelize.import('../models/spatial/shape_gis.js') 190 | model.sync({ force: true }).then(function () { 191 | db.shape_gis = model 192 | db.trip = db.sequelize.import('../models/spatial/trip.js') 193 | db.shape.findAll({ 194 | attributes: [db.Sequelize.literal('DISTINCT shape_id'), 'shape_id'] 195 | }) 196 | .then(function (shapeIds) { 197 | async.each(shapeIds, processShape, seriesCallback) 198 | }) 199 | .catch(err => { 200 | if (err.message.indexOf('t exist') > -1) { 201 | // shape table was not created, so no need to process shapes 202 | seriesCallback() 203 | } else { 204 | seriesCallback(err) 205 | } 206 | }) 207 | }) 208 | } 209 | 210 | var loadAllFiles = function (loadingCallback) { 211 | function makeCallbackLogger (name, cb) { 212 | return (err) => { 213 | if (err) { 214 | console.error('error loading ' + name) 215 | } else { 216 | console.log('finished loading ' + name) 217 | } 218 | cb(err) 219 | } 220 | } 221 | 222 | function makeLoader (loadFn, name, dependent) { 223 | return dependent 224 | ? (results, cb) => { 225 | loadFn(extractedFolder, db, makeCallbackLogger(name, cb)) 226 | } 227 | : cb => { 228 | loadFn(extractedFolder, db, makeCallbackLogger(name, cb)) 229 | } 230 | } 231 | 232 | var dialect = db.sequelize.getDialect() 233 | // sqlite can't handle simultaneous loading apparently 234 | if (dialect === 'sqlite') { 235 | async.series( 236 | [ 237 | makeLoader(loadAgency, 'loadAgency', false), 238 | makeLoader(loadStops, 'loadStops', false), 239 | makeLoader(loadRoutes, 'loadRoutes', false), 240 | makeLoader(loadCalendar, 'loadCalendar', false), 241 | makeLoader(loadCalendarDates, 'loadCalendarDates', false), 242 | makeLoader(loadTrips, 'loadTrips', false), 243 | makeLoader(loadStopTimes, 'loadStopTimes', false), 244 | makeLoader(loadFareAttributes, 'loadFareAttributes', false), 245 | makeLoader(loadFareRules, 'loadFareRules', false), 246 | makeLoader(loadShapes, 'loadShapes', false), 247 | makeLoader(loadFrequencies, 'loadFrequencies', false), 248 | makeLoader(loadTransfers, 'loadTransfers', false), 249 | makeLoader(loadFeedInfo, 'loadFeedInfo', false) 250 | ], 251 | loadingCallback 252 | ) 253 | } else { 254 | async.auto( 255 | { 256 | loadAgency: makeLoader(loadAgency, 'loadAgency', false), 257 | loadCalendar: makeLoader(loadCalendar, 'loadCalendar', false), 258 | loadCalendarDates: [ 259 | 'loadCalendar', 260 | makeLoader(loadCalendarDates, 'loadCalendarDates', true) 261 | ], 262 | loadFareAttributes: makeLoader(loadFareAttributes, 'loadFareAttributes', false), 263 | loadFareRules: [ 264 | 'loadFareAttributes', 265 | makeLoader(loadFareRules, 'loadFareRules', true) 266 | ], 267 | loadFeedInfo: makeLoader(loadFeedInfo, 'loadFeedInfo', false), 268 | loadFrequencies: [ 269 | 'loadTrips', 270 | makeLoader(loadFrequencies, 'loadFrequencies', true) 271 | ], 272 | loadRoutes: [ 273 | 'loadAgency', 274 | makeLoader(loadRoutes, 'loadRoutes', true) 275 | ], 276 | loadShapes: makeLoader(loadShapes, 'loadShapes', false), 277 | loadStops: makeLoader(loadStops, 'loadStops', false), 278 | loadStopTimes: [ 279 | 'loadStops', 280 | 'loadTrips', 281 | makeLoader(loadStopTimes, 'loadStopTimes', true) 282 | ], 283 | loadTransfers: [ 284 | 'loadStops', 285 | makeLoader(loadTransfers, 'loadTransfers', true) 286 | ], 287 | loadTrips: [ 288 | 'loadRoutes', 289 | 'loadCalendarDates', 290 | makeLoader(loadTrips, 'loadTrips', true) 291 | ] 292 | }, 293 | loadingCallback 294 | ) 295 | } 296 | } 297 | 298 | // prepare loaders for synchronous execution 299 | async.series( 300 | [ 301 | dropAllTables, 302 | loadAllFiles, 303 | postProcess 304 | ], 305 | function (err) { 306 | callback(err) 307 | } 308 | ) 309 | } 310 | 311 | var insertCSVInTable = function (insertCfg, callback) { 312 | console.log('Processing ' + insertCfg.filename) 313 | 314 | // prepare transformer in case it doesn't exist 315 | var transformer = insertCfg.transformer 316 | ? insertCfg.transformer 317 | : line => line 318 | var transformer2 = function (line) { 319 | line = transformer(line) 320 | for (var k in line) { 321 | if (line[k] === '') { 322 | line[k] = null 323 | } 324 | } 325 | return line 326 | } 327 | 328 | // prepare processing function, but don't run it until file existance is confirmed 329 | var processTable = function () { 330 | insertCfg.model.sync({force: true}).then(function () { 331 | var streamInserterCfg = util.makeStreamerConfig(insertCfg.model) 332 | var inserter = dbStreamer.getInserter(streamInserterCfg) 333 | 334 | inserter.connect(function (err) { 335 | if (err) return callback(err) 336 | csv() 337 | .fromFile(insertCfg.filename) 338 | .on('json', line => { 339 | inserter.push(transformer2(line)) 340 | }) 341 | .on('done', err => { 342 | if (err) return callback(err) 343 | inserter.setEndHandler(callback) 344 | inserter.end() 345 | }) 346 | }) 347 | }) 348 | } 349 | 350 | /** 351 | * Check for file existance 352 | * - If it exists, process table 353 | * - If it doesn't exist, skip table 354 | * - If some other error, raise 355 | */ 356 | fs.stat(insertCfg.filename, 357 | function (err, stats) { 358 | if (!err || err.code !== 'ENOENT') { 359 | processTable() 360 | } else if (err.code === 'ENOENT') { 361 | if (insertCfg.fileIsRequired === true) { 362 | err = new Error(insertCfg.filename + ' <--- FILE NOT FOUND. THIS FILE IS REQUIRED. THIS FEED IS INVALID.') 363 | console.log(err) 364 | callback(err) 365 | } else { 366 | console.log(insertCfg.filename + ' <--- FILE NOT FOUND. SKIPPING.') 367 | callback() 368 | } 369 | } else { 370 | callback(err) 371 | } 372 | } 373 | ) 374 | } 375 | 376 | var loadAgency = function (extractedFolder, db, callback) { 377 | insertCSVInTable({ 378 | fileIsRequired: true, 379 | filename: path.join(extractedFolder, 'agency.txt'), 380 | model: db.agency, 381 | transformer: function (line) { 382 | if (!line.agency_id) { 383 | // if no agency id provided, generate a unique identifier 384 | line.agency_id = uuid.v4() 385 | } 386 | lastAgencyId = line.agency_id 387 | numAgencies++ 388 | return line 389 | } 390 | }, 391 | callback) 392 | } 393 | 394 | var loadStops = function (extractedFolder, db, callback) { 395 | insertCSVInTable({ 396 | fileIsRequired: true, 397 | filename: path.join(extractedFolder, 'stops.txt'), 398 | model: db.stop 399 | }, 400 | callback) 401 | } 402 | 403 | var loadRoutes = function (extractedFolder, db, callback) { 404 | insertCSVInTable({ 405 | fileIsRequired: true, 406 | filename: path.join(extractedFolder, 'routes.txt'), 407 | model: db.route, 408 | transformer: function (line) { 409 | if (numAgencies === 1) { 410 | if (!line.agency_id) { 411 | line.agency_id = lastAgencyId 412 | } 413 | } 414 | return line 415 | } 416 | }, 417 | callback) 418 | } 419 | 420 | // keep track of min and max dates found in case of omition of service_id in calendar.txt 421 | // or omission of calendar.txt altogether 422 | var minDateFound = null 423 | var maxDateFound = null 424 | var serviceIds = [] 425 | 426 | var loadCalendar = function (extractedFolder, db, callback) { 427 | // reset all these things on each feed load 428 | minDateFound = null 429 | maxDateFound = null 430 | serviceIds = [] 431 | 432 | // first perform a check on whether calendar.txt and/or calendar_dates.txt exists 433 | fs.stat( 434 | path.join(extractedFolder, 'calendar.txt'), 435 | function (err, stats) { 436 | if (!err || err.code !== 'ENOENT') { 437 | // calendar.txt exists 438 | insertCSVInTable( 439 | { 440 | filename: path.join(extractedFolder, 'calendar.txt'), 441 | model: db.calendar, 442 | transformer: function (line) { 443 | // keep track of date range 444 | var startDate = toMoment(line.start_date) 445 | var endDate = toMoment(line.end_date) 446 | 447 | if (!minDateFound || startDate.isBefore(minDateFound)) { 448 | minDateFound = moment(startDate) 449 | } 450 | 451 | if (!maxDateFound || endDate.isAfter(maxDateFound)) { 452 | maxDateFound = moment(endDate) 453 | } 454 | 455 | // keep track track of service ids found 456 | serviceIds.push(line.service_id) 457 | 458 | return line 459 | } 460 | }, 461 | callback 462 | ) 463 | } else if (err.code === 'ENOENT') { 464 | // calendar.txt does not exist 465 | // make sure calendar_dates.txt exists 466 | fs.stat( 467 | path.join(extractedFolder, 'calendar_dates.txt'), 468 | function (err) { 469 | if (!err || err.code !== 'ENOENT') { 470 | // calendar_dates.txt exists. 471 | // Create the calendar table and then continue 472 | db.calendar 473 | .sync({force: true}) 474 | .then(() => callback()) 475 | .catch(callback) 476 | } else if (err && err.code === 'ENOENT') { 477 | // calendar_dates does not exist. Feed invalid. 478 | err = new Error('NEITHER calendar.txt OR calendar_dates.txt IS PRESENT IN THIS FEED. THIS FEED IS INVALID.') 479 | console.log(err) 480 | callback(err) 481 | } else { 482 | callback(err) 483 | } 484 | } 485 | ) 486 | } else { 487 | // some other fs error occurred 488 | callback(err) 489 | } 490 | } 491 | ) 492 | } 493 | 494 | var loadCalendarDates = function (extractedFolder, db, callback) { 495 | var filename = path.join(extractedFolder, 'calendar_dates.txt') 496 | 497 | console.log('Processing ' + filename) 498 | 499 | // prepare processing function, but don't run it until file existance is confirmed 500 | var processCalendarDates = function () { 501 | db.calendar_date.sync({force: true}).then(function () { 502 | var serviceIdsNotInCalendar = [] 503 | var calendarInserterConfig = util.makeStreamerConfig(db.calendar) 504 | 505 | // create inserter for calendar dates 506 | var calendarInserter = dbStreamer.getInserter(calendarInserterConfig) 507 | calendarInserter.connect(function (err, client) { 508 | if (err) return callback(err) 509 | var calendarDateInserterConfig = util.makeStreamerConfig(db.calendar_date) 510 | calendarDateInserterConfig.client = client 511 | calendarDateInserterConfig.deferUntilEnd = true 512 | 513 | var calendarDateInserter = dbStreamer.getInserter(calendarDateInserterConfig) 514 | 515 | csv() 516 | .fromFile(filename) 517 | .on('json', line => { 518 | var exceptionMoment = toMoment(line.date) 519 | 520 | if (!minDateFound || exceptionMoment.isBefore(minDateFound)) { 521 | minDateFound = moment(exceptionMoment) 522 | } 523 | 524 | if (!maxDateFound || exceptionMoment.isAfter(maxDateFound)) { 525 | maxDateFound = moment(exceptionMoment) 526 | } 527 | 528 | calendarDateInserter.push(line) 529 | if (serviceIds.indexOf(line.service_id) === -1 && 530 | serviceIdsNotInCalendar.indexOf(line.service_id) === -1) { 531 | // service id not found in calendar.txt, add to list to parse at end 532 | serviceIdsNotInCalendar.push(line.service_id) 533 | } 534 | }) 535 | .on('done', err => { 536 | if (err) return callback(err) 537 | calendarDateInserter.setEndHandler(callback) 538 | 539 | // done parsing, resolve service ids not in calendar.txt 540 | calendarInserter.setEndHandler(function () { 541 | calendarDateInserter.end() 542 | }) 543 | 544 | var minDate = minDateFound.format(DATE_FORMAT) 545 | var maxDate = maxDateFound.format(DATE_FORMAT) 546 | 547 | if (serviceIdsNotInCalendar.length > 0) { 548 | async.each(serviceIdsNotInCalendar, 549 | function (serviceId, itemCallback) { 550 | calendarInserter.push({ 551 | service_id: serviceId, 552 | monday: 0, 553 | tuesday: 0, 554 | wednesday: 0, 555 | thursday: 0, 556 | friday: 0, 557 | saturday: 0, 558 | sunday: 0, 559 | start_date: minDate, 560 | end_date: maxDate 561 | }) 562 | itemCallback() 563 | }, 564 | function (err) { 565 | if (err) { 566 | callback(err) 567 | } else { 568 | calendarInserter.end() 569 | } 570 | } 571 | ) 572 | } else { 573 | calendarInserter.end() 574 | } 575 | }) 576 | }) 577 | }) 578 | } 579 | 580 | /** 581 | * Check for calendar.txt file existance 582 | * - If it exists, process table 583 | * - If it doesn't exist, callback with err if calendar not found or skip table 584 | * - If some other error, callback with err 585 | */ 586 | fs.stat(filename, 587 | function (err, stats) { 588 | if (!err || err.code !== 'ENOENT') { 589 | processCalendarDates() 590 | } else if (err.code === 'ENOENT') { 591 | console.log(filename + ' <--- FILE NOT FOUND. SKIPPING.') 592 | callback() 593 | } else { 594 | callback(err) 595 | } 596 | } 597 | ) 598 | } 599 | 600 | var loadTrips = function (extractedFolder, db, callback) { 601 | insertCSVInTable({ 602 | fileIsRequired: true, 603 | filename: path.join(extractedFolder, 'trips.txt'), 604 | model: db.trip 605 | }, 606 | callback) 607 | } 608 | 609 | var loadStopTimes = function (extractedFolder, db, callback) { 610 | insertCSVInTable({ 611 | fileIsRequired: true, 612 | filename: path.join(extractedFolder, 'stop_times.txt'), 613 | model: db.stop_time, 614 | transformer: function (line) { 615 | // change arrival and departure times into integer of seconds after midnight 616 | line.arrival_time = toSecondsAfterMidnight(line.arrival_time) 617 | line.departure_time = toSecondsAfterMidnight(line.departure_time) 618 | 619 | if (line.arrival_time !== null && line.departure_time === null) { 620 | line.departure_time = line.arrival_time 621 | } else if (line.departure_time !== null && line.arrival_time === null) { 622 | line.arrival_time = line.departure_time 623 | } 624 | return line 625 | } 626 | }, 627 | callback) 628 | } 629 | 630 | var loadFareAttributes = function (extractedFolder, db, callback) { 631 | insertCSVInTable({ 632 | filename: path.join(extractedFolder, 'fare_attributes.txt'), 633 | model: db.fare_attribute 634 | }, 635 | callback) 636 | } 637 | 638 | var loadFareRules = function (extractedFolder, db, callback) { 639 | insertCSVInTable({ 640 | filename: path.join(extractedFolder, 'fare_rules.txt'), 641 | model: db.fare_rule 642 | }, 643 | callback) 644 | } 645 | 646 | var loadShapes = function (extractedFolder, db, callback) { 647 | const filename = path.join(extractedFolder, 'shapes.txt') 648 | fs.stat( 649 | filename, 650 | function (err, stats) { 651 | if (!err) { 652 | // shapes.txt exists 653 | hasShapeTable = true 654 | insertCSVInTable( 655 | { 656 | filename: path.join(extractedFolder, 'shapes.txt'), 657 | model: db.shape 658 | }, 659 | callback 660 | ) 661 | } else if (err && err.code === 'ENOENT') { 662 | console.log(`${filename} <--- FILE NOT FOUND. SKIPPING.`) 663 | callback() 664 | } else if (err) { 665 | callback(err) 666 | } 667 | } 668 | ) 669 | } 670 | 671 | var loadFrequencies = function (extractedFolder, db, callback) { 672 | insertCSVInTable({ 673 | filename: path.join(extractedFolder, 'frequencies.txt'), 674 | model: db.frequency, 675 | transformer: function (line) { 676 | line.start_time = toSecondsAfterMidnight(line.start_time) 677 | line.end_time = toSecondsAfterMidnight(line.end_time) 678 | return line 679 | } 680 | }, 681 | callback) 682 | } 683 | 684 | var loadTransfers = function (extractedFolder, db, callback) { 685 | insertCSVInTable({ 686 | filename: path.join(extractedFolder, 'transfers.txt'), 687 | model: db.transfer 688 | }, 689 | callback) 690 | } 691 | 692 | var loadFeedInfo = function (extractedFolder, db, callback) { 693 | insertCSVInTable({ 694 | filename: path.join(extractedFolder, 'feed_info.txt'), 695 | model: db.feed_info 696 | }, 697 | callback) 698 | } 699 | 700 | module.exports = function ( 701 | downloadsDir, 702 | gtfsFileOrFolder, 703 | db, 704 | isSpatial, 705 | interpolateStopTimes, 706 | callback 707 | ) { 708 | // determine if gtfs is a file or folder 709 | var gtfsPath = path.join(downloadsDir, gtfsFileOrFolder) 710 | fs.lstat(gtfsPath, function (err, stats) { 711 | if (err) { 712 | callback(err) 713 | return 714 | } 715 | 716 | if (stats.isDirectory()) { 717 | loadGtfs(gtfsPath, db, isSpatial, interpolateStopTimes, callback) 718 | } else { 719 | // create unzipper (assuming gtfs is in zip file) 720 | var extractFolder = path.join(downloadsDir, 'google_transit') 721 | var extractor = unzip.Extract({ path: extractFolder }) 722 | 723 | // create handler to process gtfs upon completion of unzip 724 | extractor.on('close', 725 | function () { 726 | loadGtfs(extractFolder, db, isSpatial, interpolateStopTimes, callback) 727 | } 728 | ) 729 | 730 | // delete former unzipped folder if it exists 731 | rimraf(extractFolder, function () { 732 | console.log('unzipping gtfs') 733 | 734 | // unzip 735 | fs.createReadStream(gtfsPath).pipe(extractor) 736 | }) 737 | } 738 | }) 739 | } 740 | --------------------------------------------------------------------------------