├── .gitignore
├── accumulator
    ├── .gitignore
    ├── Readme.md
    ├── pom.xml
    └── src
    │   └── main
    │       ├── java
    │           └── org
    │           │   └── example
    │           │       ├── CSVSink.java
    │           │       ├── Main.java
    │           │       └── PriceQuery.java
    │       └── resources
    │           └── logback.xml
├── gatherer_hafas
    ├── fetchBestPrices.mjs
    └── package.json
├── gatherer_role
    ├── Readme.md
    ├── defaults
    │   └── main.yaml
    ├── tasks
    │   └── main.yaml
    └── templates
    │   ├── fahrpreis_gatherer.service.j2
    │   ├── fahrpreis_gatherer.timer.j2
    │   └── run.sh.j2
├── images
    └── img.png
├── license.md
├── plotter
    ├── config.json
    ├── dashboard.py
    ├── plotter.py
    ├── requirements.txt
    └── wsgi.py
├── plotter_role
    ├── defaults
    │   └── main.yaml
    ├── tasks
    │   └── main.yaml
    └── templates
    │   ├── fahrpreis-plotter-config.json.j2
    │   └── fahrpreis-plotter.service.j2
└── readme.md


/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | Thumbs.db
3 | 
4 | .nvm-version
5 | node_modules
6 | npm-debug.log
7 | /package-lock.json
8 | 


--------------------------------------------------------------------------------
/accumulator/.gitignore:
--------------------------------------------------------------------------------
1 | target
2 | .idea
3 | 


--------------------------------------------------------------------------------
/accumulator/Readme.md:
--------------------------------------------------------------------------------
 1 | 
 2 | # Fahrpreis accumulator
 3 | 
 4 | transforms the brotli files that the service produces into accumulated summary
 5 | 
 6 | This is an apache beam pipeline that reads the brotli files, decompresses them and transfroms them into the single data points
 7 | 
 8 | ## todo:
 9 |  - sort the datapoints in the order they are going to be plotted
10 |  - make faster
11 |  - 
12 | 
13 | 
14 | ## db schema
15 |  a bit slow
16 | 
17 | ```sql
18 | create table preis_query
19 | (
20 |     id         int auto_increment
21 |         primary key,
22 |     `from`     int         null,
23 |     `to`       int         null,
24 |     queried_at varchar(24) null,
25 |     price      float       null,
26 |     start_date varchar(24) null
27 | );
28 | 
29 | create index preis_query_from_to_index
30 |     on preis_query (`from`, `to`);
31 | ```
32 | 
33 | 
34 | ## timings
35 | java pipelines+jackson+bz2: 345380ms
36 | java pipelines+jackson+gz: 270938ms
37 | ram+brotli4j: 438696ms
38 | 


--------------------------------------------------------------------------------
/accumulator/pom.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 5 |     <modelVersion>4.0.0</modelVersion>
 6 | 
 7 |     <groupId>org.example</groupId>
 8 |     <artifactId>accumulator</artifactId>
 9 |     <version>1.0-SNAPSHOT</version>
10 | 
11 |     <properties>
12 |         <maven.compiler.source>17</maven.compiler.source>
13 |         <maven.compiler.target>17</maven.compiler.target>
14 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
15 |     </properties>
16 | 
17 |     <build>
18 |         <plugins>
19 |             <plugin>
20 |                 <groupId>org.codehaus.mojo</groupId>
21 |                 <artifactId>exec-maven-plugin</artifactId>
22 |                 <version>3.1.0</version>
23 |                 <executions>
24 |                     <execution>
25 |                         <goals>
26 |                             <goal>java</goal>
27 |                         </goals>
28 |                     </execution>
29 |                 </executions>
30 |                 <configuration>
31 |                     <mainClass>org.example.Main</mainClass>
32 |                 </configuration>
33 |             </plugin>
34 |         </plugins>
35 |     </build>
36 | 
37 |     <dependencies>
38 |         <dependency>
39 |             <groupId>org.apache.beam</groupId>
40 |             <artifactId>beam-sdks-java-core</artifactId>
41 |             <version>2.43.0</version>
42 |         </dependency>
43 | 
44 |         <dependency>
45 |             <groupId>org.apache.beam</groupId>
46 |             <artifactId>beam-runners-direct-java</artifactId>
47 |             <version>2.43.0</version>
48 |             <scope>runtime</scope>
49 |         </dependency>
50 |         <dependency>
51 |             <groupId>com.jayway.jsonpath</groupId>
52 |             <artifactId>json-path</artifactId>
53 |             <version>2.7.0</version>
54 |         </dependency>
55 |         <dependency>
56 |             <groupId>ch.qos.logback</groupId>
57 |             <artifactId>logback-classic</artifactId>
58 |             <version>1.2.11</version>
59 |         </dependency>
60 |         <dependency>
61 |             <groupId>org.projectlombok</groupId>
62 |             <artifactId>lombok</artifactId>
63 |             <version>1.18.24</version>
64 |         </dependency>
65 |         <dependency>
66 |             <groupId>org.apache.beam</groupId>
67 |             <artifactId>beam-sdks-java-io-jdbc</artifactId>
68 |             <version>2.43.0</version>
69 |         </dependency>
70 |         <dependency>
71 |             <groupId>com.nixxcode.jvmbrotli</groupId>
72 |             <artifactId>jvmbrotli</artifactId>
73 |             <version>0.2.0</version>
74 |         </dependency>
75 |         <dependency>
76 |             <groupId>org.apache.beam</groupId>
77 |             <artifactId>beam-runners-flink-1.14</artifactId>
78 |             <version>2.43.0</version>
79 |         </dependency>
80 |         <dependency>
81 |             <groupId>org.mariadb.jdbc</groupId>
82 |             <artifactId>mariadb-java-client</artifactId>
83 |             <version>3.0.6</version>
84 |         </dependency>
85 |         <dependency>
86 |             <groupId>com.fasterxml.jackson.core</groupId>
87 |             <artifactId>jackson-databind</artifactId>
88 |             <version>2.14.0</version>
89 |         </dependency>
90 | 
91 |     </dependencies>
92 | 
93 | </project>


--------------------------------------------------------------------------------
/accumulator/src/main/java/org/example/CSVSink.java:
--------------------------------------------------------------------------------
 1 | package org.example;
 2 | 
 3 | import java.io.IOException;
 4 | import java.io.PrintWriter;
 5 | import java.nio.channels.Channels;
 6 | import java.nio.channels.WritableByteChannel;
 7 | import java.util.Collections;
 8 | import java.util.List;
 9 | 
10 | import org.apache.beam.sdk.io.FileIO;
11 | 
12 | class CSVSink implements FileIO.Sink<List<String>> {
13 |    private String header;
14 |    private PrintWriter writer;
15 | 
16 |    public CSVSink(List<String> colNames) {
17 |      this.header = String.join(",",colNames);
18 |    }
19 | 
20 |    @Override
21 |    public void open(WritableByteChannel channel) throws IOException {
22 |      writer = new PrintWriter(Channels.newOutputStream(channel));
23 |      writer.println(header);
24 |    }
25 | 
26 |    @Override
27 |    public void write(List<String> element) throws IOException {
28 |      writer.println(String.join(",",element));
29 |    }
30 | 
31 |    @Override
32 |    public void flush() throws IOException {
33 |      writer.flush();
34 |    }
35 |  }


--------------------------------------------------------------------------------
/accumulator/src/main/java/org/example/Main.java:
--------------------------------------------------------------------------------
  1 | package org.example;
  2 | 
  3 | import java.io.ByteArrayInputStream;
  4 | import java.time.Instant;
  5 | import java.time.ZoneId;
  6 | import java.time.format.DateTimeFormatter;
  7 | import java.util.ArrayList;
  8 | import java.util.Arrays;
  9 | import java.util.EnumSet;
 10 | import java.util.List;
 11 | import java.util.Set;
 12 | 
 13 | import com.jayway.jsonpath.Configuration;
 14 | import com.jayway.jsonpath.JsonPath;
 15 | import com.jayway.jsonpath.Option;
 16 | import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
 17 | import com.jayway.jsonpath.spi.json.JsonProvider;
 18 | import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider;
 19 | import com.jayway.jsonpath.spi.mapper.MappingProvider;
 20 | import com.nixxcode.jvmbrotli.common.BrotliLoader;
 21 | import com.nixxcode.jvmbrotli.dec.BrotliInputStream;
 22 | import lombok.extern.slf4j.Slf4j;
 23 | import org.apache.beam.runners.flink.FlinkPipelineOptions;
 24 | import org.apache.beam.runners.flink.FlinkRunner;
 25 | import org.apache.beam.sdk.Pipeline;
 26 | import org.apache.beam.sdk.coders.StringUtf8Coder;
 27 | import org.apache.beam.sdk.io.Compression;
 28 | import org.apache.beam.sdk.io.FileIO;
 29 | import org.apache.beam.sdk.io.jdbc.JdbcIO;
 30 | import org.apache.beam.sdk.options.PipelineOptionsFactory;
 31 | import org.apache.beam.sdk.transforms.Contextful;
 32 | import org.apache.beam.sdk.transforms.FlatMapElements;
 33 | import org.apache.beam.sdk.values.PCollection;
 34 | import org.apache.beam.sdk.values.TypeDescriptor;
 35 | 
 36 | import static org.apache.beam.sdk.io.FileIO.Write.defaultNaming;
 37 | 
 38 | @Slf4j
 39 | public class Main {
 40 |     static final DateTimeFormatter dateTimeFormatter =
 41 |             DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.systemDefault());
 42 |     public static final JsonPath PRICE_PATH = JsonPath.compile("$.data.*.*.price.amount");
 43 |     public static final JsonPath START_TIME_PATH = JsonPath.compile("$.data.*.*.legs[0].departure");
 44 |     public static final JsonPath END_TIME_PATH = JsonPath.compile("$.data.*.*.legs[-1].departure");
 45 | 
 46 |     public static void main(String[] args) {
 47 |         setupTools();
 48 |         FlinkPipelineOptions options = PipelineOptionsFactory.create().as(FlinkPipelineOptions.class);
 49 |         options.setRunner(FlinkRunner.class);
 50 |         options.setFasterCopy(true);
 51 |         Pipeline p = Pipeline.create(options);
 52 |         //todo map datetime
 53 | 
 54 |         final PCollection<PriceQuery> extractedDataPoints = p.apply(FileIO.match().filepattern("/tmp/fahrpreise/*" +
 55 |                         ".brotli"))
 56 |                 .apply(FileIO.readMatches().withCompression(Compression.UNCOMPRESSED))
 57 |                 .apply(FlatMapElements
 58 |                         // uses imports from TypeDescriptors
 59 |                         .into(
 60 |                                 new TypeDescriptor<PriceQuery>() {
 61 |                                 }
 62 |                         )
 63 |                         .via(Main::getPriceQueries));
 64 |         //writeToDB(extractedDataPoints);
 65 |         writeToCSV(extractedDataPoints);
 66 | 
 67 |         p.run().waitUntilFinish();
 68 |     }
 69 | 
 70 |     private static void writeToCSV(final PCollection<PriceQuery> extractedDataPoints) {
 71 |         extractedDataPoints.apply(FileIO.<String, PriceQuery>writeDynamic()
 72 |                 .via(Contextful.fn((PriceQuery pq) -> Arrays.asList(pq.startDate() + "",
 73 |                                 pq.queriedAt() + "",
 74 |                                 pq.price() + "")),
 75 |                         Contextful.fn(
 76 |                                 (String connection) -> new CSVSink(Arrays.asList("startDate", "queriedAt", "price"))))
 77 |                 .by(input -> input.startStation() + "-" + input.targetStation())
 78 |                 .to("/tmp/fahrpreisakku/")
 79 |                 .withDestinationCoder(StringUtf8Coder.of())
 80 |                 .withNaming(type -> defaultNaming(type, ".csv"))
 81 |                 .withCompression(Compression.GZIP)
 82 |         );
 83 |     }
 84 | 
 85 |     private static void writeToDB(final PCollection<PriceQuery> extractedDataPoints) {
 86 |         extractedDataPoints.apply(JdbcIO.<PriceQuery>write()
 87 |                 .withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create(
 88 |                         "org.mariadb.jdbc.Driver", "jdbc:mariadb://localhost:3306/fahrpreise")
 89 |                 )
 90 |                 .withStatement("INSERT INTO fahrpreise.preis_query (`from`,`to`,queried_at,price,start_date)" +
 91 |                         "VALUES (?,?,?,?,?)")
 92 |                 .withPreparedStatementSetter((JdbcIO.PreparedStatementSetter<PriceQuery>) (element, query) -> {
 93 |                     query.setLong(1, element.startStation());
 94 |                     query.setLong(2, element.targetStation());
 95 |                     query.setLong(3, element.queriedAt());
 96 |                     query.setDouble(4, element.price());
 97 |                     query.setLong(5, element.startDate());
 98 |                 })
 99 |         );
100 |     }
101 | 
102 |     private static void setupTools() {
103 |         BrotliLoader.isBrotliAvailable();
104 |         Configuration.setDefaults(new Configuration.Defaults() {
105 | 
106 |             private final JsonProvider jsonProvider = new JacksonJsonProvider();
107 |             private final MappingProvider mappingProvider = new JacksonMappingProvider();
108 | 
109 |             @Override
110 |             public JsonProvider jsonProvider() {
111 |                 return jsonProvider;
112 |             }
113 | 
114 |             @Override
115 |             public MappingProvider mappingProvider() {
116 |                 return mappingProvider;
117 |             }
118 | 
119 |             @Override
120 |             public Set<Option> options() {
121 |                 return EnumSet.noneOf(Option.class);
122 |             }
123 |         });
124 |     }
125 | 
126 |     private static Iterable<PriceQuery> getPriceQueries(FileIO.ReadableFile f) {
127 |         Object document = null;
128 |         try {
129 |             BrotliInputStream brotliInputStream = new BrotliInputStream(new ByteArrayInputStream(f.readFullyAsBytes()));
130 | 
131 |             document = Configuration.defaultConfiguration().jsonProvider().parse(brotliInputStream, "UTF-8");
132 | 
133 |             final String[] filename_params = f.getMetadata()
134 |                     .resourceId()
135 |                     .getFilename()
136 |                     .substring(0, "2168494927864-8000706-8000091".length())
137 |                     .split("-");
138 |             long startStation = Long.parseLong(filename_params[1]);
139 |             long targetStation = Long.parseLong(filename_params[2]);
140 | 
141 |             List<PriceQuery> results = new ArrayList<>();
142 |             List<Number> prices = PRICE_PATH.read(document);
143 |             // those differ from the one in the filename. why?
144 |             //List<String> startStations = JsonPath.read(document, "$.data.*.*.legs[0].origin.id");
145 |             //List<String> endStations = JsonPath.read(document, "$.data.*.*.legs[-1].origin.id");
146 |             List<String> startTimes = START_TIME_PATH.read(document);
147 |             List<String> endTimes = END_TIME_PATH.read(document);
148 |             final int size = prices.size();
149 |             if (startTimes.size() != size || endTimes.size() != size) {
150 |                 throw new Exception("sizes of entries from json are not equal");
151 |             }
152 |             final Instant queriedAt = Instant.from(dateTimeFormatter.parse(JsonPath.read(document, "$" +
153 |                     ".queried_at")));
154 |             for (int i = 0; i < size; i++) {
155 |                 try {
156 |                     PriceQuery pq = new PriceQuery(
157 |                             queriedAt.toEpochMilli(),
158 |                             prices.get(i).doubleValue(),
159 |                             startStation, targetStation,
160 |                             java.time.Instant.from(dateTimeFormatter.parse(startTimes.get(i))).toEpochMilli(),
161 |                             java.time.Instant.from(dateTimeFormatter.parse(endTimes.get(i))).toEpochMilli()
162 |                     );
163 |                     results.add(pq);
164 |                 } catch (Exception e) {
165 |                     log.warn("error during processing {} in document ({},{},{},{})", i, prices.get(i),
166 |                             startTimes.get(i), endTimes.get(i), e);
167 |                 }
168 |             }
169 |             return results;
170 |         } catch (Exception e) {
171 |             log.error("could not read file {}: {}", f.getMetadata().resourceId().getFilename(), e.getMessage());
172 |             return new ArrayList<>();
173 |         }
174 |     }
175 | }


--------------------------------------------------------------------------------
/accumulator/src/main/java/org/example/PriceQuery.java:
--------------------------------------------------------------------------------
 1 | package org.example;
 2 | 
 3 | import org.apache.beam.sdk.schemas.JavaFieldSchema;
 4 | import org.apache.beam.sdk.schemas.annotations.DefaultSchema;
 5 | 
 6 | @DefaultSchema(JavaFieldSchema.class)
 7 | record PriceQuery(
 8 |         long queriedAt,
 9 |         double price,
10 |         long startStation,
11 |         long targetStation,
12 |         long startDate,
13 |         long endDate
14 | ) {
15 | }
16 | 


--------------------------------------------------------------------------------
/accumulator/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
 1 | <configuration debug="true">
 2 |     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
 3 |         <encoder>
 4 |             <pattern>%d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n</pattern>
 5 |         </encoder>
 6 |     </appender>
 7 |     <logger name="org.apache.beam.sdk" level="ERROR"></logger>
 8 |     <logger name="org.apache.flink" level="ERROR"></logger>
 9 | 
10 |     <root level="info">
11 |         <appender-ref ref="STDOUT" />
12 |     </root>
13 | </configuration>


--------------------------------------------------------------------------------
/gatherer_hafas/fetchBestPrices.mjs:
--------------------------------------------------------------------------------
  1 | import {createClient} from 'hafas-client'
  2 | import {profile as dbProfile} from 'hafas-client/p/db/index.js'
  3 | import {DateTime, Duration} from "luxon";
  4 | 
  5 | import sqlite3 from 'sqlite3';
  6 | 
  7 | const {Database, verbose} = sqlite3;
  8 | verbose();
  9 | 
 10 | 
 11 | async function getFromTo(from, to) {
 12 |     let loc1 = await client.locations(from)
 13 |     let loc2 = await client.locations(to)
 14 |     let from_id = loc1[0].id;
 15 |     let to_id = loc2[0].id;
 16 |     return {from_id, to_id};
 17 | }
 18 | 
 19 | 
 20 | async function queryPrices(from, to, now, number_of_days) {
 21 |     let all_prices = []
 22 |     for (let day_offset = 0; day_offset < number_of_days; day_offset++) {
 23 |         let query_day = now.plus(Duration.fromObject({days: day_offset}))
 24 |         let prices = await client.bestPrices(from, to, query_day.toJSDate())
 25 |         if (!prices.bestPrices)
 26 |             continue
 27 |         let compact_prices = prices.bestPrices
 28 |             .filter(d => d && d.journeys && d.journeys.length > 0)
 29 |             .flatMap(d=>d.journeys)
 30 |             .map(journey => ({
 31 |                 "when": journey.legs[0].plannedDeparture,
 32 |                 "price": journey.price?.amount,
 33 |                 "duration": new Date(journey.legs[journey.legs.length - 1].plannedArrival) - new Date(journey.legs[0].plannedDeparture)
 34 |             })).filter(d => d.price !== undefined)
 35 |         all_prices = all_prices.concat(compact_prices)
 36 | 
 37 |     }
 38 |     return all_prices;
 39 | }
 40 | 
 41 | 
 42 | function persist_station(db, station_id, station_name) {
 43 |     const check_station_stmt = db.prepare("Select 1 from `stations` where `number` = ? and `name` = ?;");
 44 |     const insert_station_stmt = db.prepare("insert into `stations` (`number`,`name`) values (?,?);");
 45 |     check_station_stmt.get([station_id, station_name], function (err, row) {
 46 |         if (err) {
 47 |             console.error(err)
 48 |             return console.error(err.message);
 49 |         }
 50 |         if (!row) {
 51 |             insert_station_stmt.run(station_id, station_name)
 52 |         }
 53 |     });
 54 | }
 55 | 
 56 | function persistPrices(db, from, to, allPrices, queried_at) {
 57 |     let from_int = parseInt(from);
 58 |     let to_int = parseInt(to);
 59 | 
 60 |     db.serialize(() => {
 61 | 
 62 |         /* duckdb schema
 63 |         CREATE SEQUENCE id_sequence START 1;
 64 | CREATE TABLE "fahrpreise" (
 65 |                   "id" integer not null primary key DEFAULT nextval('id_sequence'),
 66 |                   "from" INT not null,
 67 |                   "to" INT not null,
 68 |                   "when" LONG not null,
 69 |                   "price_cents" INT not null,
 70 |                   "queried_at" LONG not null
 71 |                 , "travel_duration" INT not null DEFAULT -1);
 72 | 
 73 |          */
 74 |         db.run('BEGIN TRANSACTION');
 75 |         db.run(`CREATE TABLE IF NOT EXISTS \`fahrpreise\` (
 76 |                   \`id\` integer not null primary key autoincrement,
 77 |                   \`from\` INT not null,
 78 |                   \`to\` INT not null,
 79 |                   \`when\` DATETIME not null,
 80 |                   \`price_cents\` INT not null,
 81 |                   \`queried_at\` datetime not null,
 82 |                   \`travel_duration\` INT not null DEFAULT -1
 83 |                 )`
 84 |         )
 85 |         const stmt = db.prepare('INSERT INTO fahrpreise (`from`,`to`,`when`,`price_cents`,`queried_at`,`travel_duration`) VALUES (?,?,?,?,?,?);');
 86 |         for (let data of allPrices) {
 87 |             stmt.run(
 88 |                 from_int, to_int, new Date(data.when), Math.trunc(data.price * 100), queried_at, data.duration
 89 |             )
 90 |         }
 91 | 
 92 |         stmt.finalize()
 93 | 
 94 |         db.run('COMMIT', (err) => {
 95 |             if (err) {
 96 |                 console.error('Error committing transaction', err);
 97 |             } else {
 98 |                 console.log(`Wrote ${allPrices.length} entries`);
 99 |             }
100 |         });
101 |     });
102 | }
103 | 
104 | let args_array = process.argv.slice(2)
105 | let args = {"fromName": args_array[0], "toName": args_array[1], "days": args_array[2], "dbFile": args_array[3]}
106 | 
107 | const userAgent = 'github.com/thigg/fahrpreis-plotter'
108 | 
109 | const client = createClient(dbProfile, userAgent)
110 | 
111 | let db = new sqlite3.Database(args.dbFile, (err) => {
112 |     if (err) {
113 |         console.error(err)
114 |         return console.error(err.message);
115 |     }
116 | });
117 | let {from_id, to_id} = await getFromTo(args.fromName, args.toName);
118 | 
119 | let now = DateTime.now()
120 | db.run(`CREATE TABLE IF NOT EXISTS \`stations\` (
121 |                   \`id\` integer not null primary key autoincrement,
122 |                   \`number\` INT not null,
123 |                   \`name\` TEXT not null
124 |                 )`
125 | )
126 | persist_station(db, from_id, args.fromName)
127 | persist_station(db, to_id, args.toName)
128 | 
129 | let all_prices = await queryPrices(from_id, to_id, now, args.days);
130 | persistPrices(db, from_id, to_id, all_prices, now.toJSDate());
131 | 
132 | 


--------------------------------------------------------------------------------
/gatherer_hafas/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "gatherer_hafas",
 3 |   "version": "1.0.0",
 4 |   "description": "",
 5 |   "main": "fetchBestPrices.mjs",
 6 |   "repository": {
 7 |     "type": "git",
 8 |     "url": "https://github.com/thigg/fahrpreis-plotter.git"
 9 |   },
10 |   "bin": {
11 |     "db-prices": "fetchBestPrices.mjs"
12 |   },
13 |   "dependencies": {
14 |     "hafas-client": "github:thigg/hafas-client#add-BestPriceSearch",
15 |     "luxon": "^3.4.4",
16 |     "sqlite3": "^5.1.7"
17 |   },
18 |   "devDependencies": {
19 |     "eslint": "^8.0.1"
20 |   },
21 |   "scripts": {
22 |     "lint": "eslint .",
23 |     "test": "fetchBestPrices.mjs 8011160 8000261 30 /tmp/db.sqlite3",
24 |     "prepublishOnly": "npm run lint && npm test"
25 |   },
26 |   "private": true
27 | }
28 | 


--------------------------------------------------------------------------------
/gatherer_role/Readme.md:
--------------------------------------------------------------------------------
1 | You can install the gatherer to a linux host with ansible with this role.
2 | 
3 | After running this role, write your stations seperated by `;` , one at each line into `/opt/fahrpreis_gatherer/stations.csv`
4 | The gatherer runs as a systemd timer every 3 hours and writes the results to the configured sqlite file
5 | 


--------------------------------------------------------------------------------
/gatherer_role/defaults/main.yaml:
--------------------------------------------------------------------------------
1 | db_file: /mnt/fahrpreise/fahrpreise.sqlite3
2 | 


--------------------------------------------------------------------------------
/gatherer_role/tasks/main.yaml:
--------------------------------------------------------------------------------
 1 | ---
 2 | - name: Install Node.js 22.x
 3 |   block:
 4 |     - name: Import the NodeSource Node.js repository signing key
 5 |       ansible.builtin.apt_key:
 6 |         url:  https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key
 7 |         state: present
 8 | 
 9 |     - name: Add the NodeSource Node.js
10 |       ansible.builtin.apt_repository:
11 |         repo: "deb https://deb.nodesource.com/node_21.x nodistro main"
12 |         state: present
13 |         update_cache: yes
14 | 
15 |     - name: Install Node.js
16 |       ansible.builtin.apt:
17 |         name:
18 |           - nodejs
19 |           - npm
20 |         state: present
21 |         update_cache: yes
22 | 
23 | 
24 | - name: Create a user for the Node.js application
25 |   ansible.builtin.user:
26 |     name: fahrpreis_gatherer
27 |     system: yes
28 |     home: "/opt/fahrpreis_gatherer"
29 | 
30 | - name: Ensure the application directory exists
31 |   ansible.builtin.file:
32 |     path: "/opt/fahrpreis_gatherer"
33 |     state: directory
34 |     owner: fahrpreis_gatherer
35 |     group: fahrpreis_gatherer
36 | 
37 | - name: Copy the Node.js application to the server
38 |   ansible.builtin.copy:
39 |     src: "{{ role_path }}/../gatherer_hafas/"
40 |     dest: "/opt/fahrpreis_gatherer"
41 |     owner: fahrpreis_gatherer
42 |     group: fahrpreis_gatherer
43 |     mode: '0755'
44 | - name: Install Node.js application dependencies with npm
45 |   ansible.builtin.command:
46 |     cmd: npm install
47 |     chdir: /opt/fahrpreis_gatherer
48 |   become: yes
49 |   become_user: fahrpreis_gatherer
50 | 
51 | - name: Deploy the run script
52 |   ansible.builtin.template:
53 |     src: "run.sh.j2"
54 |     dest: "/opt/fahrpreis_gatherer/run.sh"
55 |     mode: 'o=rx'
56 | 
57 | - name: Deploy the systemd service template
58 |   ansible.builtin.template:
59 |     src: "fahrpreis_gatherer.service.j2"
60 |     dest: "/etc/systemd/system/fahrpreis_gatherer.service"
61 | 
62 | - name: Deploy the systemd timer template
63 |   ansible.builtin.template:
64 |     src: "fahrpreis_gatherer.timer.j2"
65 |     dest: "/etc/systemd/system/fahrpreis_gatherer.timer"
66 | 
67 | - name: Enable and start the timer
68 |   ansible.builtin.systemd:
69 |     name: fahrpreis_gatherer.timer
70 |     enabled: yes
71 |     state: started
72 | 


--------------------------------------------------------------------------------
/gatherer_role/templates/fahrpreis_gatherer.service.j2:
--------------------------------------------------------------------------------
1 | [Unit]
2 | Description=collect fahrpreise
3 | 
4 | [Service]
5 | ExecStart=/opt/fahrpreis_gatherer/run.sh
6 | User=fahrpreis_gatherer
7 | 


--------------------------------------------------------------------------------
/gatherer_role/templates/fahrpreis_gatherer.timer.j2:
--------------------------------------------------------------------------------
 1 | [Unit]
 2 | Description=schedule fahrpreis_gatherer
 3 | 
 4 | [Timer]
 5 | OnUnitActiveSec=3h
 6 | Persistent=true
 7 | 
 8 | [Install]
 9 | WantedBy=timers.target
10 | 


--------------------------------------------------------------------------------
/gatherer_role/templates/run.sh.j2:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | while IFS=';' read -r from to || [ -n "$from" ]; do
3 |     echo "connection between $from and $to"
4 |     /usr/bin/node /opt/fahrpreis_gatherer/fetchBestPrices.mjs "$from" "$to" 60 {{ db_file }}
5 |     sleep 60
6 | done < "/opt/fahrpreis_gatherer/stations.csv"
7 | 
8 | 


--------------------------------------------------------------------------------
/images/img.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/thigg/fahrpreis-plotter/ef9a9869a05181d3245eadf0918cf42c7d048756/images/img.png


--------------------------------------------------------------------------------
/license.md:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2024 thigg
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining a copy
 4 | of this software and associated documentation files (the "Software"), to deal
 5 | in the Software without restriction, including without limitation the rights
 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 7 | copies of the Software, and to permit persons to whom the Software is
 8 | furnished to do so, subject to the following conditions:
 9 | 
10 | The above copyright notice and this permission notice shall be included in all
11 | copies or substantial portions of the Software.
12 | 
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19 | SOFTWARE.


--------------------------------------------------------------------------------
/plotter/config.json:
--------------------------------------------------------------------------------
1 | {"dbfile": "/tmp/fahrpreise.sqlite3"}


--------------------------------------------------------------------------------
/plotter/dashboard.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | import json
 3 | 
 4 | import dash
 5 | from dash import dcc, html
 6 | from dash.dependencies import Input, Output
 7 | from flask_caching import Cache
 8 | 
 9 | from plotter import accumulate_sqlite, plot, get_timeframe, open_sqlite, get_connection_tuples,Query_Datatype
10 | 
11 | dbfilepath=""
12 | 
13 | if __name__ == '__main__':
14 |     parser = argparse.ArgumentParser(description='accumulate fahrpreis data')
15 |     parser.add_argument('--dbfile', help='path to the sqlitefile with the data', required=True)
16 |     parser.add_argument('--port', help='server port', required=True)
17 |     args = parser.parse_args()
18 |     dbfilepath = args.dbfile
19 | else:
20 |     # Opening JSON file
21 |     f = open('/etc/fahrpreis-plotter-config.json')
22 | 
23 |     # returns JSON object as
24 |     # a dictionary
25 |     data = json.load(f)
26 |     dbfilepath=data['dbfile']
27 | 
28 | app = dash.Dash(__name__)
29 | cache = Cache(app.server, config={
30 |     'CACHE_TYPE': 'filesystem',
31 |     'CACHE_DIR': '/tmp/fahrpreise-cache'
32 | })
33 | app.config.suppress_callback_exceptions = True
34 | app.layout = html.Div([
35 | dcc.Interval(
36 |     id="load_interval",
37 |     n_intervals=0,
38 |     max_intervals=1, #<-- only run once
39 |     interval=1
40 | ),
41 |     html.Div([
42 |         html.Label("Select Connection:"),
43 |         dcc.Dropdown(id='connection-input', clearable=False),
44 |         html.Label("Query:"),
45 |         dcc.Dropdown(id='query-input', options=[{'label': i.name, 'value': i.name} for i in Query_Datatype],
46 |                      value="price_cents"),
47 |     ], style={'width': '20em', 'display': 'inline-block'}),
48 |     dcc.Graph(id='output-plot', style={'height': '100vh', 'flex-grow': '1'})
49 | ], style={'height': '100vh', 'display': 'flex'})
50 | 
51 | @app.callback(
52 |     Output('connection-input', 'options'),
53 |     Output('connection-input', 'value'),
54 |     [Input('load_interval', 'n_intervals')]
55 | )
56 | @cache.memoize(timeout=60*60)
57 | def update_connection_options(ignore):
58 |     print("querying connections")
59 |     conn, cursor = open_sqlite(dbfilepath)
60 |     connections = [x for x in get_connection_tuples(cursor)]
61 |     conn.close()
62 |     return [{'label': f"{x[0][1]}-{x[1][1]}", 'value': f"{x[0][0]}-{x[1][0]}"} for x in
63 |                                           connections], f"{connections[0][0][0]}-{connections[0][1][0]}"
64 | 
65 | 
66 | @app.callback(
67 |     Output('output-plot', 'figure'),
68 |     [Input('connection-input', 'value'),
69 |         Input('query-input', 'value')
70 |      ]
71 | )
72 | @cache.memoize(timeout=60*60)
73 | def update_plot(connection,query):
74 |     print(f"rendering connection {connection}")
75 |     conn, cursor = open_sqlite(dbfilepath)
76 | 
77 |     station1, station2 = connection.split("-")
78 |     timeframe_start, timeframe_end = get_timeframe(60, 10)
79 | 
80 |     result = accumulate_sqlite(cursor, station1, station2, timeframe_start, timeframe_end,query)
81 |     conn.close()
82 |     return plot(result, 60, 1, query)
83 | 
84 | 
85 | # Run the Dash app
86 | if __name__ == '__main__':
87 |     app.run(port=args.port)
88 | 


--------------------------------------------------------------------------------
/plotter/plotter.py:
--------------------------------------------------------------------------------
  1 | import argparse
  2 | import logging
  3 | import sqlite3
  4 | from collections import defaultdict
  5 | from datetime import datetime, timedelta
  6 | from enum import Enum
  7 | from sqlite3 import Cursor
  8 | from typing import Dict, List, Tuple, Generator, Any
  9 | 
 10 | import plotly.express as px
 11 | from plotly.graph_objs import Figure
 12 | 
 13 | 
 14 | class Query_Datatype(Enum):
 15 |     travel_duration = {'column': 'travel_duration', 'column_idx': 2, 'z_divisor': 1000 * 60 * 60,
 16 |                        'z_axis_name': "Dauer (Stunden)"}
 17 |     price_cents = {'column': 'price_cents', 'column_idx': 1, 'z_divisor': 100, 'z_axis_name': "Preis (Euro)"}
 18 | 
 19 | 
 20 | def accumulate_sqlite(cursor: Cursor, start_station: int, end_station: int, timeframe_start: datetime,
 21 |                       timeframe_end: datetime, query_datatype: str, num_bins_per_day: int = 24 / 3) -> dict[
 22 |     str, list[tuple[int, int]]]:
 23 |     # Create a dictionary to store accumulated prices
 24 |     z_values = defaultdict(lambda: [])
 25 | 
 26 |     millis_per_bin = int(1000 * 60 * 60 * 24 / num_bins_per_day)
 27 |     query = f"""
 28 |     WITH RankedPrices AS (
 29 |     SELECT 
 30 |         `when`,
 31 |         `price_cents`,
 32 |         `travel_duration`,
 33 |         `queried_at`,
 34 |         `queried_at` / {millis_per_bin} as queried_at_bin,
 35 |         `when` / {millis_per_bin} as when_bin,
 36 |         ROW_NUMBER() OVER (PARTITION BY `queried_at` / {millis_per_bin}, `when` / {millis_per_bin} ORDER BY `price_cents` ASC) as rn
 37 |     FROM fahrpreise
 38 |     WHERE `from` = {start_station}
 39 |       AND `to` = {end_station}
 40 |       AND `price_cents` > 0
 41 |       AND `queried_at` >= {round(timeframe_start.timestamp() * 1000)}
 42 |       AND `queried_at` <= {round(timeframe_end.timestamp() * 1000)}
 43 | )
 44 | SELECT 
 45 |     `when`,
 46 |     `price_cents`,
 47 |     `travel_duration`,
 48 |     `queried_at`,
 49 |     queried_at_bin,
 50 |     when_bin
 51 | FROM RankedPrices
 52 | WHERE rn = 1;
 53 |     """
 54 | 
 55 | #     query = f"""
 56 | #     SELECT `when`,
 57 | #        min(`price_cents`),
 58 | #        `travel_duration`,
 59 | #        `queried_at`,
 60 | #        `queried_at` / {millis_per_bin} as queried_at_bin,
 61 | #        `when` / {millis_per_bin}       as when_bin
 62 | # FROM fahrpreise t1
 63 | # where `from` = {start_station}
 64 | #   and `to` = {end_station}
 65 | #   and `queried_at` >= {round(timeframe_start.timestamp() * 1000)}
 66 | #   and `queried_at` <= {round(timeframe_end.timestamp() * 1000)}
 67 | #   and  (SELECT id
 68 | #                      FROM fahrpreise t2
 69 | #                       where `from` = {start_station}
 70 | #                       and `to` = {end_station}
 71 | #                       and `queried_at` / {millis_per_bin}  = t1.`queried_at`  / {millis_per_bin}
 72 | #                       and `when` / {millis_per_bin}    = t1.`when` / {millis_per_bin}
 73 | #                       order by `price_cents`
 74 | #                       limit 1
 75 | #                      ) = t1.`id`
 76 | # group by `queried_at_bin`, `when_bin`
 77 | #                 """
 78 |     print(query)
 79 |     cursor.execute(query)
 80 | 
 81 |     # Fetch all rows and accumulate prices/durations
 82 |     rows = cursor.fetchall()
 83 |     for row in rows:
 84 |         when = row[5] * millis_per_bin  # use when_bin
 85 |         z = row[Query_Datatype[query_datatype].value['column_idx']]
 86 |         queried_at = row[3]
 87 |         z_values[when].append((int(queried_at), int(z)))
 88 | 
 89 |     return z_values
 90 | 
 91 | 
 92 | def open_sqlite(filename):
 93 |     conn = sqlite3.connect(filename)
 94 |     cursor = conn.cursor()
 95 |     return conn, cursor
 96 | 
 97 | 
 98 | def plot(result: Dict[str, List[tuple[int, int]]], x_day_range_past: int, x_day_range_future: int, query_datatype: str,
 99 |          num_bins_per_day: int = 24 / 3) -> Figure:
100 |     """
101 |     :param query_datatype: config for the query
102 |     :param result: the data to ingest for the plot
103 |     :param x_day_range_past: how many days before departure to display
104 |     :param x_day_range_future: how many days after departure to display
105 |     :param num_bins_per_day: how many prices to consider per day, should match the query interval (defaults to every 3 hours)
106 |     """
107 |     query_config = Query_Datatype[query_datatype]
108 |     queried_at_num_bins = int((x_day_range_past + x_day_range_future) * num_bins_per_day)
109 |     queried_at_departure_bin_index = queried_at_num_bins - int((x_day_range_future) * num_bins_per_day)
110 |     zmap: Dict[datetime, List[float]] = defaultdict(lambda: [-1] * (queried_at_num_bins))
111 | 
112 |     for i, travelprices in enumerate(result.items()):
113 |         try:
114 |             when, price_records = travelprices
115 |             starttime: datetime = datetime.fromtimestamp(int(when) / 1000)
116 |             price_record: Dict[str, int]
117 |             for queried_at, z in price_records:
118 |                 queried_at_date = datetime.fromtimestamp(queried_at / 1000)
119 |                 days_to_departure = (starttime - queried_at_date).total_seconds() / (60 * 60 * 24)
120 |                 queried_at_bin_index = queried_at_departure_bin_index - int(days_to_departure * num_bins_per_day)
121 |                 if queried_at_bin_index >= queried_at_num_bins or queried_at_bin_index < 0:
122 |                     # discard datapoint because out of rendering range
123 |                     continue
124 |                 try:
125 |                     zmap[starttime][queried_at_bin_index] = z / query_config.value['z_divisor']
126 |                 except:
127 |                     print(f"{queried_at_bin_index} {days_to_departure} {starttime} {queried_at_date}")
128 |                     raise
129 |         except:
130 |             logging.exception("exception while prepping plot %d %s", i, travelprices)
131 | 
132 |     sorted_pricemap = sorted(zmap.items())
133 |     fig = px.imshow([x[1] for x in sorted_pricemap],
134 |                     labels=dict(x="tage bis abfahrt / wann ich buche", y="datum der reise / wann ich fahren möchte",
135 |                                 color=query_config.value['z_axis_name']),
136 |                     x=[f"{-(x - queried_at_departure_bin_index) / num_bins_per_day:.2f}" for x in
137 |                        range(int(queried_at_num_bins))],
138 |                     y=[k.strftime('%A %d-%m-%Y, %H:%M') for k, v in sorted_pricemap]
139 |                     )
140 |     return fig
141 | 
142 | 
143 | def get_timeframe(past, future):
144 |     timeframe_start: datetime = datetime.now() - timedelta(days=past)
145 |     timeframe_end: datetime = datetime.now() + timedelta(days=future)
146 |     return timeframe_start, timeframe_end
147 | 
148 | 
149 | def get_connection_tuples(cursor: Cursor) -> Generator[Tuple[Tuple[int, str], Tuple[int, str]], Any, None]:
150 |     cursor.execute("select distinct `from`,`from_station`.`name`,`to`,`to_station`.`name` from fahrpreise "
151 |                    "join `stations` as from_station , `stations` as `to_station` "
152 |                    "where `from`=`from_station`.`number` and `to`=`to_station`.`number`;")
153 |     return ((
154 |         (int(row[0]), row[1]),
155 |         (int(row[2]), row[3])
156 |     ) for row in cursor.fetchall())
157 | 
158 | 
159 | if __name__ == '__main__':
160 |     parser = argparse.ArgumentParser(description='accumulate fahrpreis data')
161 |     parser.add_argument('--dbfile', help='path to the sqlitefile with the data', required=True)
162 |     parser.add_argument('--output_file', help='if set, writes the html to the given file')
163 |     parser.add_argument('--start_station', help='start station id', required=True)
164 |     parser.add_argument('--end_station', help='end station id', required=True)
165 |     parser.add_argument('--plot_timeframe_past', help='oldest travel start date on the plot, days relative to now',
166 |                         default=60)
167 |     parser.add_argument('--plot_timeframe_future', help='newest travel start date on the plot, days relative to now',
168 |                         default=10)
169 |     parser.add_argument('--plot_timeframe_date', help='what now is for timeframe',
170 |                         default=datetime.now())
171 | 
172 |     args = parser.parse_args()
173 |     timeframe_start, timeframe_end = get_timeframe(args.plot_timeframe_past, args.plot_timeframe_future)
174 |     conn, cursor = open_sqlite(args.dbfile)
175 |     result = accumulate_sqlite(cursor, int(args.start_station), int(args.end_station),
176 |                                timeframe_start, timeframe_end,
177 |                                Query_Datatype.price_cents.name)
178 |     conn.close()
179 |     figure = plot(result, 60, 1, Query_Datatype.price_cents.name)
180 |     if args.output_file:
181 |         html = figure.to_html()
182 |         with open(args.output_file, "w") as text_file:
183 |             text_file.write(html)
184 |     else:
185 |         figure.show()
186 | 


--------------------------------------------------------------------------------
/plotter/requirements.txt:
--------------------------------------------------------------------------------
1 | plotly-express
2 | dash
3 | flask-caching
4 | pyuwsgi
5 | 


--------------------------------------------------------------------------------
/plotter/wsgi.py:
--------------------------------------------------------------------------------
1 | from dashboard import app
2 | server=app.server


--------------------------------------------------------------------------------
/plotter_role/defaults/main.yaml:
--------------------------------------------------------------------------------
1 | db_file: /mnt/fahrpreise/fahrpreise.sqlite3
2 | 


--------------------------------------------------------------------------------
/plotter_role/tasks/main.yaml:
--------------------------------------------------------------------------------
 1 | ---
 2 | - name: Ensure application user exists
 3 |   ansible.builtin.user:
 4 |     name: fahrpreis-plotter
 5 |     groups: fahrpreis_gatherer
 6 |     state: present
 7 | 
 8 | - name: Ensure python3-venv is installed
 9 |   ansible.builtin.package:
10 |     name: "{{ item }}"
11 |     state: present
12 |   loop:
13 |     - python3-venv
14 | 
15 | - name: Create a directory for the app
16 |   ansible.builtin.file:
17 |     path: /opt/fahrpreis-plotter
18 |     state: directory
19 |     owner: fahrpreis-plotter
20 |     group: fahrpreis-plotter
21 | 
22 | - name: Copy the application code
23 |   ansible.builtin.copy:
24 |     src: "{{ role_path }}/../plotter/"
25 |     dest: /opt/fahrpreis-plotter
26 |     owner: fahrpreis-plotter
27 |     group: fahrpreis-plotter
28 |     remote_src: no
29 | 
30 | - name: Create a Python virtual environment
31 |   ansible.builtin.command:
32 |     cmd: python3 -m venv /opt/fahrpreis-plotter/venv
33 |   args:
34 |     creates: "/opt/fahrpreis-plotter/venv"
35 |   become: yes
36 |   become_user: fahrpreis-plotter
37 | 
38 | - name: Install dependencies using pip
39 |   ansible.builtin.command:
40 |     cmd: /opt/fahrpreis-plotter/venv/bin/pip install -r /opt/fahrpreis-plotter/requirements.txt
41 |   become: yes
42 |   become_user: fahrpreis-plotter
43 | 
44 | - name: Create systemd-tmpfiles configuration
45 |   ansible.builtin.copy:
46 |     dest: "/etc/tmpfiles.d/fahrpreis-plotter.conf"
47 |     content: |
48 |       d /run/fahrpreis-plotter 0755 fahrpreis-plotter fahrpreis-plotter -
49 |     owner: "root"
50 |     group: "root"
51 |     mode: "0644"
52 | - name: Apply systemd-tmpfiles configuration
53 |   ansible.builtin.command:
54 |     cmd: "systemd-tmpfiles --create"
55 | 
56 | - name: Deploy the systemd service file
57 |   ansible.builtin.template:
58 |     src: fahrpreis-plotter.service.j2
59 |     dest: /etc/systemd/system/fahrpreis-plotter.service
60 | - name: Deploy the config file
61 |   ansible.builtin.template:
62 |     src: fahrpreis-plotter-config.json.j2
63 |     dest: /etc/fahrpreis-plotter-config.json
64 | - name: start fahrpreis-plotter service
65 |   ansible.builtin.systemd:
66 |     name: fahrpreis-plotter
67 |     state: started
68 |     enabled: yes
69 |     daemon_reload: yes
70 | 


--------------------------------------------------------------------------------
/plotter_role/templates/fahrpreis-plotter-config.json.j2:
--------------------------------------------------------------------------------
1 | {"dbfile": "{{ db_file }}"}


--------------------------------------------------------------------------------
/plotter_role/templates/fahrpreis-plotter.service.j2:
--------------------------------------------------------------------------------
 1 | [Unit]
 2 | Description=Fahrpreis Plotter Service
 3 | After=network.target
 4 | 
 5 | [Service]
 6 | User=fahrpreis-plotter
 7 | Group=fahrpreis-plotter
 8 | WorkingDirectory=/opt/fahrpreis-plotter
 9 | ExecStart=/opt/fahrpreis-plotter/venv/bin/uwsgi -C --socket /run/fahrpreis-plotter/socket.sock --master -p 1 -w wsgi:server
10 | Restart=always
11 | # File system access restrictions
12 | ProtectSystem=strict
13 | ReadWritePaths=/run/fahrpreis-plotter/
14 | ReadOnlyPaths=/opt/fahrpreis-plotter /mnt/fahrpreise/fahrpreise.sqlite3
15 | ProtectHome=true
16 | PrivateTmp=true
17 | NoNewPrivileges=true
18 | 
19 | [Install]
20 | WantedBy=multi-user.target
21 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | # fahrpreis statistiken
 2 | 
 3 | Collect connection prices (fahrpreise) of the Deutsche Bahn every couple hours,
 4 | to see how connection prices change over time.
 5 | Includes a tool for plotting them as well!
 6 | 
 7 | It is a tool you can run to gather every vew hours prices for a connection you specified.
 8 | Those prices can then be plotted to see how connection prices change over time.
 9 | 
10 | ## Result
11 | These are the prices I collected for a connection I was interested in.
12 |  - X-axis is the price until departure
13 |  - Y-axis the date until departure.
14 | 
15 | Findings:
16 |  - the vetical line at ~4 weeks before departure shows a discontinuity
17 |  - some connections (especially weekends/holidays) are always expensive
18 | 
19 | ![fahpreise plot](images/img.png)
20 | 
21 | ## How to run
22 | I run the service on a server and create the plots from my laptop. I sync the database file to my laptop for plotting.
23 | ### install gatherer service
24 | You can install the gatherer via ansible using this [role](./gatherer_role).
25 | This sets the [node app](./gatherer_hafas) up as a separate user and adds a systemd service and timer
26 | ### plot
27 |  1. get the file
28 |  2. run `plotter/main.py` (install dependencies before?) with required arguments
29 | 
30 | # Contributing
31 | For questions/issues use the issue tracker. Feel free to create pull requests with your modifications.
32 | 
33 | If you created interesting results with this tool and published them, let me know, I'll add you to the readme page.
34 | 
35 | 
36 | # License
37 | This was based on [db-prices-cli](https://github.com/derhuerst/db-prices-cli)
38 | and by Jannis R. Thanks go to him for the great base.
39 | 
40 | Reworked this to work with a fork of [hafas-client](https://github.com/public-transport/hafas-client). thanks to the authos
41 | 
42 | See the [license file](license.md) for further information.
43 | 


--------------------------------------------------------------------------------