├── apis ├── __init__.py ├── tempo.py ├── datagouvfr.py └── myelectricaldata.py ├── requirements.txt ├── requirements ├── desktop.txt ├── dev.txt └── common.txt ├── .env.example ├── .gitignore ├── config.py ├── hacks.py ├── web_ui.py ├── main.py ├── LICENSE ├── elecanalysis.spec ├── README.md ├── db.py ├── desktop.py ├── edf_plan.py ├── fetch_edf.py └── ui.py /apis/__init__.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements/common.txt -------------------------------------------------------------------------------- /requirements/desktop.txt: -------------------------------------------------------------------------------- 1 | -r common.txt 2 | platformdirs~=4.2.0 -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | MED_TOKEN="abcdef" 2 | METER_ID="123456" 3 | PORT="8129" -------------------------------------------------------------------------------- /requirements/dev.txt: -------------------------------------------------------------------------------- 1 | -r common.txt 2 | pyinstaller~=6.3.0 3 | pywebview~=4.4.1 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .venv 2 | venv 3 | .idea 4 | .env 5 | __pycache__ 6 | app.db 7 | build/ 8 | dist/ 9 | _version.py -------------------------------------------------------------------------------- /requirements/common.txt: -------------------------------------------------------------------------------- 1 | matplotlib==3.8.2 2 | python-dotenv==1.0.0 3 | aiohttp-requests~=0.2.3 4 | urllib3==2.1.0 5 | dataclasses-json>=0.6.3 6 | numpy~=1.26.2 7 | nicegui~=1.4.8 8 | plotly~=5.18.0 9 | pandas~=2.1.4 -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from dotenv import dotenv_values 3 | 4 | DOTENV_PATH = ".env" 5 | 6 | config = None 7 | 8 | version = "dev" 9 | 10 | def load(): 11 | global config 12 | config = dotenv_values(DOTENV_PATH) 13 | 14 | load() 15 | -------------------------------------------------------------------------------- /hacks.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import sys 3 | import warnings 4 | 5 | in_bundle = getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS') 6 | 7 | def init(): 8 | import numpy 9 | _ = numpy 10 | 11 | with warnings.catch_warnings(): 12 | warnings.filterwarnings('ignore', r'All-NaN (slice|axis) encountered') 13 | -------------------------------------------------------------------------------- /web_ui.py: -------------------------------------------------------------------------------- 1 | from nicegui import ui as nui 2 | 3 | from config import config 4 | 5 | async def run_ui(): 6 | import db 7 | await db.load_meter_info() 8 | import fetch_edf 9 | await fetch_edf.fetch_loop() 10 | import ui 11 | _ = ui 12 | @nui.page("/") 13 | def index(): 14 | return ui.index() 15 | nui.run(port=int(config["PORT"]), show=False, title="elecanalysis") 16 | -------------------------------------------------------------------------------- /apis/tempo.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from aiohttp_requests import requests 3 | 4 | API_FORMAT = "https://www.api-couleur-tempo.fr/api/{endpoint}" 5 | 6 | 7 | async def get_days(days: list[str]) -> list[dict]: 8 | url = API_FORMAT.format(endpoint="joursTempo") 9 | req = await requests.get(url, params={"dateJour[]": days}) 10 | req.raise_for_status() 11 | res = await req.json() 12 | return res 13 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | 4 | import hacks 5 | 6 | parser = argparse.ArgumentParser() 7 | parser.add_argument("--app", action="store_true", help="Run as desktop app", default=hacks.in_bundle) 8 | args, unknown = parser.parse_known_args() 9 | 10 | hacks.init() 11 | 12 | if args.app: 13 | import desktop 14 | desktop.run() 15 | else: 16 | import asyncio 17 | import web_ui 18 | asyncio.run(web_ui.run_ui()) 19 | -------------------------------------------------------------------------------- /apis/datagouvfr.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from aiohttp_requests import requests 3 | 4 | DATA_GOUV_ROOT = "https://www.data.gouv.fr" 5 | API_FORMAT = DATA_GOUV_ROOT + "/api/1/{endpoint}" 6 | 7 | 8 | async def get_resource_info(dataset: str, resource: str): 9 | url = API_FORMAT.format(endpoint=f"datasets/{dataset}/resources/{resource}") 10 | req = await requests.get(url) 11 | req.raise_for_status() 12 | return await req.json() 13 | 14 | 15 | async def get_resource_content(resource: str): 16 | url = f"{DATA_GOUV_ROOT}/fr/datasets/r/{resource}" 17 | req = await requests.get(url) 18 | req.raise_for_status() 19 | return await req.text("utf-8") 20 | -------------------------------------------------------------------------------- /apis/myelectricaldata.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | from datetime import date 3 | from typing import Optional 4 | 5 | from aiohttp_requests import requests 6 | import config 7 | 8 | API_FORMAT = "https://www.myelectricaldata.fr/{endpoint}/{meter_id}{params}/cache/" 9 | 10 | 11 | async def fetch_api(endpoint, range: Optional[tuple[date, date]] = None): 12 | url = API_FORMAT.format(endpoint=endpoint, meter_id=config.config["METER_ID"], 13 | params="" if range is None else f"/start/{range[0]}/end/{range[1]}") 14 | req = await requests.get(url, headers={"Authorization": config.config["MED_TOKEN"]}) 15 | res = await req.json() 16 | if not req.ok: 17 | print(res) 18 | req.raise_for_status() 19 | return res 20 | 21 | 22 | async def get_meter_info(): 23 | return await fetch_api("contracts") 24 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Tom Niget 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /elecanalysis.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python ; coding: utf-8 -*- 2 | 3 | import nicegui 4 | from pathlib import Path 5 | 6 | a = Analysis( 7 | ['main.py'], 8 | pathex=[], 9 | binaries=[], 10 | datas=[(str(Path(nicegui.__file__).parent), 'nicegui')], 11 | hiddenimports=['socketio'], 12 | hookspath=[], 13 | hooksconfig={}, 14 | runtime_hooks=[], 15 | excludes=['PyQt5', 'PyQt6', 'matplotlib', 'PIL', 'tcl', 'tk', 'tcl8'], 16 | noarchive=False, 17 | ) 18 | 19 | EXCLUDED = [ 20 | r"nicegui\elements\lib\mermaid", 21 | r"nicegui\elements\lib\echarts", 22 | r"nicegui\elements\lib\vanilla-jsoneditor", 23 | r"nicegui\elements\lib\aggrid", 24 | r"nicegui\elements\lib\three", 25 | r"nicegui\elements\lib\leaflet", 26 | r"nicegui\elements\lib\nipplejs" 27 | ] 28 | 29 | a.datas = [k for k in a.datas if all(ex not in k[0] for ex in EXCLUDED)] 30 | 31 | pyz = PYZ(a.pure) 32 | 33 | exe = EXE( 34 | pyz, 35 | a.scripts, 36 | a.binaries, 37 | a.datas, 38 | [], 39 | name='elecanalysis', 40 | debug=False, 41 | bootloader_ignore_signals=False, 42 | strip=False, 43 | upx=True, 44 | upx_exclude=[], 45 | runtime_tmpdir=None, 46 | console=False, 47 | disable_windowed_traceback=False, 48 | argv_emulation=False, 49 | target_arch=None, 50 | codesign_identity=None, 51 | entitlements_file=None, 52 | ) 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | > [!NOTE] 2 | > This is in French because this project is pretty much only useful for people living in France, since the systems it allows integrating with are only present in France. 3 | 4 | # elecanalysis 5 | 6 | Outil d'analyse de consommation électrique et de comparaison de tarifs. 7 | 8 | À la base, conçu pour tester la rentabilité de Tempo face aux autres offres (spoiler : à moins de vivre en haute montagne et de se chauffer avec des convecteurs d'un autre temps, c'est rentable). 9 | 10 | ![image](https://github.com/zdimension/elecanalysis/assets/4533568/9e6795f3-db47-4d8d-82da-4b256476dd46) 11 | 12 | ![image](https://github.com/zdimension/elecanalysis/assets/4533568/69c724ba-2348-40b1-a6ef-f99e0a3c6643) 13 | 14 | ## Fonctionnement 15 | 16 | Récupère : 17 | - les données de conso depuis Enedis via [myElectricalData](https://www.myelectricaldata.fr/) 18 | - les données de tarifs Bleu depuis [data.gouv.fr](https://www.data.gouv.fr) 19 | - (pour les autres offres, c'est... codé en dur, je les mets à jour à la main tous les quelques mois en copiant depuis les PDF, je n'ai pas trouvé mieux) 20 | - les données de jour Tempo depuis [api-couleur-tempo](https://www.api-couleur-tempo.fr/) 21 | 22 | ## Usage 23 | 24 | ### Utilisateur 25 | 26 | Télécharger l'exe depuis la section Releases à droite. 27 | 28 | ### Développeur 29 | 30 | Nécessite : 31 | - Python ≥ 3.10 32 | - les dépendances (`pip install -r requirements.txt`) 33 | - un fichier `.env` à créer (voir `.env.example`) 34 | - `PORT`: port d'écoute (par défaut 8129) 35 | - `MED_TOKEN`: jeton myElectricalData 36 | - `METER_ID`: numéro de compteur myElectricalData 37 | 38 | Le premier lancement prend un peu de temps, car toutes les informations de consommation depuis l'activation du compteur sont récupérées. Aux lancements suivants, seules les données manquantes sont récupérées. 39 | -------------------------------------------------------------------------------- /db.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import asyncio 3 | import json 4 | import sqlite3 5 | from datetime import date 6 | 7 | from apis import myelectricaldata 8 | from config import config 9 | 10 | try: 11 | db = sqlite3.connect("file:app.db?mode=rw", uri=True) 12 | cur = db.cursor() 13 | except sqlite3.OperationalError: 14 | db = sqlite3.connect("app.db") 15 | cur = db.cursor() 16 | cur.execute("""CREATE TABLE consumption ( 17 | year INTEGER, 18 | month INTEGER, 19 | day INTEGER, 20 | slice INTEGER CHECK (slice BETWEEN 0 AND 47), 21 | value INTEGER, 22 | date TEXT GENERATED ALWAYS AS (PRINTF('%04d-%02d-%02d', year, month, day)) VIRTUAL, 23 | hour integer GENERATED ALWAYS AS (slice / 2) VIRTUAL, 24 | PRIMARY KEY (year, month, day, slice) 25 | );""") 26 | cur.execute("""CREATE TABLE tempo ( 27 | year INTEGER, 28 | month INTEGER, 29 | day INTEGER, 30 | tempo INTEGER CHECK (tempo BETWEEN 0 AND 3), 31 | date TEXT GENERATED ALWAYS AS (PRINTF('%04d-%02d-%02d', year, month, day)) VIRTUAL, 32 | PRIMARY KEY (year, month, day) 33 | );""") 34 | cur.execute(""" 35 | CREATE UNIQUE INDEX tempo_date ON tempo (date); 36 | """) 37 | cur.execute("""CREATE TABLE config ( 38 | key TEXT PRIMARY KEY, 39 | value TEXT 40 | );""") 41 | cur.execute("""CREATE TABLE edf_plan_slice ( 42 | plan_id TEXT, 43 | start TEXT, 44 | power INTEGER, 45 | subscription INTEGER, 46 | day_kind INTEGER, 47 | kwh_hp INTEGER, 48 | kwh_hc INTEGER, 49 | end TEXT, 50 | PRIMARY KEY (plan_id, power, day_kind, start) 51 | );""") 52 | db.commit() 53 | 54 | async def load_meter_info(): 55 | res = cur.execute("SELECT value FROM config WHERE key = 'meter_info'").fetchone() 56 | if res is not None: 57 | meter_info = json.loads(res[0]) 58 | else: 59 | meter_info = await myelectricaldata.get_meter_info() 60 | cur.execute("INSERT INTO config (key, value) VALUES ('meter_info', ?)", 61 | (json.dumps(meter_info),)) 62 | db.commit() 63 | global sub_power, activation_date 64 | sub_power = int(meter_info["customer"]["usage_points"][0]["contracts"]["subscribed_power"].split(" ")[0]) 65 | activation_date = date.fromisoformat( 66 | meter_info["customer"]["usage_points"][0]["contracts"]["last_activation_date"][:10]) 67 | if override_date := config.get("OVERRIDE_START_DATE"): 68 | activation_date = date.fromisoformat(override_date) 69 | 70 | -------------------------------------------------------------------------------- /desktop.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import asyncio 3 | 4 | import nicegui.events 5 | from nicegui import native, ui as nui, run as nrun 6 | import webbrowser 7 | import platformdirs 8 | from starlette.responses import RedirectResponse 9 | import sys 10 | 11 | DATA_DIR = platformdirs.user_data_path("elecanalysis", "zdimension") 12 | 13 | DATA_DIR.mkdir(parents=True, exist_ok=True) 14 | # set cwd to the data directory 15 | import os 16 | os.chdir(DATA_DIR) 17 | 18 | sys.stdout = open('logs.txt', 'w', buffering=1) 19 | sys.stderr = sys.stdout 20 | print("Using", DATA_DIR, "as storage dir") 21 | 22 | if not (DATA_DIR / ".env").exists(): 23 | with open(DATA_DIR / ".env", "w") as f: 24 | f.write("PORT=8129\n") 25 | 26 | import config 27 | 28 | @nui.page("/setup") 29 | def setup(): 30 | nui.markdown("# Bienvenue dans elecanalysis") 31 | nui.label("Pour commencer, il faut établir une connexion à MyElectricalData pour pouvoir récupérer vos données " 32 | "de consommation.") 33 | nui.label("Cliquez ici pour ouvrir votre espace Enedis et activer l'accès :") 34 | nui.button("Connexion à MyElectricalData", 35 | on_click=lambda: webbrowser.open("https://mon-compte-particulier.enedis.fr/dataconnect/v1/oauth2/authorize?client_id=e551937c-5250-48bc-b4a6-2323af68db92&duration=P36M&response_type=code")) 36 | nui.label("Une fois que c'est fait, entrez le numéro de PDL et la clef :") 37 | pdl = nui.input("Point de livraison :") 38 | key = nui.input("Clef :") 39 | 40 | def save_settings(): 41 | from dotenv import set_key 42 | set_key(".env", "METER_ID", pdl.value) 43 | set_key(".env", "MED_TOKEN", key.value) 44 | 45 | config.load() 46 | nui.open("/") 47 | 48 | nui.button("Valider", on_click=save_settings) 49 | 50 | @nui.page("/loading") 51 | def loading(): 52 | log = "" 53 | def logger(*args): 54 | nonlocal log 55 | print("logger:", *args) 56 | log += " ".join(map(str, args)) + "\n" 57 | log_display.refresh() 58 | @nui.refreshable 59 | def log_display(): 60 | nui.code(log, language="text").classes("w-full") 61 | import fetch_edf 62 | fetch_edf.log_callback = logger 63 | nui.markdown("# Récupération des données") 64 | log_display() 65 | task = asyncio.create_task(fetch_edf.fetch_loop()) 66 | import ui 67 | _ = ui 68 | done = False 69 | def timer_callback(): 70 | nonlocal done 71 | if not done and task.done(): 72 | nui.open("/app") 73 | done = True 74 | nui.timer(0.1, timer_callback) 75 | 76 | @nui.page("/") 77 | async def index(): 78 | if not ("METER_ID" in config.config and "MED_TOKEN" in config.config): 79 | return setup() 80 | 81 | import db 82 | await db.load_meter_info() 83 | return RedirectResponse("/loading") 84 | 85 | def run(): 86 | title = "elecanalysis" 87 | 88 | import hacks 89 | if hacks.in_bundle: 90 | import _version 91 | title += f" {_version.__version__}" 92 | nui.run(reload=False, port=native.find_open_port(), native=True, window_size=(1366, 768), title=title) -------------------------------------------------------------------------------- /edf_plan.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import enum 3 | from typing import Optional 4 | 5 | from db import sub_power 6 | 7 | 8 | class EdfPlan(enum.Enum): 9 | BASE = "base" 10 | HPHC = "hphc" 11 | TEMPO = "tempo" 12 | ZENFLEX = "zenflex" 13 | ZENWEEKEND = "zenweekend" 14 | ZENWEEKENDHC = "zenweekendhc" 15 | TOTALSTDFIXE = "totalstdfixe" 16 | TOTALSTDFIXEHC = "totalstdfixehc" 17 | 18 | def display_name(self) -> str: 19 | match self: 20 | case EdfPlan.BASE: 21 | return "Bleu Base" 22 | case EdfPlan.HPHC: 23 | return "Bleu Heures Creuses" 24 | case EdfPlan.TEMPO: 25 | return "Bleu Tempo" 26 | case EdfPlan.ZENFLEX: 27 | return "Zen Flex" 28 | case EdfPlan.ZENWEEKEND: 29 | return "Zen Week-End" 30 | case EdfPlan.ZENWEEKENDHC: 31 | return "Zen Week-End + Heures Creuses" 32 | case EdfPlan.TOTALSTDFIXE: 33 | return "Total Standard Fixe" 34 | case EdfPlan.TOTALSTDFIXEHC: 35 | return "Total Standard Fixe + Heures Creuses" 36 | 37 | def is_hp_sql(self) -> Optional[str]: 38 | """ 39 | Gives an SQL expression that evaluates to true if the current hour is in the HP period. 40 | 41 | Assumes `hour` exists and is an INTEGER. 42 | 43 | If the plan doesn't differentiate HC/HP, returns None. 44 | """ 45 | match self: 46 | case EdfPlan.HPHC | EdfPlan.TEMPO | EdfPlan.ZENWEEKENDHC | EdfPlan.TOTALSTDFIXEHC: 47 | return "6 <= hour and hour < 22" 48 | case EdfPlan.ZENFLEX: 49 | return "8 <= hour and hour < 13 or 18 <= hour and hour < 20" 50 | case _: 51 | return None 52 | 53 | def day_kind_sql(self) -> Optional[str]: 54 | """ 55 | Gives an SQL expression that evaluates to the day kind (e.g. 1/2/3 for Tempo, 1/2 for Zen Week-End, 0 for Base, ...). 56 | 57 | Assumes `c` is a table with a `date` column (YYYY-MM-DD) and an `hour` column (0-23). 58 | """ 59 | match self: 60 | case EdfPlan.TEMPO: 61 | # noinspection SqlResolve 62 | return "SELECT tempo FROM tempo t WHERE t.date = IIF(c.hour < 6, DATE(c.date, '-1 day'), c.date)" 63 | case EdfPlan.ZENFLEX: 64 | # todo 65 | # noinspection SqlResolve 66 | return "SELECT IIF(tempo = 3, 2, 1) FROM tempo t WHERE t.date = c.date" 67 | case EdfPlan.ZENWEEKEND | EdfPlan.ZENWEEKENDHC: 68 | return "SELECT IIF(strftime('%w', c.date) IN ('0', '6'), 2, 1)" 69 | case _: 70 | return None 71 | 72 | 73 | def query_plan_stats(plans: list[EdfPlan] = EdfPlan) -> str: 74 | """ 75 | Gives an SQL statement that returns the consumption stats with the following columns: 76 | - hp_{plan}: 1 if the current hour is in the HP period for {plan} 77 | - day_{plan}: the day kind for {plan} 78 | - date: YYYY-MM-DD 79 | - hour: hour (0-23) 80 | - slice: slice index (0-47) 81 | - value: consumption in Wh 82 | """ 83 | return "SELECT " + ",".join([ 84 | f"({p.is_hp_sql() or '0'}) as hp_{p.value}, ({p.day_kind_sql() or '0'}) as day_{p.value}" for p in plans 85 | ]) + """, 86 | c.date, 87 | hour, 88 | c.slice, 89 | c.value / 2 as value 90 | FROM consumption c""" 91 | 92 | 93 | def query_plan_prices_bihourly(plans: list[EdfPlan] = EdfPlan, price_mode = "real") -> str: 94 | """ 95 | Gives an SQL statement that returns the summarized consumption stats for each 30min slice with the following columns: 96 | - date: YYYY-MM-DD 97 | - slice: slice index (0-47) 98 | - value: consumption in Wh 99 | - eur_{plan}: cost in € for {plan} 100 | """ 101 | match price_mode: 102 | case "real": 103 | price_query = "and c.date between start and end" 104 | case "current": 105 | price_query = "order by start desc limit 1" 106 | case _: 107 | raise NotImplementedError(price_mode) 108 | return "SELECT c.date, c.slice, c.value, " + ",".join([ 109 | f"""COALESCE(( 110 | select 111 | iif(hp_{p.value}, kwh_hp, kwh_hc) * c.value + 112 | subscription * 100000 / 12 / cast(JULIANDAY(date, '+1 month') - JULIANDAY(date) as integer) / 48 113 | from edf_plan_slice s 114 | where plan_id='{p.value}' and power = {sub_power} and day_kind = day_{p.value} 115 | {price_query}), 1e999) as eur_{p.value}""" for p in plans 116 | ]) + f" FROM ({query_plan_stats(plans)}) c" 117 | 118 | 119 | def query_plan_prices_monthly(plans: list[EdfPlan] = EdfPlan) -> str: 120 | """ 121 | Gives an SQL statement that returns the summarized consumption stats for each month with the following columns: 122 | - date: YYYY-MM 123 | - value: consumption in Wh 124 | - eur_{plan}: cost in € for {plan} 125 | """ 126 | return query_plan_prices_period(plans, date="strftime('%Y-%m', c.date)", filter="1") 127 | 128 | 129 | def query_plan_prices_period(plans: list[EdfPlan] = EdfPlan, price_mode="real", date: str = "c.date", filter: str = "1", with_total: bool = False) -> str: 130 | """ 131 | Gives an SQL statement that returns the summarized consumption stats for each day with the following columns: 132 | - date: YYYY-MM-DD 133 | - value: consumption in Wh 134 | - eur_{plan}: cost in € for {plan} 135 | """ 136 | query = f"SELECT {date}, sum(c.value) as value, " + ",".join([ 137 | f"SUM(eur_{p.value}) as eur_{p.value}" for p in plans 138 | ]) + f" FROM ({query_plan_prices_bihourly(plans, price_mode)}) c WHERE {filter} GROUP BY {date}" 139 | if with_total: 140 | return f""" 141 | WITH prices AS ({query}) 142 | SELECT * FROM prices 143 | UNION ALL 144 | SELECT 'Total', sum(value), """ + ",".join([ 145 | f"SUM(eur_{p.value}) as eur_{p.value}" for p in plans 146 | ]) + " FROM prices" 147 | else: 148 | return query 149 | -------------------------------------------------------------------------------- /fetch_edf.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import asyncio 3 | import io 4 | import itertools 5 | from datetime import date, timedelta, datetime 6 | from decimal import Decimal 7 | 8 | import aiohttp 9 | import pandas as pd 10 | import requests 11 | 12 | from apis import myelectricaldata, tempo, datagouvfr 13 | from db import cur, db, activation_date 14 | from edf_plan import EdfPlan 15 | 16 | log_callback = print 17 | 18 | async def fetch_enedis(upto=None): 19 | """ 20 | Fetches the consumption data from Enedis using the MyElectricalData API. 21 | 22 | 7 days of consumption are retrieved at a time. The loop starts at the meter's activation date and continues until 23 | either MyElectricalData returns an error or the current day is reached. 24 | """ 25 | start_date = None 26 | 27 | try: 28 | last_info = date(*map(int, cur.execute( 29 | "SELECT year, month, day FROM consumption ORDER BY year DESC, month DESC, day DESC LIMIT 1").fetchone())) 30 | except TypeError: 31 | last_info = activation_date - timedelta(days=1) 32 | 33 | last_info = max(date.today() - timedelta(days=2 * 365), last_info) 34 | 35 | while True: 36 | # last info is max ymd from db 37 | new_start_date = last_info + timedelta(days=1) 38 | if new_start_date == start_date or new_start_date >= date.today(): 39 | break 40 | start_date = new_start_date 41 | end_date = start_date + timedelta(days=7) 42 | try: 43 | log_callback("Fetching MED for", str(start_date), "to", str(end_date)) 44 | conso_data = (await myelectricaldata.fetch_api("consumption_load_curve", (start_date, end_date)))[ 45 | "meter_reading"]["interval_reading"] 46 | except aiohttp.ClientResponseError as e: 47 | log_callback(e) 48 | break 49 | log_callback("Saving", len(conso_data), "Enedis rows") 50 | for reading in conso_data: 51 | dt = datetime.fromisoformat(reading["date"]) - timedelta(minutes=30) 52 | 53 | slice_idx = dt.hour * 2 + dt.minute // 30 54 | 55 | if dt.date() > last_info: 56 | last_info = dt.date() 57 | 58 | cur.execute("INSERT OR REPLACE INTO consumption VALUES (?, ?, ?, ?, ?)", 59 | (dt.year, dt.month, dt.day, slice_idx, int(reading["value"]))) 60 | db.commit() 61 | 62 | 63 | async def fetch_tempo(): 64 | """ 65 | Fetches the Tempo data from the api-couleur-tempo.fr API. 66 | 67 | 7 days of data are retrieved at a time. The loop starts at the meter's activation date and continues until 68 | either api-couleur-tempo.fr returns an error or the current day is reached. 69 | 70 | The day kind is {1, 2, 3}. If the API returns 0 for a day, it means the day kind for the day hasn't been retrieved 71 | yet -- in that case, the day is ignored and not inserted in the database. 72 | 73 | TODO: can the API ever return 0 for a day other than the last one? 74 | """ 75 | start_date = None 76 | 77 | try: 78 | last_info = date(*map(int, cur.execute( 79 | "SELECT year, month, day FROM tempo ORDER BY year DESC, month DESC, day DESC LIMIT 1").fetchone())) 80 | except TypeError: 81 | last_info = activation_date - timedelta(days=2) 82 | 83 | last_info = max(date.today() - timedelta(days=2 * 365 + 1), last_info) 84 | 85 | while True: 86 | # last info is max ymd from db 87 | new_start_date = last_info + timedelta(days=1) 88 | if new_start_date == start_date or new_start_date >= date.today(): 89 | break 90 | start_date = new_start_date 91 | try: 92 | tempo_data = await tempo.get_days([str(start_date + timedelta(days=i)) for i in range(100)]) 93 | except aiohttp.ClientResponseError as e: 94 | log_callback(e) 95 | break 96 | 97 | log_callback("Saving", len(tempo_data), "Tempo rows") 98 | for reading in tempo_data: 99 | dt = date.fromisoformat(reading["dateJour"]) 100 | val = reading["codeJour"] 101 | 102 | if val != 0: 103 | cur.execute("INSERT OR REPLACE INTO tempo VALUES (?, ?, ?, ?)", 104 | (dt.year, dt.month, dt.day, val)) 105 | 106 | if dt > last_info: 107 | last_info = dt 108 | db.commit() 109 | 110 | 111 | # polyfill for Python <3.12 112 | if not hasattr(itertools, "batched"): 113 | def batched(iterable, n): 114 | it = iter(iterable) 115 | while True: 116 | chunk = tuple(itertools.islice(it, n)) 117 | if not chunk: 118 | return 119 | yield chunk 120 | 121 | 122 | setattr(itertools, "batched", batched) 123 | 124 | 125 | async def fetch_prices(): 126 | """ 127 | Fetches the prices for Base and Bleu from data.gouv.fr and inserts them in the database. 128 | 129 | The datasets are in CSV format and contain both the yearly subscription price and the price per kWh for each 130 | pricing period. 131 | """ 132 | for name, rid in ( 133 | ("base", "c13d05e5-9e55-4d03-bf7e-042a2ade7e49"), ("hphc", "f7303b3a-93c7-4242-813d-84919034c416")): 134 | existing = cur.execute(f"SELECT value FROM config WHERE key = 'tarif_{name}'").fetchone() 135 | if existing is None: 136 | update = True 137 | else: 138 | update = date.today() - date.fromisoformat(existing[0]) > timedelta(days=1) 139 | 140 | if update: 141 | csv = await datagouvfr.get_resource_content(rid) 142 | # parse using pandas 143 | df = pd.read_csv(io.StringIO(csv), sep=";") 144 | # check if column PART_VARIABLE_TTC exists 145 | if "PART_VARIABLE_TTC" in df.columns: 146 | df["PART_VARIABLE_HC_TTC"] = df["PART_VARIABLE_TTC"] 147 | df["PART_VARIABLE_HP_TTC"] = df["PART_VARIABLE_TTC"] 148 | 149 | def dmy_to_iso(dmy): 150 | if type(dmy) is not str: 151 | return "9999-12-31" 152 | d, m, y = dmy.split("/") 153 | return f"{y}-{m}-{d}" 154 | 155 | def dec_to_fixed(dec, digits): 156 | # go from "0,0578" to 578 157 | return int(Decimal(dec.replace(",", ".") if type(dec) == str else dec) * (10 ** digits)) 158 | 159 | for index, row in df.iterrows(): 160 | if type(row["DATE_DEBUT"]) is not str: 161 | continue 162 | cur.execute("INSERT OR REPLACE INTO edf_plan_slice VALUES (?, ?, ?, ?, 0, ?, ?, ?)", 163 | (name, dmy_to_iso(row["DATE_DEBUT"]), row["P_SOUSCRITE"], 164 | dec_to_fixed(row["PART_FIXE_TTC"], 2), dec_to_fixed(row["PART_VARIABLE_HP_TTC"], 4), 165 | dec_to_fixed(row["PART_VARIABLE_HC_TTC"], 4), dmy_to_iso(row["DATE_FIN"]))) 166 | 167 | log_callback("Updated tariff", name) 168 | cur.execute(f"INSERT OR REPLACE INTO config VALUES ('tarif_{name}', ?)", (date.today().isoformat(),)) 169 | db.commit() 170 | 171 | def add_prices_pdf(): 172 | """ 173 | Some prices aren't provided by any API, so we have to get them from the public PDF price sheets. One day, this will 174 | maybe download the files and try and extract the tables from them, but for now, it's just a hardcoded dict. 175 | 176 | Since the prices are laid out in tables, it's easy to copy and paste the data from a PDF file to here. 177 | """ 178 | 179 | # https://particulier.edf.fr/content/dam/2-Actifs/Documents/Offres/Grille_prix_Tarif_Bleu.pdf 180 | # https://particulier.edf.fr/content/dam/2-Actifs/Documents/Offres/Grille-prix-zen-flex.pdf 181 | # https://particulier.edf.fr/content/dam/2-Actifs/Documents/Offres/grille-prix-zen-week-end.pdf 182 | 183 | edf_pdf_data = { 184 | "tempo": { 185 | "2023-01-01": """ 186 | 6 12,28 9,70 12,49 11,40 15,08 12,16 67,12 187 | 9 15,33 9,70 12,49 11,40 15,08 12,16 67,12 188 | 12 18,78 9,70 12,49 11,40 15,08 12,16 67,12 189 | 15 21,27 9,70 12,49 11,40 15,08 12,16 67,12 190 | 18 23,98 9,70 12,49 11,40 15,08 12,16 67,12 191 | 30 36,06 9,70 12,49 11,40 15,08 12,16 67,12 192 | 36 41,90 9,70 12,49 11,40 15,08 12,16 67,12 193 | """, 194 | "2023-08-01": """ 195 | 6 12,80 10,56 13,69 12,46 16,54 13,28 73,24 196 | 9 16,00 10,56 13,69 12,46 16,54 13,28 73,24 197 | 12 19,29 10,56 13,69 12,46 16,54 13,28 73,24 198 | 15 22,30 10,56 13,69 12,46 16,54 13,28 73,24 199 | 18 25,29 10,56 13,69 12,46 16,54 13,28 73,24 200 | 30 38,13 10,56 13,69 12,46 16,54 13,28 73,24 201 | 36 44,28 10,56 13,69 12,46 16,54 13,28 73,24 202 | """, 203 | "2024-02-01": """ 204 | 6 12,96 12,96 16,09 14,86 18,94 15,68 75,62 205 | 9 16,16 12,96 16,09 14,86 18,94 15,68 75,62 206 | 12 19,44 12,96 16,09 14,86 18,94 15,68 75,62 207 | 15 22,45 12,96 16,09 14,86 18,94 15,68 75,62 208 | 18 25,44 12,96 16,09 14,86 18,94 15,68 75,62 209 | 30 38,29 12,96 16,09 14,86 18,94 15,68 75,62 210 | 36 44,42 12,96 16,09 14,86 18,94 15,68 75,62 211 | """ 212 | }, # TODO: corriger quand les chiffres officiels sortent 213 | # "hphc": { 214 | # "2024-02-01": """ 215 | # 6 13,01 20,68 27,00 216 | # 9 16,70 20,68 27,00 217 | # 12 20,13 20,68 27,00 218 | # 15 23,40 20,68 27,00 219 | # 18 26,64 20,68 27,00 220 | # 24 33,44 20,68 27,00 221 | # 30 39,63 20,68 27,00 222 | # 36 44,79 20,68 27,00 223 | # """ 224 | # }, 225 | # "base": { 226 | # "2024-02-01": """ 227 | # 3 9,63 25,16 228 | # 6 12,60 25,16 229 | # 9 15,79 25,16 230 | # 12 19,04 25,16 231 | # 15 22,07 25,16 232 | # 18 25,09 25,16 233 | # 24 31,76 25,16 234 | # 30 37,44 25,16 235 | # 36 44,82 25,16 236 | # """ 237 | # }, 238 | "zenflex": { 239 | "2023-08-01": """ 240 | 6 12,62 12,95 22,28 22,28 67,12 241 | 9 15,99 12,95 22,28 22,28 67,12 242 | 12 19,27 12,95 22,28 22,28 67,12 243 | 15 22,40 12,95 22,28 22,28 67,12 244 | 18 25,46 12,95 22,28 22,28 67,12 245 | 24 32,01 12,95 22,28 22,28 67,12 246 | 30 38,07 12,95 22,28 22,28 67,12 247 | 36 43,88 12,95 22,28 22,28 67,12 248 | """, 249 | "2023-09-14": """ 250 | 6 13,03 14,64 24,60 24,60 73,24 251 | 9 16,55 14,64 24,60 24,60 73,24 252 | 12 19,97 14,64 24,60 24,60 73,24 253 | 15 23,24 14,64 24,60 24,60 73,24 254 | 18 26,48 14,64 24,60 24,60 73,24 255 | 24 33,28 14,64 24,60 24,60 73,24 256 | 30 39,46 14,64 24,60 24,60 73,24 257 | 36 45,72 14,64 24,60 24,60 73,24 258 | """, 259 | "2024-02-01": """ 260 | 6 13,03 17,04 27,00 27,00 75,64 261 | 9 16,55 17,04 27,00 27,00 75,64 262 | 12 19,97 17,04 27,00 27,00 75,64 263 | 15 23,24 17,04 27,00 27,00 75,64 264 | 18 26,48 17,04 27,00 27,00 75,64 265 | 24 33,28 17,04 27,00 27,00 75,64 266 | 30 39,46 17,04 27,00 27,00 75,64 267 | 36 45,72 17,04 27,00 27,00 75,64 268 | """ 269 | }, 270 | "zenweekend": { 271 | "2023-09-14": """ 272 | 3 9,47 25,25 17,71 273 | 6 12,44 25,25 17,71 274 | 9 15,63 25,25 17,71 275 | 12 19,25 25,25 17,71 276 | 15 22,37 25,25 17,71 277 | 18 25,46 25,25 17,71 278 | 24 32,32 25,25 17,71 279 | 30 37,29 25,25 17,71 280 | 36 43,99 25,25 17,71 281 | """, 282 | "2024-02-01": """ 283 | 3 9,47 27,65 20,11 284 | 6 12,44 27,65 20,11 285 | 9 15,63 27,65 20,11 286 | 12 19,25 27,65 20,11 287 | 15 22,37 27,65 20,11 288 | 18 25,46 27,65 20,11 289 | 24 32,32 27,65 20,11 290 | 30 37,29 27,65 20,11 291 | 36 43,99 27,65 20,11 292 | """ 293 | }, 294 | "zenweekendhc": { 295 | "2023-09-14": """ 296 | 6 13,03 26,83 18,81 18,81 18,81 297 | 9 16,55 26,83 18,81 18,81 18,81 298 | 12 19,97 26,83 18,81 18,81 18,81 299 | 15 23,24 26,83 18,81 18,81 18,81 300 | 18 26,48 26,83 18,81 18,81 18,81 301 | 24 33,28 26,83 18,81 18,81 18,81 302 | 30 39,46 26,83 18,81 18,81 18,81 303 | 36 45,72 26,83 18,81 18,81 18,81 304 | """, 305 | "2024-02-01": """ 306 | 6 13,03 29,23 21,21 21,21 21,21 307 | 9 16,55 29,23 21,21 21,21 21,21 308 | 12 19,97 29,23 21,21 21,21 21,21 309 | 15 23,24 29,23 21,21 21,21 21,21 310 | 18 26,48 29,23 21,21 21,21 21,21 311 | 24 33,28 29,23 21,21 21,21 21,21 312 | 30 39,46 29,23 21,21 21,21 21,21 313 | 36 45,72 29,23 21,21 21,21 21,21 314 | """ 315 | }, 316 | "totalstdfixe": { 317 | "2024-01-17": """ 318 | 3 9,51 0,1892 319 | 6 12,50 0,1892 320 | 12 19,08 0,1892 321 | 15 22,14 0,1892 322 | 18 25,17 0,1892 323 | 24 32,05 0,1892 324 | 30 37,71 0,1892 325 | 36 44,62 0,1892 326 | """ 327 | }, 328 | "totalstdfixehc": { 329 | "2024-01-17": """ 330 | 6 13,00 0,1511 0,2048 331 | 12 19,97 0,1511 0,2048 332 | 15 23,21 0,1511 0,2048 333 | 18 26,41 0,1511 0,2048 334 | 24 33,22 0,1511 0,2048 335 | 30 39,27 0,1511 0,2048 336 | 36 45,40 0,1511 0,2048 337 | """ 338 | } 339 | } 340 | 341 | for plan, vals in edf_pdf_data.items(): 342 | plan_data = EdfPlan(plan) 343 | for (dt, val), dt_end in zip(vals.items(), [(date.fromisoformat(k) - timedelta(days=1)).isoformat() for k in vals.keys()][1:] + ["9999-12-31"]): 344 | val = val.strip().replace(",", "").split("\n") 345 | for row in val: 346 | power, sub, *kwh = row.split(" ") 347 | sub = 12 * int(sub) 348 | day_start = 0 if plan_data.day_kind_sql() is None else 1 349 | if plan_data.is_hp_sql() is None: 350 | for day_kind, hchp in enumerate(kwh, day_start): 351 | cur.execute("INSERT OR REPLACE INTO edf_plan_slice VALUES (?, ?, ?, ?, ?, ?, ?, ?)", 352 | (plan, dt, power, sub, day_kind, hchp, hchp, dt_end)) 353 | else: 354 | for day_kind, (hc, hp) in enumerate(itertools.batched(kwh, 2), day_start): 355 | cur.execute("INSERT OR REPLACE INTO edf_plan_slice VALUES (?, ?, ?, ?, ?, ?, ?, ?)", 356 | (plan, dt, power, sub, day_kind, hp, hc, dt_end)) 357 | 358 | log_callback("committing to db") 359 | db.commit() 360 | log_callback("done") 361 | 362 | async def fetch_apis(): 363 | await fetch_enedis() 364 | await fetch_tempo() 365 | await fetch_prices() 366 | 367 | 368 | async def fetch_loop(): 369 | log_callback("fetch loop") 370 | await fetch_apis() 371 | add_prices_pdf() 372 | -------------------------------------------------------------------------------- /ui.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import math 3 | from calendar import monthrange 4 | from collections import defaultdict 5 | from dataclasses import dataclass 6 | from datetime import date, time, timedelta, datetime 7 | 8 | import numpy as np 9 | import plotly.graph_objects as go 10 | from nicegui import ui, context 11 | from plotly.subplots import make_subplots 12 | 13 | import fetch_edf 14 | from config import config 15 | from db import cur, activation_date 16 | from edf_plan import EdfPlan, query_plan_prices_period 17 | 18 | 19 | @dataclass 20 | class YearMonthInput: 21 | on_change: callable 22 | year: int = date.today().year 23 | month: int = date.today().month 24 | 25 | def view(self): 26 | def set_view_date(y, m): 27 | while m < 1: 28 | y -= 1 29 | m += 12 30 | while m > 12: 31 | y += 1 32 | m -= 12 33 | year.set_value(str(y)) 34 | month.set_value(str(m)) 35 | change_handler() 36 | 37 | def change_handler(): 38 | self.year = int(year.value) 39 | self.month = int(month.value) 40 | self.on_change(self.year, self.month) 41 | 42 | with ui.row().classes("items-end year-month-input") as row: 43 | previous_month = ui.button("◀", on_click=lambda: set_view_date(int(year.value), int(month.value) - 1)) 44 | year = ui.select(options=[str(y) for y in range(activation_date.year, date.today().year + 1)], 45 | value=str(self.year), on_change=change_handler, label="Année") 46 | month = ui.select(options=[str(m) for m in range(1, 13)], value=str(self.month), on_change=change_handler, 47 | label="Mois") 48 | next_month = ui.button("▶", on_click=lambda: set_view_date(int(year.value), int(month.value) + 1)) 49 | return row 50 | 51 | 52 | tabs = [] 53 | 54 | 55 | def tab(name): 56 | def decorator(f): 57 | tabs.append([name, f, None]) 58 | return f 59 | 60 | return decorator 61 | 62 | 63 | @tab("Consommation par jour") 64 | def content(): 65 | def nanmax(a): 66 | # if all nan 67 | if np.isnan(a).all(): 68 | return np.nan 69 | return np.nanmax(a) 70 | 71 | def update_plot(y, m): 72 | fig.data = [] 73 | days_in_month = monthrange(y, m)[1] 74 | tempo_data_db = np.transpose(np.array(cur.execute("SELECT day, tempo FROM tempo WHERE year = ? AND month = ? ORDER BY day", (y, m)).fetchall())) 75 | tempo_data = np.full(days_in_month, np.nan, dtype=np.float32) 76 | if len(tempo_data_db) > 0: 77 | tempo_data[tempo_data_db[0] - 1] = tempo_data_db[1] 78 | fig.add_trace(go.Heatmap( 79 | z=tempo_data.reshape((days_in_month, 1)), 80 | zmin=1, 81 | # zmax=nanmax(month_data), 82 | zmax=3, 83 | colorscale=[ 84 | [0, "rgb(21, 101, 192)"], 85 | [0.5, "rgb(240, 240, 240)"], 86 | [1, "rgb(198, 40, 40)"] 87 | ], 88 | text=[[["n/a", "Bleu", "Blanc", "Rouge"][0 if math.isnan(x) else int(x)]] for x in tempo_data], 89 | hovertemplate=f"%{{y}}/{m:02d}: Jour %{{text}}", 90 | showscale=False, 91 | ygap=1 92 | ), row=1, col=1) 93 | month_data_db = np.transpose(np.array(cur.execute( 94 | "SELECT (day - 1) * 48 + slice, value FROM consumption WHERE year = ? AND month = ? ORDER BY day, slice", 95 | (y, m)).fetchall(), 96 | dtype=np.int32)) 97 | month_data = np.full(48 * days_in_month, np.nan, dtype=np.float32) 98 | if len(month_data_db) > 0: 99 | month_data[month_data_db[0]] = month_data_db[1] 100 | month_data = month_data.reshape((days_in_month, 48)) / 2 101 | fig.add_trace(go.Heatmap( 102 | z=month_data, 103 | zmin=0, 104 | # zmax=nanmax(month_data), 105 | zmax=2000, 106 | colorscale='hot', 107 | colorbar=dict( 108 | x=0.748, 109 | ticksuffix=" Wh", 110 | tickformat="d", 111 | ), 112 | text=[[*timelabels[1:], "00:00"]] * days_in_month, 113 | hovertemplate=f"%{{y}}/{m:02d}, %{{x}}-%{{text}}: %{{z}} Wh"), row=1, col=2) 114 | fig.update_yaxes(tickvals=list(range(days_in_month)), ticktext=list(map(str, range(1, days_in_month + 1)))) 115 | sum_per_day = np.nansum(month_data, axis=1).reshape((-1, 1)) / 1000 116 | sum_per_day[np.isnan(month_data).all(axis=1)] = np.nan 117 | fig.add_trace(go.Heatmap( 118 | z=sum_per_day, 119 | zmin=0, 120 | # zmax=nanmax(sum_per_day), 121 | zmax=40, 122 | colorscale='hot', 123 | colorbar=dict( 124 | x=1, 125 | ticksuffix=" kWh", 126 | tickformat="d" 127 | ), 128 | hovertemplate=f"%{{y}}/{m:02d}: %{{z:.1f}} kWh"), row=1, col=3) 129 | plot.update() 130 | 131 | date_sel = YearMonthInput(update_plot) 132 | date_sel.view() 133 | 134 | fig = make_subplots(rows=1, cols=3, column_widths=[0.03, 0.75, 0.15], subplot_titles=("Tempo", "Consommation", "Total par jour"), 135 | horizontal_spacing=0, specs=[ 136 | [{}, {"l": 0.02, "r": 0.09}, {}] 137 | ]) 138 | timelabels = [f"{i:02d}:{j:02d}" for i in range(24) for j in (0, 30)] 139 | fig.update_yaxes(autorange="reversed") 140 | fig.update_xaxes(tickvals=list(range(48)), ticktext=timelabels, col=2) 141 | fig.update_xaxes(visible=False, col=3) 142 | fig.update_xaxes(visible=False, col=1) 143 | fig.update_yaxes(showticklabels=False, col=1) 144 | plot = ui.plotly(fig).classes('h-full w-full') 145 | update_plot(date_sel.year, date_sel.month) 146 | 147 | 148 | @tab("Coût") 149 | def content(): 150 | ui.html(""" 151 |