└── processing.py /processing.py: -------------------------------------------------------------------------------- 1 | #!pip install --upgrade firebase-admin 2 | import os 3 | import pandas as pd 4 | import numpy as np 5 | import firebase_admin 6 | import requests 7 | from io import StringIO 8 | from firebase_admin import credentials 9 | from firebase_admin import firestore 10 | import scipy.stats as ss 11 | from scipy.optimize import curve_fit 12 | 13 | dynamic_link_aggregate_URL = 'https://docs.google.com/spreadsheet/ccc?key=1G3lXYx_p_wv63k4ZmADm_wR_SNffMWqZjpVMC-IMWps&output=csv' 14 | dynamic_link_time_URL = 'https://docs.google.com/spreadsheet/ccc?key=1G3lXYx_p_wv63k4ZmADm_wR_SNffMWqZjpVMC-IMWps&gid=14439463&output=csv' 15 | payment_keys = ['user','userNickName','date','time','payment_for', 'gross','net','epoch','isoDate'] 16 | payment_key_types = {'user':'string','userNickName':'string','date':'string','time':'string','payment_for':'string', 'gross':'float','net':'float','epoch':'int','isoDate':'string'} 17 | tip_keys = ['user','userNickName','date', 'time','amount','post','epoch','isoDate'] 18 | tip_key_types = {'user':'string','userNickName':'string','date':'string','time':'string','amount':'float','post':'string','epoch':'int','isoDate':'string'} 19 | 20 | dynamic_links = (dynamic_link_aggregate_URL, dynamic_link_time_URL) 21 | 22 | def get_db_instance(): 23 | cred = credentials.Certificate('secret.json') 24 | firebase_admin.initialize_app(cred) 25 | db = firestore.client() 26 | return db 27 | 28 | def get_google_spreadsheet(spreadsheet_url): 29 | response = requests.get(spreadsheet_url) 30 | assert response.status_code == 200, 'Wrong status code' 31 | return response.content.decode() 32 | 33 | def make_dynamic_link_data(dynamic_link_URLs): 34 | df_agg = pd.read_csv(StringIO(get_google_spreadsheet(dynamic_link_URLs[0]))) 35 | df_time = pd.read_csv(StringIO(get_google_spreadsheet(dynamic_link_URLs[1]))) 36 | return df_agg, df_time 37 | 38 | def get_docs(user, latest_only=False): 39 | docs = db.collection(u'users').document(user).get().to_dict() 40 | return docs 41 | 42 | def clean_docs(docs): 43 | payments = {key:[] for key in payment_keys} 44 | tips = {key:[] for key in tip_keys} 45 | for key in docs.keys(): 46 | if key[:7] == "payment": 47 | for subkey in payment_keys: 48 | if payment_key_types[subkey] == 'string': 49 | payments[subkey].append(docs[key].get(subkey,'')) 50 | elif payment_key_types[subkey] == 'float': 51 | payments[subkey].append(docs[key].get(subkey, 0.0)) 52 | elif payment_key_types[subkey] == 'int': 53 | payments[subkey].append(docs[key].get(subkey, 0)) 54 | elif key[:3] == "tip": 55 | for subkey in tip_keys: 56 | if tip_key_types[subkey] == 'string': 57 | tips[subkey].append(docs[key].get(subkey,'')) 58 | elif tip_key_types[subkey] == 'float': 59 | tips[subkey].append(docs[key].get(subkey, 0.0)) 60 | elif tip_key_types[subkey] == 'int': 61 | tips[subkey].append(docs[key].get(subkey, 0)) 62 | return pd.DataFrame.from_dict(payments), pd.DataFrame.from_dict(tips) 63 | 64 | 65 | try: 66 | df_agg, df_time = make_dynamic_link_data(dynamic_links) 67 | db = get_db_instance() 68 | docs = get_docs("borst_jessica@yahoo.com") 69 | payments, tips = clean_docs(docs) 70 | payments = payments.sort_values(['epoch']) 71 | tips = tips.sort_values(['epoch']) 72 | except Exception as e: 73 | print(e) 74 | 75 | sub_payment_count = payments[payments['payment_for']=='Subscription'].groupby(pd.Grouper(key='date')).count()['user'] 76 | sub_payments = payments[payments['payment_for']=='Subscription'].groupby(pd.Grouper(key='date')).sum() 77 | sub_payments['sub_count'] = sub_payment_count 78 | sub_payments = sub_payments.drop(['epoch'],axis=1) 79 | idx = pd.period_range('2020-01-31', '2020-05-18').to_timestamp() 80 | sub_payments.index = pd.DatetimeIndex(sub_payments.index) 81 | sub_payments = sub_payments.reindex(index=idx, fill_value=0) 82 | sub_payments.index = pd.DatetimeIndex(sub_payments.index) 83 | df_time.index = pd.DatetimeIndex(df_time.Date) 84 | df_time = df_time.sort_index(ascending=True) 85 | df_time = df_time.fillna(0) 86 | merged = sub_payments.join(df_time) 87 | merged['fetlife'] = df_time[['https://gingersexkitten.com/pussypics', 88 | 'https://gingersexkitten.com/boobiepic', 89 | 'https://gingersexkitten.com/fetlife', 90 | 'https://gingersexkitten.com/daddyfetlife', 91 | 'https://gingersexkitten.page.link/fetlifebutt', 92 | 'https://gingersexkitten.page.link/fetlifeonlyfans1']].sum(axis=1) 93 | 94 | merged['facebook'] = df_time[['https://gingersexkitten.com/SHSPromo' 95 | ,'https://gingersexkitten.com/SHS' 96 | , 'https://gingersexkitten.com/FBpromo']].sum(axis=1) 97 | merged['facebook_covid'] = df_time[['https://gingersexkitten.com/SHSPromo' 98 | ,'https://gingersexkitten.com/SHS']].sum(axis=1) 99 | merged['instagram'] = df_time['https://gingersexkitten.com/IG'] 100 | merged['reddit'] = df_time['https://gingersexkitten.com/custom'] 101 | 102 | mask_pairs = (('2020-04-05','2020-04-07'), ('2020-04-12','2020-04-14') 103 | , ('2020-04-19','2020-04-21'), ('2020-04-26','2020-04-28') 104 | ,('2020-05-03','2020-05-05'),('2020-05-10','2020-05-12')) 105 | 106 | exponential_fits = [] 107 | 108 | #Define your function 109 | def func(x, a, b): 110 | return a*np.exp(-b*x) 111 | 112 | for i in range(len(mask_pairs)): 113 | mask = (merged.index >= mask_pairs[i][0]) & (merged.index <= mask_pairs[i][1]) 114 | if i == 0: 115 | Sundays = merged.loc[mask] 116 | else: 117 | Sundays = Sundays.append(merged.loc[mask]) 118 | exponential_fits.append(curve_fit(func, [0,1,2], list((Sundays[(3*i):(3*(i+1))]['sub_count'])))[0]) 119 | 120 | unique_subs = payments[payments['payment_for'] == 'Subscription'] 121 | campaigns = ['facebook','facebook_covid','fetlife','reddit','instagram','date'] 122 | merged['date'] = merged['Date'] 123 | m_campaigns = merged[campaigns] 124 | pd.merge(unique_subs, m_campaigns,on=['date']) 125 | 126 | unique_subs_with_source = pd.merge(unique_subs, m_campaigns,on=['date']) 127 | unique_subs_with_source['total_clicks'] = unique_subs_with_source[['facebook','fetlife','reddit','instagram']].sum(axis=1) 128 | unique_subs_with_source['p_facebook'] = unique_subs_with_source['facebook']/unique_subs_with_source['total_clicks'] 129 | unique_subs_with_source['p_fetlife'] = unique_subs_with_source['fetlife']/unique_subs_with_source['total_clicks'] 130 | unique_subs_with_source['p_reddit'] = unique_subs_with_source['reddit']/unique_subs_with_source['total_clicks'] 131 | unique_subs_with_source['p_instagram'] = unique_subs_with_source['instagram']/unique_subs_with_source['total_clicks'] 132 | unique_subs_with_source['predicted_source'] = unique_subs_with_source[['instagram','facebook','reddit','fetlife']].idxmax(axis=1) --------------------------------------------------------------------------------