51 | {% endblock %}
52 | {% block script %}
53 |
54 |
75 | {% endblock %}
--------------------------------------------------------------------------------
/webapp/views.py:
--------------------------------------------------------------------------------
1 | from webapp import app
2 | import flask
3 | import classes
4 |
5 | def init_data(config):
6 | global CONFIG
7 | CONFIG = config
8 |
9 | def check_login():
10 | print(flask.session.get('logged_in', None))
11 | if flask.session.get('logged_in', None) is None:
12 | return flask.redirect(flask.url_for('login'))
13 |
14 | @app.route('/login', methods=['GET', 'POST'])
15 | def login():
16 | error = None
17 | if flask.request.method == 'POST':
18 | if (flask.request.form['username'] != CONFIG.settings.username
19 | or not classes.helpers.verify_password(flask.request.form['password'], CONFIG.settings.password)):
20 | error = 'Invalid username/password'
21 | else:
22 | flask.session['logged_in'] = True
23 | flask.flash('Successfully logged in!', 'success')
24 | return flask.redirect(flask.url_for('start_page'))
25 | return flask.render_template('login.html', error=error)
26 |
27 | @app.route('/logout')
28 | def logout():
29 | flask.session.pop('logged_in', None)
30 | return flask.redirect(flask.url_for('start_page'))
31 |
32 | @app.route('/')
33 | def start_page():
34 | return check_login() or flask.render_template(
35 | 'start_page.html', recording=classes.recording.RecordingThread.currently_recording_models,
36 | wanted=CONFIG.filter.wanted.dict,
37 | condition_text=classes.helpers.condition_text)
38 |
39 | @app.route('/MFC/wanted', methods=['GET', 'POST'])
40 | def wanted():
41 | check = check_login()
42 | if check is not None:
43 | return check
44 |
45 | if flask.request.method == 'POST':
46 | CONFIG.filter.wanted.set_dict(flask.request.form)
47 |
48 | return flask.render_template('wanted.html', wanted=CONFIG.filter.wanted.dict)
49 |
50 | @app.route('/MFC/config', methods=['GET', 'POST'])
51 | def config():
52 | check = check_login()
53 | if check is not None:
54 | return check
55 |
56 | if flask.request.method == 'POST':
57 | #special treatment for password
58 | #form data is immutable dict, we want to edit that here
59 | #dict(form) would give us a list of values per key (since it allows multiple values per key)
60 | #when iterating over form.items(), we only get the first entry per key, so we do that here
61 | #(mutliple entries for bool values, since they always send False and only additionally True)
62 | dict_ = {key:value for key, value in flask.request.form.items()}
63 | print(dict_)
64 | old = dict_.pop('password0')
65 | pw1 = dict_.pop('password1')
66 | pw2 = dict_.pop('password2')
67 | if old != '':
68 | if not classes.helpers.verify_password(old, CONFIG.settings.password):
69 | flask.flash('wrong old password, new password not set', 'danger')
70 | elif pw1 != pw2:
71 | flask.flash('new passwords didn\'t match, new password not set', 'danger')
72 | elif pw1 == '':
73 | flask.flash('new password is empty, not setting new password', 'danger')
74 | else:
75 | dict_['web:password'] = classes.helpers.hash_password(pw1)
76 |
77 | CONFIG.update(dict_)
78 | flask.flash('settings have been saved', 'success')
79 |
80 | return flask.render_template('config.html', config=CONFIG)
81 |
82 | @app.route('/MFC/add', methods=['GET'])
83 | def add():
84 | return check_login() or add_or_remove(_add)
85 |
86 | def _add(uid, name):
87 | result = CONFIG.filter.wanted.add(uid, name)
88 | if result is None:
89 | flask.flash('{} with uid {} successfully added'.format(name, uid), 'success')
90 | else:
91 | flask.flash('{} with uid {} already in wanted list (named "{}")'.format(name, uid, result['custom_name']), 'info')
92 |
93 | @app.route('/MFC/remove', methods=['GET'])
94 | def remove():
95 | return check_login() or add_or_remove(_remove)
96 |
97 | def _remove(uid, name):
98 | result = CONFIG.filter.wanted.remove(uid)
99 | if result is not None:
100 | flask.flash('{} with uid {} (named "{}") successfully removed'.format(name, uid, result['custom_name']), 'success')
101 | else:
102 | flask.flash('{} with uid {} not in wanted list'.format(name, uid), 'info')
103 |
104 | def add_or_remove(action):
105 | uid_or_name = classes.helpers.try_eval(flask.request.args['uid_or_name'])
106 | result = classes.models.get_model(uid_or_name)
107 | if result is None:
108 | flask.flash('uid or name "{}" not found'.format(uid_or_name), 'danger')
109 | else:
110 | action(*result)
111 | return flask.redirect(flask.url_for('start_page'))
112 |
113 | @app.route('/MFC/thumbnails/')
114 | def thumbnail(uid):
115 | #TODO: this might take very long and caching would probably be a good idea
116 | uid = int(uid)
117 | #try to get thumbnail from current video
118 | result = classes.helpers.get_live_thumbnail(
119 | uid, classes.recording.RecordingThread.currently_recording_models.get(uid, {}).get('camserv'))
120 | if result is None:
121 | #fallback to avatar from mfc
122 | result = classes.helpers.get_avatar(uid)
123 | if result is not None:
124 | mimetype, img = result
125 | return flask.send_file(img, mimetype=mimetype)
126 | return flask.abort(404)
127 |
--------------------------------------------------------------------------------
/config.conf:
--------------------------------------------------------------------------------
1 | [paths]
2 | #all paths (wishlist, save_directory, blacklist and completed_directory) can be relative to the config.conf location
3 | wishlist_path = ./wanted.json
4 | save_directory = ./captures
5 |
6 | # set the directory structure - default is "{path}/{uid}/{year}.{month}.{day}_{hour}.{minutes}.{seconds}_{model}.mp4"
7 | # {auto} = for autoRecording conditions below, it is the reason why the recording was started followed by "_" - These are: TAGS_, VIEWERS_, SCORE_, and NEW_. If a model is in the wanted list, this will be blank
8 | # {path} = save_directory set above. (your directory structure should start with this)
9 | # {model} = the models name
10 | # {uid} = models uid or broadcaster id. This is a number which is always consistent. The models name can be changed, so this is what I find best for keeping all videos of a model in the same directory
11 | # "{year}, {month}, {day}, {hour}, {minutes}, {seconds}" should all be easy enough to figure out what these are
12 | # you can also change ".mp4" to another extension. ".ts" is what the stream is actual broadcast as.
13 | # example using a madeup "hannah" who has the uid 208562: {path}/{uid}/{year}/{year}.{month}.{day}_{hour}.{minutes}.{seconds}_{model}.mp4 = "/Users/Joe/MFC/208562/2017/2017.07.26_19.34.47_hannah.mp4"
14 | # This will also be the "Download directory" or temp directory if you set a "completed_directory"
15 |
16 | directory_structure = {path}/{model}/{year}.{month}.{day}_{hour}.{minutes}.{seconds}_{model}.mp4
17 |
18 | # (OPTIONAL) - leave blank if you dont want files moved after completed
19 | # The files will be moved here once the stream has ended. The same naming structure will be used as above
20 | # if this is left empty, the videos wll remain in the same directory they were originally saved to
21 | # This path should be to a directory, not a filename! so do not include the filename portion, only the directory.
22 |
23 | completed_directory =
24 |
25 |
26 | [settings]
27 | check_interval = 20
28 |
29 | #minimum space in GB - when the available disk space available on the mnt where the download directory is located, the recordings will stop.
30 | min_space = 0
31 |
32 | # (OPTIONAL) - leave blank if you dont want to run a post processing script on the files
33 | # You can set a command to be ran on the file once it is completed. This can be any sort of a script you would like.
34 | # You can create a script to convert the video via ffmpeg to make it compatible for certain devices, create a contact sheet of the video
35 | # upload the video to a cloud storage drive via rclone, or whatever else you see fit.
36 | # set the string to be the same as you you would type into terminal to call the script manually.
37 | # The peramaters which will be passed to the script are as follows:
38 | # 1 = full file path (ie: /Users/Joe/MFC/208562/2017/2017.07.26_19.34.47_hannah.mp4)
39 | # 2 = filename (ie : 2017.07.26_19.34.47_hannah.mp4)
40 | # 3 = directory (ie : /Users/Joe/MFC/208562/hannah/2017/)
41 | # 4 = models name (ie: hannah)
42 | # 5 = uid (ie: 208562) (explained above)
43 | # to call a bash script called "MoveToGoogleDrive.sh" and located in the user Joes home directory, you would use:
44 | # postProcessingCommand = "bash /Users/Joe/home/MoveToGoogleDrive.sh"
45 | # this script will be ran on the files "download location" prior to it being moved to its "completed location".
46 | # The moving of the file will not take place if a post processing script is ran, so if you want to move it once it is completed, do so through commands in the post processing script.
47 |
48 | post_processing_command =
49 |
50 |
51 | # Because depending on what the post processing script does, it may be demanding on the system.
52 | # Set the maximum number of concurrent post processing scripts you would like to be ran at one time.
53 | # (required if using a post processing script)
54 |
55 | post_processing_thread_count = 1
56 |
57 | # minimum number of viewers required in a room before the model will be recorded. 0 = disabled
58 | # if stopViewers is greater that this number, this number will be set to stopViewers
59 | min_viewers = 0
60 |
61 | # stop recording if the number of viewers becomes less than stopViewers value. This should not be more than "minViewers" value.
62 | # if it is greater than minViewers value, minViewers will be automatically adjusted to the value of stopViewers.
63 | # I would suggest keeping a fair difference between minViewers and stopViewers, otherwise you may get many very short clips if the
64 | # number of viewers in a room is "teetering" between the two numbers.
65 | # 0 = disabled
66 | stop_viewers = 0
67 |
68 | # only records models with priority value greater than or equal to this number
69 | priority = 0
70 |
71 |
72 | [auto_recording]
73 | # This section is for grabbing models who are not on the wanted list, but meet certian conditions.
74 | # for all settings 0 is disabled.
75 | # a blacklist file must be set and must exist to use these settings. The the minViewers setting above applies.
76 |
77 | # start recording if the number of viewers in there room is greater than "viewers" variable
78 | # if autoStopViewers is greater that this number, this number will be set to autoStopViewers
79 | viewers = 0
80 |
81 | # same as "stopViewers" from above, but only applies to the models who are not on the wanted list grabbed
82 | auto_stop_viewers = 0
83 |
84 | # start recording if camscore is greater than "score" variable
85 | score = 0
86 |
87 | # start recording any model whos account was created less than "newerThanHours" hours ago.
88 | newer_than_hours = 0
89 |
90 | # automatically record models who are not in the wanted list, but have set certain tags.
91 | # set the desired tags, with each tag separated by a comma.
92 | # set the minimum number of tags required to start recording a model.
93 | tags = 0
94 | min_tags = 0
95 | # minimum numbers of viewers to start tag recordings
96 | # if tag_stop_viewers is greater that this number, this number will be set to tag_stop_viewers
97 | tag_viewers = 0
98 | # number of viewers to stop tag recordings
99 | tag_stop_viewers = 0
100 |
101 | [web]
102 | port = 8778
103 | enabled = true
104 | # login username for webapp
105 | username = user
106 | # password for webapp
107 | # Dont't put plain text here, use classes.helpers.hash_password(). Default password is 'change this'
108 | password = CVezFsDBdW4+0cgMV1VEnvbqferiQKliMWTcDN4+mkge3a9QWE/jYH8+x5xDocAoSD7pmxv48q8FPbq4Y6ZQiA==
--------------------------------------------------------------------------------
/webapp/templates/config.html:
--------------------------------------------------------------------------------
1 | {% extends 'layout.html' %}
2 |
3 | {% block head %}
4 | {{ super() }}
5 | MFCRecorder - config
6 |
7 |
8 | {% endblock %}
9 |
10 | {% macro input(name, label, value, tooltip='', width_class='w30', type='text') %}
11 |
set the directory structure - default is "{path}/{uid}/{year}.{month}.{day}_{hour}.{minutes}.{seconds}_{model}.mp4"
36 |
{auto} = for autoRecording conditions below, it is the reason why the recording was started followed by "_" - These are: TAGS_, VIEWERS_, SCORE_, and NEW_. If a model is in the wanted list, this will be blank
37 |
{path} = save_directory set above. (your directory structure should start with this)
38 |
{model} = the models name
39 |
{uid} = models uid or broadcaster id. This is a number which is always consistent. The models name can be changed, so this is what I find best for keeping all videos of a model in the same directory
40 |
{year}, {month}, {day}, {hour}, {minutes}, {seconds} should all be easy enough to figure out what these are
41 |
you can also change ".mp4" to another extension. ".ts" is what the stream is actual broadcast as.
42 |
example using a madeup "hannah" who has the uid 208562: {path}/{uid}/{year}/{year}.{month}.{day}_{hour}.{minutes}.{seconds}_{model}.mp4 = "/Users/Joe/MFC/208562/2017/2017.07.26_19.34.47_hannah.mp4"
43 |
This will also be the "Download directory" or temp directory if you set a "completed_directory"
this script will be ran on the files "download location" prior to it being moved to its "completed location".
72 |
The moving of the file will not take place if a post processing script is ran, so if you want to move it once it is completed, do so through commands in the post processing script.
153 | {% endblock %}
--------------------------------------------------------------------------------
/classes/config.py:
--------------------------------------------------------------------------------
1 | import configparser
2 | import time
3 | import os
4 | import platform
5 | import ctypes
6 | import json
7 | import threading
8 | import classes.helpers as helpers
9 |
10 | LIST_MODE_WANTED = 0
11 | LIST_MODE_BLACKLISTED = 1
12 |
13 | class Settings():
14 | def __init__(self, parser, make_absolute):
15 | self._make_absolute = make_absolute
16 | self.conf_save_directory = parser.get('paths', 'save_directory')
17 | self.conf_wishlist_path = parser.get('paths', 'wishlist_path')
18 | self.interval = parser.getint('settings', 'check_interval')
19 | self.directory_structure = parser.get('paths', 'directory_structure').lower()
20 | self.post_processing_command = parser.get('settings', 'post_processing_command')
21 | self.post_processing_thread_count = parser.getint('settings', 'post_processing_thread_count')
22 | self.port = parser.getint('web', 'port')
23 | self.web_enabled = parser.getboolean('web', 'enabled')
24 | self.min_space = parser.getint('settings', 'min_space')
25 | self.conf_completed_directory = parser.get('paths', 'completed_directory').lower()
26 | self.priority = parser.getint('settings', 'priority')
27 | self.username = parser.get('web', 'username')
28 | self.password = parser.get('web', 'password')
29 |
30 | #make save directory so that _get_free_diskspace can work
31 | os.makedirs(self.save_directory, exist_ok=True)
32 |
33 | @property
34 | def save_directory(self):
35 | return self._make_absolute(self.conf_save_directory)
36 |
37 | @property
38 | def wishlist_path(self):
39 | return self._make_absolute(self.conf_wishlist_path)
40 |
41 | @property
42 | def completed_directory(self):
43 | return self._make_absolute(self.conf_completed_directory)
44 |
45 | class Filter():
46 | def __init__(self, parser, settings):
47 | self.newer_than_hours = parser.getint('auto_recording', 'newer_than_hours')
48 | self.score = parser.getint('auto_recording', 'score')
49 | self.auto_stop_viewers = parser.getint('auto_recording', 'auto_stop_viewers')
50 | self.stop_viewers = parser.getint('settings', 'stop_viewers')
51 | self.min_tags = max(1, parser.getint('auto_recording', 'min_tags'))
52 | self._wanted_tags_str = parser.get('auto_recording', 'tags')
53 | self._update_tags()
54 | self.tag_stop_viewers = parser.getint('auto_recording', 'tag_stop_viewers')
55 | #account for when stop is greater than min
56 | self.min_viewers = max(self.stop_viewers, parser.getint('settings', 'min_viewers'))
57 | self.viewers = max(self.auto_stop_viewers, parser.getint('auto_recording', 'viewers'))
58 | self.tag_viewers = max(self.tag_stop_viewers, parser.getint('auto_recording', 'tag_viewers'))
59 |
60 | self.wanted = Wanted(settings)
61 |
62 | @property
63 | def wanted_tags_str(self):
64 | return self._wanted_tags_str
65 |
66 | @wanted_tags_str.setter
67 | def wanted_tags_str(self, value):
68 | self._wanted_tags_str = value
69 | self._update_tags()
70 |
71 | def _update_tags(self):
72 | self.wanted_tags = {s.strip().lower() for s in self._wanted_tags_str.split(',')}
73 |
74 | class Config():
75 | def __init__(self, config_file_path):
76 | self._lock = threading.Lock()
77 | self._config_file_path = config_file_path
78 | self._parser = configparser.ConfigParser()
79 | self.refresh()
80 |
81 | @property
82 | def settings(self):
83 | return self._settings
84 |
85 | @property
86 | def filter(self):
87 | return self._filter
88 |
89 | def _make_absolute(self, path):
90 | if not path or os.path.isabs(path):
91 | return path
92 | return os.path.join(os.path.dirname(self._config_file_path), path)
93 |
94 | def refresh(self):
95 | '''load config again to get fresh values'''
96 | self._parse()
97 | self._settings = Settings(self._parser, self._make_absolute)
98 | self._filter = Filter(self._parser, self.settings)
99 | self._available_space = self._get_free_diskspace()
100 |
101 | def _parse(self):
102 | with self._lock:
103 | self._parser.read(self._config_file_path)
104 |
105 | def update(self, data):
106 | '''expects a dictionary with section:option as key and the value as value'''
107 | #will delete comments in the config, but when this method is used, config was edited in webapp,
108 | #so there are comments there and in the sample config
109 | with self._lock:
110 | for key, value in data.items():
111 | section, option = key.split(':')
112 | self._parser.set(section, option, value)
113 | self._write()
114 | self.refresh()
115 |
116 | def _write(self):
117 | with open(self._config_file_path, 'w') as target:
118 | self._parser.write(target)
119 |
120 | #maybe belongs more into a filter class, but then we would have to create one
121 | def does_model_pass_filter(self, model):
122 | '''determines whether a recording should start'''
123 | f = self.filter
124 | try:
125 | if f.wanted.is_wanted(model.uid):
126 | #TODO: do we want a global min_viewers if model specific is not set??
127 | m_settings = f.wanted.dict[model.uid]
128 | if model.session['rc'] < max(m_settings['min_viewers'], m_settings['stop_viewers']):
129 | return False
130 | else:
131 | model.session['condition'] = helpers.Condition.WANTED
132 | return True
133 | if f.wanted.is_blacklisted(model.uid):
134 | return False
135 | if f.wanted_tags:
136 | matches = f.wanted_tags.intersection(model.tags if model.tags is not None else [])
137 | if len(matches) >= f.min_tags and model.session['rc'] >= f.tag_viewers:
138 | model.session['condition'] = helpers.Condition.TAGS
139 | model.session['condition-text'] = ','.join(matches)
140 | return True
141 | if f.newer_than_hours and model.session['creation'] > int(time.time()) - f.newer_than_hours * 60 * 60:
142 | model.session['condition'] = helpers.Condition.NEW
143 | return True
144 | if f.score and model.session['camscore'] > f.score:
145 | model.session['condition'] = helpers.Condition.SCORE
146 | return True
147 | if f.viewers and model.session['rc'] > f.viewers:
148 | model.session['condition'] = helpers.Condition.VIEWERS
149 | return True
150 | return False
151 | except Exception as e:
152 | print(e)
153 | return False
154 |
155 | def _get_free_diskspace(self):
156 | '''https://stackoverflow.com/questions/51658/cross-platform-space-remaining-on-volume-using-python'''
157 | if platform.system() == 'Windows':
158 | free_bytes = ctypes.c_ulonglong(0)
159 | ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(self.settings.save_directory), None, None, ctypes.pointer(free_bytes))
160 | return free_bytes.value / 1024 / 1024
161 | st = os.statvfs(self.settings.save_directory)
162 | return st.f_bavail * st.f_frsize / 1024 / 1024 / 1024
163 |
164 | def keep_recording(self, session):
165 | '''determines whether a recording should continue'''
166 | try:
167 | #would it be possible that no entry is existing if we are already recording?
168 | #TODO: global stop_viewers if no model specific is set??
169 | if session['condition'] == helpers.Condition.VIEWERS:
170 | min_viewers = self.filter.auto_stop_viewers
171 | elif session['condition'] == helpers.Condition.WANTED:
172 | min_viewers = self.filter.wanted.dict[session['uid']]['stop_viewers']
173 | elif session['condition'] == helpers.Condition.TAGS:
174 | min_viewers = self.filter.tag_stop_viewers
175 | else:
176 | min_viewers = 0
177 | return session['rc'] >= min_viewers and self._available_space > self.settings.min_space
178 | except Exception as e:
179 | print(e)
180 | return True
181 |
182 | class Wanted():
183 | def __init__(self, settings):
184 | self._lock = threading.RLock()
185 | self._settings = settings
186 | #create new empty wanted file
187 | try:
188 | with open(self._settings.wishlist_path, 'x') as file:
189 | file.write('{}')
190 | except FileExistsError:
191 | pass
192 | self._load()
193 |
194 | def _load(self):
195 | with self._lock:
196 | with open(self._settings.wishlist_path, 'r+') as file:
197 | self.dict = {int(uid): data for uid, data in json.load(file).items()}
198 |
199 | def _save(self):
200 | with open(self._settings.wishlist_path, 'w') as file:
201 | json.dump(self.dict, file, indent=4)
202 |
203 | def set_dict(self, data):
204 | '''expects dictionary with uid:key as keys and value as value'''
205 |
206 | #building the new wanted dict
207 | new = {}
208 | for key, value in data.items():
209 | uid, key = key.split(':')
210 | uid = int(uid)
211 | #relies on enabled being the first argument that is passed per model, maybe a bit dirty
212 | if key == 'enabled':
213 | new[uid] = {}
214 | print(value)
215 | new[uid][key] = helpers.try_eval(value)
216 |
217 | with self._lock:
218 | self.dict = new
219 | self._save()
220 |
221 | def add(self, uid, custom_name='', list_mode=LIST_MODE_WANTED):
222 | '''Adds model to dict and returns None. If already existing, returns model settings.'''
223 | with self._lock:
224 | settings = self.dict.get(uid)
225 | if settings is not None:
226 | return settings
227 | self._set_data(uid, list_mode=list_mode, custom_name=custom_name)
228 |
229 | def remove(self, uid):
230 | '''removes model from dict and returns settings, if not existing returns None'''
231 | with self._lock:
232 | result = self.dict.pop(uid, None)
233 | self._save()
234 | return result
235 |
236 | def _set_data(self, uid, enabled=True, list_mode=LIST_MODE_WANTED,
237 | custom_name='', comment='', min_viewers=0, stop_viewers=0, priority=0):
238 | '''same as _set_data_dict, but takes named arguments instead of a dict'''
239 | data = {
240 | 'enabled': enabled,
241 | 'list_mode': list_mode,
242 | 'custom_name': custom_name,
243 | 'comment': comment,
244 | 'min_viewers': min_viewers,
245 | 'stop_viewers': stop_viewers,
246 | 'priority': priority,
247 | }
248 | with self._lock:
249 | self._set_data_dict(uid, data)
250 |
251 | def _set_data_dict(self, uid, data):
252 | '''Set data dictionary for model uid, existing or not'''
253 | with self._lock:
254 | self.dict[uid] = data
255 | self._save()
256 |
257 | def is_wanted(self, uid):
258 | '''determines if model is enabled and wanted'''
259 | return self._is_list_mode_value(uid, LIST_MODE_WANTED)
260 |
261 | def is_blacklisted(self, uid):
262 | '''determines if model is enabled and blacklisted'''
263 | return self._is_list_mode_value(uid, LIST_MODE_BLACKLISTED)
264 |
265 | def _is_list_mode_value(self, uid, value):
266 | '''determines if list_mode equals the specified one, but only if the item is enabled'''
267 | entry = self.dict.get(uid)
268 | if not (entry and entry['enabled'] and self._settings.priority <= entry['priority']):
269 | return False
270 | return entry['list_mode'] == value
271 |
--------------------------------------------------------------------------------
/scripts/merge.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | #created GitHub user sKanoodle
3 | import os, subprocess, argparse, time, re
4 | from datetime import datetime, timedelta
5 |
6 | #directory with model ID subdirectories
7 | sourcefolder = "/home/user/MFC/src"
8 | #directory to save the encoded files in
9 | destinationfolder = "/home/user/MFC/encoded"
10 | #creation date regex (works for default file names, if changed in config change here as well)
11 | #is applied to the whole path, so custom folders (for example for the year) are possible here
12 | creationregex = '(?P\d{4}).(?P\d{2}).(?P\d{2})_(?P\d{2})\.(?P\d{2})\.(?P\d{2})'
13 | #logfile path (leave as empty string if no logging is desired)
14 | logfilepath = "/home/user/MFC/encoding.log"
15 | #{0} is the absoulte source file path, {1} is the absolute target file path
16 | ffmpegcommand = "ffmpeg -loglevel quiet -i {0} -vcodec libx264 -crf 23 {1}"
17 | #extension of the encoded file
18 | extension = ".mp4"
19 | #{0} is the absolute path of the file with parts to concat, {1} is the absolute target file path
20 | ffmpegmergecommand = "ffmpeg -v error -f concat -safe 0 -i {0} -c copy {1}"
21 | #filename must not exist already directly in the sourcefolder
22 | tmpconcatfilename = "concat.mp4"
23 | #max time in minutes that is allowed between the end of a video and the beginning of the next video to concatinate them
24 | concatmaxtime = 60
25 | #time in minutes that has to have passed since the last modification of a recording to include it for encoding
26 | #(should always be larger than concatmaxtime, otherwise the file will be encoded even if a next file would have been eligible to be concatinated to it)
27 | ignorefreshvideostime = 60
28 | #datetime format for logging purposes
29 | datetimeformat = "{:%Y-%m-%d %X}"
30 |
31 | parser = argparse.ArgumentParser()
32 | parser.add_argument("-d", "--dryrun", action="store_true", dest="dryrun", help="Simulates encoding of all files in the source folder. Size and duration of some videos might differ, because there is no concatination performed, although the status output expects concatinated videos. It will therefore only show size and duration of the first file that should be concatinated.")
33 | parser.add_argument("-c", "--copy", action="store_true", dest="copy", help="Only copies the video files instead of encoding them, but still merges them beforehand.")
34 | parser.add_argument("-r", "--remove", action="store_true", dest="remove", help="Deletes video when detected as faulty when trying to merge videos, otherwise the file will just be ignored")
35 | args = parser.parse_args()
36 |
37 | def log_and_print(string):
38 | if not args.dryrun and logfilepath:
39 | with open(logfilepath, "a") as file:
40 | file.write(string + "\n")
41 | print(string)
42 |
43 | def format_seconds(totalseconds):
44 | totalseconds = int(totalseconds)
45 | totalminutes, seconds = divmod(totalseconds, 60)
46 | totalhours, minutes = divmod(totalminutes, 60)
47 | return "{0}:{1:02d}:{2:02d}".format(totalhours, minutes, seconds)
48 |
49 | def get_video_length_seconds(path):
50 | if not os.path.exists(path):
51 | return 0
52 | try:
53 | lengthraw = subprocess.check_output("ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 {0}".format(path), shell=True)
54 | return float(lengthraw.strip())
55 | except:
56 | return 0
57 |
58 | def get_file_encoding_infos(sourcepath):
59 | if args.dryrun:
60 | if not os.path.exists(sourcepath):
61 | return {"source": sourcepath, "target": "", "size": 0, "length": 0}
62 |
63 | #relies on set structure: sourcefolder/modelID/video
64 | directory, file = os.path.split(sourcepath)
65 | filename, ext = os.path.splitext(file)
66 | return {"source": sourcepath,
67 | "target": os.path.join(destinationfolder, os.path.basename(directory), filename + extension),
68 | "size": os.path.getsize(sourcepath) / 1024 / 1024,
69 | "length": get_video_length_seconds(sourcepath)}
70 |
71 | def parse_creation_time(path):
72 | m = re.search(creationregex, path)
73 | if not m:
74 | print('error in creation date regex')
75 | return
76 | dict = {k:int(v) for k, v in m.groupdict().items()}
77 | return datetime(dict['year'], dict['month'], dict['day'], dict['hour'], dict['minute'], dict['second'])
78 |
79 | def calculate_eta(starttime, progress):
80 | if progress <= 0:
81 | return "calculating ETA"
82 | if progress >= 1:
83 | return "done at {}".format(datetimeformat.format(datetime.now()))
84 | passedseconds = (datetime.now() - starttime).total_seconds()
85 | estimatedduration = passedseconds / progress
86 | return "ETA: {}".format(datetimeformat.format(starttime + timedelta(seconds=estimatedduration)))
87 |
88 | def concat_files(files, name):
89 | log_and_print("{0} merging into {1}:".format(datetimeformat.format(datetime.now()), name))
90 | for file in files:
91 | log_and_print("[{:>10,.2f} MiB] [{}] {}".format(os.path.getsize(file) / 1024 / 1024, format_seconds(get_video_length_seconds(file)), file))
92 | mergefilepath = os.path.join(os.path.dirname(name), "tempmergefile.txt")
93 | tmp = os.path.join(sourcefolder, tmpconcatfilename)
94 | ffmpeg = ffmpegmergecommand.format(mergefilepath, tmp)
95 | if args.dryrun:
96 | print("[DRYRUN] would create mergefile {0}".format(mergefilepath))
97 | print("[DRYRUN] would run {0}".format(ffmpeg))
98 | print("[DRYRUN] would move {0} to {1}".format(tmp, name))
99 | else:
100 | #create mergefile with info about parts
101 | mergefile = open(mergefilepath, "w")
102 | for file in files:
103 | mergefile.write("file '{0}'\n".format(file))
104 | mergefile.close()
105 | #concat videos
106 | os.system(ffmpeg)
107 | #delete source parts
108 | for file in files:
109 | os.remove(file)
110 | #move concatinated video from temp location to final location
111 | os.rename(tmp, name)
112 | #remmove the mergefile
113 | os.remove(mergefilepath)
114 |
115 |
116 | def merge_files_in_model_directory(directory):
117 | #the files need to be scanned into a list first, so we can look ahead to the next file
118 | entries = []
119 | for file in sorted(os.listdir(directory)):
120 | filepath = os.path.join(directory, file)
121 | if not file.endswith(".mp4"):
122 | continue
123 | #detects empty files
124 | length = get_video_length_seconds(filepath)
125 | if not length:
126 | if args.remove:
127 | log_and_print("removing empty or faulty video file: {}".format(filepath))
128 | os.remove(filepath)
129 | else:
130 | log_and_print("ignoring empty or faulty video file: {}".format(filepath))
131 | continue
132 | entries.append({"creation": parse_creation_time(file),
133 | "modification": datetime.fromtimestamp(os.path.getmtime(filepath)),
134 | "length": length,
135 | "file": filepath})
136 |
137 | #now we can traverse the files we found and check if the next file is directly following the previous and merge them if necessary
138 | filestoencode = []
139 | concatlist = []
140 | for i in range(len(entries)):
141 | #last run of the loop, we dont want further execution here, just adding the last file/performing the last concatination
142 | if i == len(entries) - 1:
143 | #make sure the latest file is not written to anymore
144 | if entries[i]["modification"] + timedelta(minutes=ignorefreshvideostime) > datetime.now():
145 | log_and_print("ignoring {0} and possible previous mergable files".format(entries[i]["file"]))
146 | #exit the loop and dont add the files in the concat list to filestoencode, so we can merge them the next time this script runs
147 | break
148 | #last file is not being merged, add it to the filestoencode list
149 | if len(concatlist) < 2:
150 | filestoencode.append(get_file_encoding_infos(entries[i]["file"]))
151 | #last file is being merged, merge and then add merged file to the filestoencode list
152 | else:
153 | concat_files(concatlist, concatlist[0])
154 | filestoencode.append(get_file_encoding_infos(concatlist[0]))
155 | concatlist = []
156 | break
157 | #print("{3} {0} {1} {2}".format(entries[i]["creation"], entries[i]["modification"], entries[i]["file"], i))
158 | m = entries[i]["modification"]
159 | c = entries[i + 1]["creation"]
160 | #current file has a following up file that needs to be merged
161 | if m < c and m + timedelta(minutes=concatmaxtime) > c:
162 | if not entries[i]["file"] in concatlist:
163 | concatlist.append(entries[i]["file"])
164 | concatlist.append(entries[i + 1]["file"])
165 | #there is nothig more to be merged
166 | else:
167 | #concatlist is empty, so treat the file as normal video to be encoded
168 | if len(concatlist) < 1:
169 | filestoencode.append(get_file_encoding_infos(entries[i]["file"]))
170 | continue
171 | #concat list has a single entry, should never happen, because there is nothing to concat
172 | elif len(concatlist) == 1:
173 | log_and_print("single file in concat list?????? {0}".format(concatlist[0]))
174 | #concat the files and then encode the resulting new video
175 | else:
176 | concat_files(concatlist, concatlist[0])
177 | filestoencode.append(get_file_encoding_infos(concatlist[0]))
178 | concatlist = []
179 | return filestoencode
180 |
181 | def merge_and_encode_everything():
182 | print("finding files to encode ...", end="\r")
183 |
184 | entries = []
185 | #each ID in the source folder
186 | for id in os.listdir(sourcefolder):
187 | #ID-directory
188 | dir = os.path.join(sourcefolder, id)
189 | if os.path.isdir(dir):
190 | entries.extend(merge_files_in_model_directory(dir))
191 |
192 | index = 0
193 | #hack to prevent division by 0
194 | totalsize = max(sum([entry["size"] for entry in entries]), 1)
195 | sizedone = 0
196 | #hack to prevent division by 0
197 | totallength = max(sum([entry["length"] for entry in entries]), 1)
198 | lengthdone = 0
199 | starttime = datetime.now()
200 | progresstemplate = " {0:,.2f}/{1:,.2f} MiB ({2:.2%}) [{7}] | {3}/{4} ({5:.2%}) [{8}] | [{6}]"
201 |
202 | def get_stats():
203 | return [sizedone,
204 | totalsize,
205 | sizedone / totalsize,
206 | format_seconds(lengthdone),
207 | format_seconds(totallength),
208 | lengthdone / totallength,
209 | format_seconds((datetime.now() - starttime).total_seconds()),
210 | calculate_eta(starttime, sizedone / totalsize),
211 | calculate_eta(starttime, lengthdone / totallength)]
212 |
213 | for entry in entries:
214 | #create encoding target folder in case it doesnt exist
215 | if not args.dryrun and not os.path.exists(os.path.dirname(entry["target"])):
216 | os.makedirs(os.path.dirname(entry["target"]))
217 | index += 1
218 | log_and_print("{5} {0}: [{1:>10,.2f} MiB] [{2}] source: {3}, target: {4}"
219 | .format(datetimeformat.format(datetime.now()), entry["size"], format_seconds(entry["length"]), entry["source"], entry["target"], "{0}/{1}".format(index, len(entries)).rjust(9)))
220 | #print with carriage return at the end, so that this line can be overwritten by the next print
221 | print(progresstemplate.format(*get_stats()), end="\r")
222 | sizedone += entry["size"]
223 | lengthdone += entry["length"]
224 | if not args.dryrun:
225 | if not args.copy:
226 | #actual call to encode the video
227 | os.system(ffmpegcommand.format(entry["source"], entry["target"]))
228 | os.remove(entry["source"])
229 | else:
230 | #only move the video file without encoding
231 | os.rename(entry["source"], entry["target"])
232 |
233 | #final progress, should always show 100%
234 | print(progresstemplate.format(*get_stats()))
235 |
236 | merge_and_encode_everything()
237 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # MFCRecorder
2 |
3 | This is script automates the recording of public webcam shows from myfreecams.
4 |
5 |
6 | ## Requirements
7 |
8 | I have only tested this on debian(7+8) and Mac OS X (10.10.4), but it should run on other OSs
9 |
10 | Requires python3.6 or newer. You can grab the newest python release from https://www.python.org/downloads/
11 | and mfcauto.py (https://github.com/Damianonymous/mfcauto.py)
12 |
13 | ## installing and Cloning with the required modules:
14 |
15 | to install the required modules, run: (For Debain/Ubuntu)
16 | ```
17 | sudo apt-install update && sudo apt-install upgrade
18 | sudo apt-install python3-pip && sudo apt-get install git
19 | cd /home/yourusername
20 | git clone https://github.com/Damianonymous/MFCRecorder
21 | cd MFCRecorder
22 | python3.6 -m pip install -r requirements.txt
23 | python3.6 -m pip install --upgrade git+https://github.com/Damianonymous/mfcauto.py@master
24 | Now edit the config.conf file and set the appropirate paths to your directories and wanted.txt file (see Setup)
25 |
26 | ```
27 |
28 | to install required modules, run: (For Arch Linux, Antergos, Manjaro, etc.)
29 | ```
30 | pacman -Syuu
31 | pacman -S python-pip git
32 | cd /home/yourusername
33 | git clone https://github.com/Damianonymous/MFCRecorder
34 | cd MFCRecorder
35 | python3.6 -m pip install -r requirements.txt
36 | python3.6 -m pip install --upgrade git+https://github.com/Damianonymous/mfcauto.py@master
37 | Now edit the config.conf file and set the appropirate paths to your directories and wanted.txt file (see Setup)
38 |
39 | ```
40 |
41 | to install the required modules, run: (For CentOS/Red Hat/Fedora) NOT TESTED BUT GOES LIKE:
42 | ```
43 | yum update
44 | yum upgrade
45 | yum python-pip
46 | yum install git
47 | cd /home/yourusername
48 | git clone https://github.com/Damianonymous/MFCRecorder
49 | cd MFCRecorder
50 | python3.6 -m pip install -r requirements.txt
51 | python3.6 -m pip install --upgrade git+https://github.com/Damianonymous/mfcauto.py@master
52 | Now edit the config.conf file and set the appropirate paths to your directories and wanted.txt file (see Setup)
53 |
54 | ```
55 |
56 | ## Setup
57 |
58 | edit the config.conf file and set the appropirate paths to your directories and wanted.txt file.
59 |
60 |
61 | ## adding models to wanted list
62 |
63 | Add models UID (user ID) to the "wanted.txt" file (only one model per line). This uses the UID instead of the name becaue the models can change their name at anytime, but their UID always stays the same. There is a number of ways to get the models UID, but the easiest would probably be to get it from the URL for their profile image. The profile image URL is formatted as (or similar to):
64 | ```
65 | https://img.mfcimg.com/photos2/###/{uid}/avatar.90x90.jpg
66 | ```
67 | "{uid}" is the models UID which is the number you will want to add to the "wanted.txt" file. the "###" is the first 3 digits of the models UID. For example, if the models UID is "123456789" the URL for their profile picture will be:
68 | ```
69 | https://img.mfcimg.com/photos2/123/123456789/avatar.90x90.jpg
70 | ```
71 |
72 | alternatively, you can use the add.py script to add models to the MFC list to enable or disable their recordings. If a model already exists, her values will be updated to any new values passed in the arguments. (requires python3.5 or newer)
73 |
74 | Its usage is as follows:
75 |
76 | add.py [model display name or uid] [options]
77 |
78 | Add models to the MFC list to enable or disable their recordings. If a model
79 | already exists, her values will be updated to any new values passed in the
80 | arguments
81 |
82 | positional arguments:
83 | model REQUIRED: models name or uid.
84 |
85 | optional arguments:
86 | -h, --help show this help message and exit
87 | -n CUSTOM_NAME, --custom_name CUSTOM_NAME
88 | set a custom name for the model, otherwise the models
89 | current display name will be used.
90 | -c COMMENT, --comment COMMENT
91 | specify a comment or not for the user.
92 | -m MIN_VIEWERS, --min_viewers MIN_VIEWERS
93 | set the minimum number of viewers this model must have
94 | before recording starts
95 | -s STOP_VIEWERS, --stop_viewers STOP_VIEWERS
96 | set the number of viewers in which the recording will
97 | stop (should be less than minviewers
98 | -l LIST_MODE, --list_mode LIST_MODE
99 | set the list mode for the model
100 | -b, --block will add the model as blocked so she will not be
101 | recorded even if auto recording conditions are met
102 | -p PRIORITY, --priority PRIORITY
103 | set the priority value for the model
104 |
105 | Not passing options will add a model with all the default values, and the models current display name will be the default custom name if custom_name is not specified.
106 | ```
107 | python3 add.py AspenRae
108 | ```
109 |
110 | ## Web Interface (added Sep 4 2017)
111 | There is also a web interface now, although very limited. It currently only displays the recording models along with some system statistics. If you click on a models avatar, it will open their chatroom on myfreecams site. If you click on their name above their image, it will load their profile. Below their avatar it indicates the condition for their recording (wanted, tags, viewers...) as well as the number of viewers in their room at the moment. In addition to this, there is an "Add Model" text box where you can add a model by typing in their username and submitting it. (Doing anything with the web interface, html, or css is completely new to me, so I plan to add more features, but it will take me some time since Im learning as I go.)
112 |
113 | The default port is 8778, but it can be changed in the config file. So to view the web interface, open (https://127.0.0.1:8778/) in any web browser (note the https). If you want to access it from another computer on the same lan, replace the ip address with the ip address of that machine.
114 |
115 | 
116 |
117 |
118 | ## Additional options
119 |
120 | you can now set a custom "completed" directory where the videos will be moved when the stream ends. The variables which can be used in the naming are as follows:
121 |
122 | **{path}** = the value set to "save directory"
123 |
124 | **{model}** = the display name of the model
125 |
126 | **{uid}** = the uid (user id) or broadcasters id as its often reffered in MFCs code which is a static number for the model
127 |
128 | **{year}** = the current 4 digit year (ie:2017)
129 |
130 | **{month}** = the current two digit month (ie: 01 for January)
131 |
132 | **{day}** = the two digit day of the month
133 |
134 | **{hour}** = the two digit hour in 24 hour format (ie: 1pm = 13)
135 |
136 | **{minute}** = the current minute value in two digit format (ie: 1:28 = 28)
137 |
138 | **{seconds}** = the current times seconds value in 2 digit format
139 |
140 | **{auto}** = reason why the model was recorded if not in wanted list (see auto recording based on conditions below)
141 |
142 | For example, if a made up model named "hannah" who has the uid 208562, and the "save_directory" in the config file == "/Users/Joe/MFC/": {path}/{uid}/{year}/{year}.{month}.{day}_{hour}.{minutes}.{seconds}_{model}.mp4 = "/Users/Joe/MFC/208562/2017/2017.07.26_19.34.47_hannah.mp4"
143 |
144 |
145 | You can create your own "post processing" script which can be called at the end of the stream. The parameters which will be passed to the script are as follows:
146 |
147 | 1 = full file path (ie: /Users/Joe/MFC/208562/2017/2017.07.26_19.34.47_hannah.mp4)
148 |
149 | 2 = filename (ie : 2017.07.26_19.34.47_hannah.mp4)
150 |
151 | 3 = directory (ie : /Users/Joe/MFC/208562/hannah/2017/)
152 |
153 | 4 = models name (ie: hannah)
154 |
155 | 5 = uid (ie: 208562 as given in the directory/file naming structure example above)
156 |
157 |
158 | ## Conditional recording
159 |
160 | In the config file you can specify conditions in which models who are not in the wanted list should be recorded. There is also a blacklist you can create and add models UID to if you want to specify models who will not be recorded even if these conditions are met.
161 |
162 | **Tags**: you can add a comma (,) separated list of tags which models will be checked against each models specified tags.
163 |
164 | **minTags**: indicates the minimum number of tags from the "Tags" option which must be met to start recording a model.
165 |
166 | **newerThanHours**: If a model has joined the site in less than the number of hours specified here, the model will be recorded until she has been a model for longer than this time (it will continue to record any active recording started prior to this time).
167 |
168 | **score**: any model with a camscore greater than this number will be recorded
169 |
170 | **viewers**: when a model reachest this number of viewers in her chatroom, she will be recorded. This can be used to catch models as many users are entering the chat which usually indicates some sort of show has started.
171 |
172 | **autoStopViewers**: This only apples to models who are being recorded based on the viewers condition above. The session will stop recording when the number of viewers drops below this number. Make sure there is enough of a difference between these two numbers (viewers and autoStopViewers) to avoid the show continuously starting and stopping as the number of viewers moves above/below these numbers.
173 |
174 |
175 |
176 |
177 | # User Submitted Scripts
178 |
179 | User submitted scripts can be found in the 'scripts' directory. These are not scripts which are created by me (beaston02), but other users who are sharing with the comunity.
180 |
181 | ## merge.py
182 | Created by [sKanoodle](https://github.com/sKanoodle)
183 |
184 | This script will encode and merge recordings from individual models.
185 |
186 | #### SETTINGS
187 |
188 | **sourcefolder**: directory with model ID subdirectories
189 |
190 | **destinationfolder**: directory to save the encoded files in
191 |
192 | **creationregex**: regex for parsing creation date and time from filename
193 |
194 | **logfilepath**: logfile path (leave as empty string if no logging is desired)
195 |
196 | **ffmpegcommand**: {0} is the absoulte source file path, {1} is the absolute target file path
197 |
198 | **extension**: extension of the encoded file
199 |
200 | **ffmpegmergecommand**: {0} is the absolute path of the file with parts to concat, {1} is the absolute target file path
201 |
202 | **tmpconcatfilename**: name to use for the temp file. filename must not exist already directly in the sourcefolder
203 |
204 | **concatmaxtime**: max time in minutes that is allowed between the end of a video and the beginning of the next video to concatinate them
205 |
206 | **ignorefreshvideostime**: time in minutes that has to have passed since the last modification of a recording to include it for encoding. (should always be larger than concatmaxtime, otherwise the file will be encoded even if a next file would have been eligible to be concatinated to it)
207 |
208 | **datetimeformat**: format of time and date in the file names
209 |
210 |
211 |
212 | #### OPTIONS
213 |
214 | **-d, --dryrun**: Simulates encoding of all files in the source folder. Size and duration of some videos might differ, because there is no concatination performed, although the status output expects concatinated videos. It will therefore only show size and duration of the first file that should be concatinated
215 |
216 |
217 | **-c, --copy**: Only copies the video files instead of encoding them, but still merges them beforehand
218 |
219 | **-r, --remove**: Deletes video when detected as faulty when trying to merge videos, otherwise the file will just be ignored
220 |
221 |
222 |
223 | #### NOTES
224 |
225 | only tested on linux.
226 | Can be ran as a cron job to automatically merge and encode files. Encoding should reduce the size of the files
227 |
228 | ## symlink.py
229 | Created by [sKanoodle](https://github.com/sKanoodle)
230 |
231 | #### SETTINGS
232 |
233 | **models**: a dictionary where the keys are the models UIDs, and the values are their usernames.
234 |
235 | **encodedfilesdir**: the directory containing the recorded videos
236 |
237 | **symlinkdir**: the directory where you want the recordings to be linked to using the models name instead of their UID as the directory name.
238 |
239 | **wantedfile**: the path to the wanted file used by MFCRecorder.
240 |
241 | #### NOTES
242 |
243 | This script will create a symlink for the models UID directories to a directory using the models name. This will make it easier to browse through the recorded files by having directories named after the models instead of their UIDs, while still keeping all of their recordings in a single directory if/when the model changes her display name.
244 |
245 | Only tested on linux
246 |
247 | ## postProcessing.py and test_postProcessing.py
248 |
249 | Example of a post processing script to encode (or move, filter, etc.) files directly after the recording ended. test_postProcessing.py emulates the call from the main script (MFCRecorder.py), so that postProcessing.py can be tested independently. For more information refer to the comments inside the scripts.
250 |
--------------------------------------------------------------------------------
![{{ session['nm'] }} thumbnail]({{ url_for('thumbnail', uid=uid) }})
34 |
42 |