├── .gitattributes ├── .gitignore ├── README.md ├── apis ├── Facebook.v14_0.oa3.json ├── Facebook.v20_0.oa3.json ├── Facebook.v4_0.oa3.json ├── Generic.CrowdTangle.oa3.json ├── Twitter.v1_1.oa3.json ├── Twitter.v2.oa3.json ├── TwitterStreaming.v1_1.oa3.json └── YouTube.v3.oa3.json ├── build ├── osx │ ├── Facepager.spec │ ├── build.pyinstaller.command │ ├── entitlements.plist │ ├── fix_app_qt_folder_names_for_codesign.py │ └── hooks │ │ └── hook-PySide2.QtWebEngineWidgets.py ├── readme.md └── windows │ ├── Facepager.spec │ ├── Facepager_Setupscript.nsi │ ├── build.bat │ ├── cchardet │ ├── __init__.py │ ├── _cchardet.cp37-win_amd64.pyd │ └── version.py │ ├── dlls │ ├── LIBEAY32.dll │ ├── SSLEAY32.dll │ ├── mfc90.dll │ ├── pythoncom27.dll │ └── pywintypes27.dll │ └── upx-3.96-win64 │ ├── BUGS │ ├── COPYING │ ├── LICENSE │ ├── NEWS │ ├── README │ ├── README.1ST │ ├── THANKS │ ├── upx.1 │ ├── upx.doc │ ├── upx.exe │ └── upx.html ├── docs ├── index.html ├── login_facebook.html ├── login_twitter.html ├── login_youtube.html ├── oauth_feedback.html └── oauth_waiting.html ├── icons ├── add.png ├── addcolumn.png ├── apis.png ├── apply.png ├── browser.png ├── clear.png ├── collapse.png ├── delete.png ├── expand.png ├── export.png ├── fetch.png ├── help.png ├── icon_facepager.icns ├── icon_facepager.ico ├── icon_facepager.png ├── icons.py ├── icons.qrc ├── more.png ├── new.png ├── presets.png ├── readme.md ├── save.png ├── search.png ├── timer.png ├── toclip.png ├── transfer.png └── unpack.png ├── presets ├── Amazon-Get_bucket_content.3_10.json ├── Amazon-Get_buckets.3_10.json ├── Bibliometrics-OpenAlex_1_Get_a_single_work.fp4.json ├── Bibliometrics-OpenAlex_2_Get_author.fp4.json ├── Bibliometrics-OpenAlex_3_Get_list_of_authors.fp4.json ├── Bibliometrics-OpenAlex_4_Get_list_of_references_.fp4.json ├── Bibliometrics-OpenAlex_5_Get_list_of_works_by_search_term.fp4.json ├── Bibliometrics-OpenAlex_6_Get_list_of_collaborations_between_countries.fp4.json ├── Bibliometrics-OpenCitations_Index_COCI_Metadata_from_doi.fp4.json ├── Bibliometrics-OpenCitations_Index_COCI_citations.fp4.json ├── Bibliometrics-OpenCitations_Index_COCI_references.fp4.json ├── Bibliometrics-Open_Library_Author_metadata_Authors_API_.fp4.json ├── Bibliometrics-Open_Library_Book_metadata_from_title_Search_API_.fp4.json ├── Crowdtangle-1_Get_lists.fp4.json ├── Crowdtangle-2_Get_accounts_from_lists.fp4.json ├── Crowdtangle-3_Get_posts_from_list.fp4.json ├── Crowdtangle-4_Get_detail_data_about_posts.fp4.json ├── Crowdtangle-Get_posts_by_links.fp4.json ├── Crowdtangle-Get_posts_by_searchterms.fp4.json ├── Facebook-1_Get_page_data_v20_0_.fp4.json ├── Facebook-2_Get_posts_v20_0_.fp4.json ├── Facebook-3_Get_comments_v20_0_.fp4.json ├── Facebook-Detail_data_about_posts_v20_0_.fp4.json ├── Facebook-Download_images.3_10.json ├── Facebook-Get_page_insights_v20_0_.fp4.json ├── Facebook-Get_reactions_v20_0_.fp4.json ├── Facebook-Pages_a_page_likes_v20_0_.fp4.json ├── Files-Download_Google_Cloud_Storage_buckets.3_10.json ├── GPT_3_from_Open_AI_-GTP_3_Completion.fp4.json ├── GPT_3_from_Open_AI_-GTP_3_Text_Classification_.fp4.json ├── Generic-Asynchronous_Speech_Recognition_using_Google_Cloud_Storage.3_10.json ├── Generic-Extract_links_from_website.3_10.json ├── Generic-Get_Bucket_Contents_from_Google Cloud Storage.3_10.json ├── Generic-Get_publications_from_arxiv_.3_10.json ├── Generic-Google_Maps_API_Geocoding.3_9.json ├── Generic-Result_of_asychronous_speech_recognition.3_10.json ├── Generic-Scrape_Wikipedia_tables.3_10.json ├── Generic-Synchrounous_Speech_Recognition.3_10.json ├── Generic-Upload_Google_Cloud_Storage_buckets.3_10.json ├── Generic-Wikipedia_Full_text_search.3_9.json ├── Generic-Wikipedia_Metadata.3_9.json ├── Knowledge_Graph-Culture_Knowledge_Graph.fp4.json ├── Knowledge_Graph-DBPedia_Scientists_who_have_birhtday_today.fp4.json ├── Knowledge_Graph-Getty_Fetch_triples_of_geographical_names.fp4.json ├── Knowledge_Graph-NFDI4Culture_1_Get_GND_IDs_of_Ferdinand_Gregorovius_s_addressees.fp4.json ├── Knowledge_Graph-NFDI4Culture_2_Translate_GND_ID_to_a_person_s_name.fp4.json ├── Knowledge_Graph-SemOpenAlex_Get_a_single_work.fp4.json ├── Knowledge_Graph-Wikidata_Get_gender_of_solo_artist_band_members.fp4.json ├── Knowledge_Graph-Wikidata_Get_list_of_biblical_characters.fp4.json ├── Knowledge_Graph-Wikidata_Which_politicans_did_not_die_on_natural_causes.fp4.json ├── Knowledge_Graph-Wikidata_Writers_Network.fp4.json ├── Mastodon-Get_public_timeline.fp4.json ├── Reddit-1_Get_posts_in_a_subreddit.fp4.json ├── Reddit-2_Get_comments.fp4.json ├── Reddit-3_Get_posts_in_a_subreddit_via_pushshift_.fp4.json ├── Reddit-4_Get_comments_via_pushshift_.fp4.json ├── Reddit-5_Search_submissions_via_pushshift_.fp4.json ├── Reddit-6_Search_comments_via_pushshift_.fp4.json ├── Scraping-BBC_Article_content.fp4.json ├── Scraping-BBC_Search_results.fp4.json ├── Scraping-Capture_screenshots.fp4.json ├── Scraping-Extract_meta_tags.fp4.json ├── Scraping-Get_target_of_short_links.fp4.json ├── Scraping-Scrape_teaser_texts_from_Zeit_Online.fp4.json ├── Scraping-Search_results_from_the_Financial_Times.fp4.json ├── Scraping-Show_meta_tags_in_columns.fp4.json ├── Twitter-1_Get_tweets_by_hashtag_v1_1_.fp4.json ├── Twitter-1_Get_tweets_by_hashtag_v2_.fp4.json ├── Twitter-2_Get_information_about_users_v1_1_.fp4.json ├── Twitter-2_Get_information_about_users_v2_.fp4.json ├── Twitter-3_Get_tweets_of_users_v1_1_.fp4.json ├── Twitter-3_Get_tweets_of_users_v2_.fp4.json ├── Twitter-4_Get_list_of_followers_v1_1_.fp4.json ├── Twitter-4_Get_list_of_followers_v2_.fp4.json ├── Twitter-5_Get_list_of_followees_v1_1_.fp4.json ├── Twitter-5_Get_list_of_followees_v2_.fp4.json ├── YouTube-Channel_information.3_9.json ├── YouTube-Get_comments.3_9.json ├── YouTube-Get_playlist_items.3_10.json ├── YouTube-Get_related_videos.3_10.json ├── YouTube-Get_replies.3_10.json ├── YouTube-Get_video_statistics.3_9.json └── YouTube-Search_YouTube_videos.3_9.json ├── requirements.txt ├── src ├── Facepager.py ├── __init__.py ├── actions.py ├── apimodules.py ├── apithread.py ├── aws │ ├── authorize.js │ └── blacklist.js ├── credentials.py.readme ├── database.py ├── dialogs │ ├── __init__.py │ ├── apiviewer.py │ ├── dataviewer.py │ ├── export.py │ ├── folder.py │ ├── help.py │ ├── presets.py │ ├── selectnodes.py │ ├── timer.py │ ├── transfernodes.py │ └── webdialog.py ├── icons.py ├── readme.md ├── server.py ├── settings.py ├── utilities.py └── widgets │ ├── __init__.py │ ├── datatree.py │ ├── dictionarytree.py │ ├── paramedit.py │ ├── progressbar.py │ └── textviewer.py └── tests ├── __init__.py ├── test_apiviewer.py └── test_utilities.py /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tools/ 2 | data/ 3 | private/ 4 | misc/ 5 | src/build 6 | src/dist 7 | src/hooks 8 | src/*.pyc 9 | src/*.pyo 10 | __pycache__ 11 | src/credentials.py 12 | build/windows/PySide2 13 | build/osx/copyfromvm.bat 14 | build/osx/*.app 15 | build/osx/*.pkg 16 | build/osx/Certificates 17 | build/osx/Codesign 18 | build/osx/notarytool_log.txt 19 | pyenv 20 | venv 21 | .project 22 | .pydevproject 23 | .settings/ 24 | .idea/ 25 | .idea/* 26 | .settings/org.eclipse.core.resources.prefs 27 | .DS_Store 28 | src/.eggs 29 | src/Facepager.spec 30 | Facepager*.exe 31 | *.app.zip 32 | *.app.macosx.zip 33 | *.bat 34 | .Rproj.user 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Logo](https://raw.github.com/strohne/Facepager/master/icons/icon_facepager.png) 2 | 3 | Facepager was made for fetching public available data from YouTube, Twitter and other websites on the basis of APIs and webscraping. All data is stored in a SQLite database and may be exported to csv. 4 | 5 | ### Installer 6 | 7 | Installation packages for each version are available on the [releases page](https://github.com/strohne/Facepager/releases). Database files may be incompatible between versions. 8 | 9 | - **Windows**: Download and execute the latest exe-installer from the [releases page](https://github.com/strohne/Facepager/releases). 10 | If Windows complains about an unknown publisher and refuses to launch the app click "More info" and start anyway. 11 | - **Mac OS X**: Download and install the latest package from the [releases page](https://github.com/strohne/Facepager/releases). 12 | Your computer will complain that it can't install the package, see https://support.apple.com/guide/mac-help/open-a-mac-app-from-an-unidentified-developer-mh40616/mac for further information. 13 | Head to the "System Settings", then click "Privacy & Security" in the sidebar. 14 | Next, scroll all the way down to the "Security" section, then click Open and Open Anyway. 15 | Older versions of Facepager were distributed in zip files and not code signed. Download and unzip the file from the releases page, drag & drop the app to your "Applications" folder. Next, you need to disable the download flag. Open the terminal, go to the folder where Facepager is stored (e.g. `cd /Applications`) and use `xattr -cr Facepager.app` to remove the download flag. Then open the app using ctrl-click. 16 | 17 | - **Linux**: There is no binary release, see [src/readme.md](https://github.com/strohne/Facepager/blob/master/src/readme.md) for steps to run under linux. 18 | 19 | If you want to run from source, see [src/readme.md](https://github.com/strohne/Facepager/blob/master/src/readme.md). 20 | 21 | ### Getting help 22 | 23 | Try the help button built into Facepager or directly go to the [Wiki](https://github.com/strohne/Facepager/wiki). There you find everything to [get you started](https://github.com/strohne/Facepager/wiki/Getting-Started). Further, you will find some [Tutorials on YouTube](https://www.youtube.com/channel/UCiIbKv5b5rz-6LPTLQgVGug). 24 | 25 | You can get help regarding specific problems in the [Facepager Usergroup on Facebook](https://www.facebook.com/groups/136224396995428/). If you want to be informed about updates please follow the [Facebook Page](https://www.facebook.com/facepagerpage). 26 | 27 | 28 | ### Citation 29 | 30 | [Jünger, Jakob](https://www.uni-muenster.de/Kowi/personen/jakob-juenger.html) / [Keyling, Till](http://tillkeyling.com/) (2019). Facepager. An application for automated data retrieval on the web. Source code and releases available at https://github.com/strohne/Facepager/. 31 | 32 | ### Licence 33 | 34 | 35 | MIT License 36 | 37 | Copyright (c) 2019 Jakob Jünger and Till Keyling 38 | 39 | Permission is hereby granted, free of charge, to any person obtaining a copy 40 | of this software and associated documentation files (the "Software"), to deal 41 | in the Software without restriction, including without limitation the rights 42 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 43 | copies of the Software, and to permit persons to whom the Software is 44 | furnished to do so, subject to the following conditions: 45 | 46 | The above copyright notice and this permission notice shall be included in all 47 | copies or substantial portions of the Software. 48 | 49 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 50 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 51 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 52 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 53 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 54 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 55 | SOFTWARE. 56 | 57 | -------------------------------------------------------------------------------- /build/osx/Facepager.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | block_cipher = None 4 | 5 | 6 | a = Analysis(['Facepager.py'], 7 | pathex=['../src'], 8 | binaries=[], 9 | datas=[], 10 | hiddenimports=['PySide2.QtPrintSupport'], 11 | hookspath=['hooks'], 12 | runtime_hooks=[], 13 | excludes=[], 14 | win_no_prefer_redirects=False, 15 | win_private_assemblies=False, 16 | cipher=block_cipher, 17 | noarchive=False) 18 | pyz = PYZ(a.pure, a.zipped_data, 19 | cipher=block_cipher) 20 | exe = EXE(pyz, 21 | a.scripts, 22 | [], 23 | exclude_binaries=True, 24 | name='Facepager', 25 | debug=False, 26 | bootloader_ignore_signals=False, 27 | strip=False, 28 | upx=False, 29 | console=False , icon='../icons/icon_facepager.icns') 30 | coll = COLLECT(exe, 31 | a.binaries, 32 | a.zipfiles, 33 | a.datas, 34 | strip=False, 35 | upx=False, 36 | name='Facepager') 37 | app = BUNDLE(coll, 38 | name='Facepager.app', 39 | icon='../icons/icon_facepager.icns', 40 | bundle_identifier='com.strohne.facepager') 41 | -------------------------------------------------------------------------------- /build/osx/build.pyinstaller.command: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | cd "`dirname "$0"`" 4 | cd ../.. 5 | 6 | cp build/osx/Facepager.spec src/Facepager.spec 7 | cp -r build/osx/hooks src/hooks 8 | 9 | cd src 10 | rm -rf build 11 | rm -rf dist 12 | 13 | source ../venv/bin/activate 14 | ../venv/bin/pyinstaller Facepager.spec 15 | #../venv/bin/pyinstaller --onefile --windowed --noconfirm --upx-dir=/usr/local/bin/ Facepager.spec 16 | cd .. 17 | 18 | python3 build/osx/fix_app_qt_folder_names_for_codesign.py src/dist/Facepager.app 19 | 20 | codesign --deep --force --options=runtime --entitlements build/osx/entitlements.plist --sign "C5675C9047BC5F500D88849509790AEEBCB99534" --timestamp src/dist/Facepager.app 21 | 22 | rm -rf build/osx/Facepager.pkg 23 | 24 | productbuild --identifier "com.strohne.facepager.pkg" --sign "AC630C1E0415944E2C2DCDE3210ADC5C8F20A02E" --timestamp --root src/dist/Facepager.app /Applications/Facepager.app build/osx/Facepager.pkg 25 | 26 | read -p "Press any key to continue..." -n1 -s -------------------------------------------------------------------------------- /build/osx/entitlements.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | com.apple.security.cs.allow-jit 7 | 8 | com.apple.security.cs.allow-unsigned-executable-memory 9 | 10 | com.apple.security.cs.disable-library-validation 11 | 12 | 13 | -------------------------------------------------------------------------------- /build/windows/Facepager.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | block_cipher = None 4 | 5 | 6 | a = Analysis(['Facepager.py'], 7 | pathex=['.'], 8 | binaries=[('../build/windows/cchardet', 'cchardet')], 9 | datas=[], 10 | hiddenimports=['PySide2.QtPrintSupport'], 11 | hookspath=[], 12 | runtime_hooks=[], 13 | excludes=[], 14 | win_no_prefer_redirects=False, 15 | win_private_assemblies=False, 16 | cipher=block_cipher, 17 | noarchive=False) 18 | pyz = PYZ(a.pure, a.zipped_data, 19 | cipher=block_cipher) 20 | exe = EXE(pyz, 21 | a.scripts, 22 | [], 23 | exclude_binaries=True, 24 | name='Facepager', 25 | debug=True, 26 | bootloader_ignore_signals=False, 27 | strip=False, 28 | upx=True, 29 | console=False , icon='..\\icons\\icon_facepager.ico') 30 | coll = COLLECT(exe, 31 | a.binaries, 32 | a.zipfiles, 33 | a.datas, 34 | strip=False, 35 | upx=True, 36 | name='Facepager') 37 | -------------------------------------------------------------------------------- /build/windows/Facepager_Setupscript.nsi: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/build/windows/Facepager_Setupscript.nsi -------------------------------------------------------------------------------- /build/windows/build.bat: -------------------------------------------------------------------------------- 1 | cd ../.. 2 | call "venv\Scripts\activate.bat" 3 | 4 | cd src 5 | rmdir /s / q build 6 | rmdir /s / q dist 7 | 8 | copy "..\build\windows\Facepager.spec" Facepager.spec 9 | 10 | pyinstaller --noconfirm Facepager.spec 11 | @rem pyinstaller --debug all --noconfirm --upx-exclude vcruntime140.dll Facepager.spec --upx-dir ..\build\windows\upx-3.96-win64\ 12 | 13 | cd dist 14 | copy ..\..\build\windows\Facepager_Setupscript.nsi Facepager_Setupscript.nsi 15 | copy ..\..\icons\icon_facepager.ico icon_facepager.ico 16 | "C:\Program Files (x86)\NSIS\makensis.exe" Facepager_Setupscript.nsi 17 | copy Facepager_Setup.exe ..\..\build\windows\Facepager_Setup.exe 18 | cd ../../build/windows 19 | endlocal 20 | pause -------------------------------------------------------------------------------- /build/windows/cchardet/__init__.py: -------------------------------------------------------------------------------- 1 | from cchardet import _cchardet 2 | from .version import __version__ 3 | 4 | 5 | def detect(msg): 6 | """ 7 | Args: 8 | msg: str 9 | Returns: 10 | { 11 | "encoding": str, 12 | "confidence": float 13 | } 14 | """ 15 | encoding, confidence = _cchardet.detect_with_confidence(msg) 16 | if isinstance(encoding, bytes): 17 | encoding = encoding.decode() 18 | return {"encoding": encoding, "confidence": confidence} 19 | 20 | 21 | class UniversalDetector(object): 22 | def __init__(self): 23 | self._detector = _cchardet.UniversalDetector() 24 | 25 | def __enter__(self): 26 | return self 27 | 28 | def __exit__(self, exception_type, exception_value, traceback): 29 | self.close() 30 | return False 31 | 32 | def reset(self): 33 | self._detector.reset() 34 | 35 | def feed(self, data): 36 | self._detector.feed(data) 37 | 38 | def close(self): 39 | self._detector.close() 40 | 41 | @property 42 | def done(self): 43 | return self._detector.done 44 | 45 | @property 46 | def result(self): 47 | encoding, confidence = self._detector.result 48 | if isinstance(encoding, bytes): 49 | encoding = encoding.decode() 50 | return {"encoding": encoding, "confidence": confidence} 51 | -------------------------------------------------------------------------------- /build/windows/cchardet/_cchardet.cp37-win_amd64.pyd: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/build/windows/cchardet/_cchardet.cp37-win_amd64.pyd -------------------------------------------------------------------------------- /build/windows/cchardet/version.py: -------------------------------------------------------------------------------- 1 | __version__ = '2.1.5' 2 | -------------------------------------------------------------------------------- /build/windows/dlls/LIBEAY32.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/build/windows/dlls/LIBEAY32.dll -------------------------------------------------------------------------------- /build/windows/dlls/SSLEAY32.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/build/windows/dlls/SSLEAY32.dll -------------------------------------------------------------------------------- /build/windows/dlls/mfc90.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/build/windows/dlls/mfc90.dll -------------------------------------------------------------------------------- /build/windows/dlls/pythoncom27.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/build/windows/dlls/pythoncom27.dll -------------------------------------------------------------------------------- /build/windows/dlls/pywintypes27.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/build/windows/dlls/pywintypes27.dll -------------------------------------------------------------------------------- /build/windows/upx-3.96-win64/BUGS: -------------------------------------------------------------------------------- 1 | ooooo ooo ooooooooo. ooooooo ooooo 2 | `888' `8' `888 `Y88. `8888 d8' 3 | 888 8 888 .d88' Y888..8P 4 | 888 8 888ooo88P' `8888' 5 | 888 8 888 .8PY888. 6 | `88. .8' 888 d8' `888b 7 | `YbodP' o888o o888o o88888o 8 | 9 | 10 | The Ultimate Packer for eXecutables 11 | Copyright (c) 1996-2020 Markus Oberhumer, Laszlo Molnar & John Reiser 12 | https://upx.github.io 13 | 14 | 15 | Limitations and other things which are not (yet) supported: 16 | =========================================================== 17 | 18 | djgpp2/coff 19 | ----------- 20 | * all overlays (except Allegro pakfiles) are silently stripped 21 | 22 | dos/exe 23 | ------- 24 | * normal dos/exes with new exe headers 25 | * max ~24000 relocation records (...should be enough for everyone ;-) 26 | * exe + sys combined images 27 | 28 | watcom/le 29 | --------- 30 | * 16-bit selector alias fixups 31 | * 16-bit offset relocation for objects larger than 4 KiB 32 | * 16:16 fixups 33 | 34 | If you need any of the above (they're very rare), send us an URL of a 35 | test file. 36 | 37 | * 16-bit objects are not loaded into DOS memory 38 | * There is still a problem with the wdosx extender: if you compress a 39 | watcom/le file which does NOT contain the wdosx extender, and after this 40 | you bind the wdosx stub to the compressed file, then it will work. 41 | Otherwise it won't. 42 | * unpacked pmwlite compressed programs might not work when compressed 43 | with UPX (this is a bug in pmwunlite) 44 | 45 | win32/pe 46 | -------- 47 | * writable shared sections (`--force' *may* work) 48 | * certificates in the image 49 | * compressing files which contain a big BSS requires lots of memory 50 | during compression 51 | 52 | -------------------------------------------------------------------------------- /build/windows/upx-3.96-win64/README.1ST: -------------------------------------------------------------------------------- 1 | ooooo ooo ooooooooo. ooooooo ooooo 2 | `888' `8' `888 `Y88. `8888 d8' 3 | 888 8 888 .d88' Y888..8P 4 | 888 8 888ooo88P' `8888' 5 | 888 8 888 .8PY888. 6 | `88. .8' 888 d8' `888b 7 | `YbodP' o888o o888o o88888o 8 | 9 | 10 | The Ultimate Packer for eXecutables 11 | Copyright (c) 1996-2020 Markus Oberhumer, Laszlo Molnar & John Reiser 12 | https://upx.github.io 13 | 14 | 15 | MAJOR NEWS IN UPX 3 16 | =================== 17 | 18 | The main news since UPX 2 are: 19 | 20 | * new format: added support for linux elf/arm 21 | 22 | * better compression using the LZMA algorithm (option '--lzma') 23 | 24 | * better support for MacOS 25 | -------------------------------------------------------------------------------- /build/windows/upx-3.96-win64/THANKS: -------------------------------------------------------------------------------- 1 | ooooo ooo ooooooooo. ooooooo ooooo 2 | `888' `8' `888 `Y88. `8888 d8' 3 | 888 8 888 .d88' Y888..8P 4 | 888 8 888ooo88P' `8888' 5 | 888 8 888 .8PY888. 6 | `88. .8' 888 d8' `888b 7 | `YbodP' o888o o888o o88888o 8 | 9 | 10 | The Ultimate Packer for eXecutables 11 | Copyright (c) 1996-2020 Markus Oberhumer, Laszlo Molnar & John Reiser 12 | https://upx.github.io 13 | 14 | 15 | .___.. . 16 | | |_ _.._ ;_/ __ 17 | | [ )(_][ )| \_) 18 | -------------------- 19 | 20 | UPX would not be what it is today without the invaluable help of 21 | everybody who was kind enough to spend time testing it, using it 22 | in applications and reporting bugs. 23 | 24 | The following people made especially gracious contributions of their 25 | time and energy in helping to track down bugs, add new features, and 26 | generally assist in the UPX maintainership process: 27 | 28 | Adam Ierymenko 29 | for severals ideas for the Linux version 30 | Andi Kleen and Jamie Lokier 31 | for the /proc/self/fd/X and other Linux suggestions 32 | Andreas Muegge 33 | for the Win32 GUI 34 | Atli Mar Gudmundsson 35 | for several comments on the win32/pe stub 36 | Charles W. Sandmann 37 | for the idea with the stubless decompressor in djgpp2/coff 38 | Ice 39 | for debugging the PE headersize problem down 40 | Joergen Ibsen and d'b 41 | for the relocation & address optimization ideas 42 | John S. Fine 43 | for the new version of the dos/exe decompressor 44 | Lukundoo 45 | for beta testing 46 | Michael Devore 47 | for initial dos/exe device driver support 48 | Oleg V. Volkov 49 | for various FreeBSD specific informations 50 | The Owl & G-RoM 51 | for the --compress-icons fix 52 | Ralph Roth 53 | for reporting several bugs 54 | Salvador Eduardo Tropea 55 | for beta testing 56 | Stefan Widmann 57 | for the win32/pe TLS callback support 58 | The WINE project (http://www.winehq.com/) 59 | for lots of useful information found in their PE loader sources 60 | Natascha 61 | 62 | -------------------------------------------------------------------------------- /build/windows/upx-3.96-win64/upx.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/build/windows/upx-3.96-win64/upx.exe -------------------------------------------------------------------------------- /docs/login_facebook.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Facebook Login 12 | 13 | 61 | 62 | 63 | 64 |

Before using the Facebook API, you need to read and acknowledge the following usage terms. 65 | You have two options:

66 |
    67 |
  1. 68 | Use the pre-registered Facepager app -> Proceed
    69 | To comply with the Facebook API terms, Facepager has to maintain a user list. 70 | Instead of requiring you to register at Facepager, we only store anonymized tokens. 71 | Thus, we extremely minimize the transferred data. 72 | See the privacy policy for further information. 73 | All users share the number of requests (rate limit). Please don't go to the limits for larger time frames and contact us if you plan a larger research project. Be fair! 74 | If you want to use this option, click "Proceed". 75 |
    2. 76 |
  2. Register an own app at Facebook -> Cancel
    77 | You have full control over the API but need to go through the app review yourself. 78 | If you want to use this option, click "Cancel", enter the Client ID of your registered app 79 | into the settings and login. 80 |
    3. 81 |
82 | 83 |

In any case, you need to comply with the Facebook terms.

84 | 89 |

Read the terms!
Be responsible!

90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /docs/login_twitter.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Facebook Login 12 | 13 | 61 | 62 | 63 | 64 |

Before using the Twitter API, you need to read and acknowledge the following usage terms. 65 | You have two options:

66 |
    67 |
  1. 68 | Use the pre-registered Facepager app -> Proceed
    69 | To comply with the API terms, Facepager has to maintain a user list. 70 | Instead of requiring you to register at Facepager, we only store anonymized tokens. 71 | Thus, we extremely minimize the transferred data. 72 | See the privacy policy for further information. 73 | If you want to use this option, click "Proceed". 74 |
    2. 75 |
  2. Register an own app at Twitter -> Cancel
    76 | You have full control over the API but need to go through the app review yourself. 77 | If you want to use this option, click "Cancel", enter the Client ID of your registered app 78 | into the settings and login. 79 |
    3. 80 |
81 | 82 |

In any case, you need to comply with the Twitter terms.

83 | 87 |

Read the terms!
Be responsible!

88 | 89 | 90 | 91 | 92 | 93 | -------------------------------------------------------------------------------- /docs/login_youtube.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Facebook Login 12 | 13 | 61 | 62 | 63 | 64 |

Before using the YouTube API, you need to read and acknowledge the following usage terms. 65 | You have two options:

66 |
    67 |
  1. 68 | Use the pre-registered Facepager app -> Proceed
    69 | To comply with the API terms, Facepager has to maintain a user list. 70 | Instead of requiring you to register at Facepager, we only store anonymized tokens. 71 | Thus, we extremely minimize the transferred data. 72 | See the privacy policy for further information. 73 | If you want to use this option, click "Proceed". 74 |
    2. 75 |
  2. Register an own app at YouTube -> Cancel
    76 | You have full control over the API but need to go through the app review yourself. 77 | If you want to use this option, click "Cancel", enter the Client ID of your registered app 78 | into the settings and login. 79 | Alternatively, probably more easy, use the token authorization method and provide you access token. 80 |
    3. 81 |
82 | 83 |

In any case, you need to comply with the YouTube terms.

84 | 90 |

Read the terms!
Be responsible!

91 | 92 | 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /docs/oauth_feedback.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Facepager Login 12 | 13 | 61 | 62 | 63 | 64 |

65 | Login completed. You can now close this window and go back to Facepager. 66 |

67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /docs/oauth_waiting.html: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | Facepager Login 12 | 13 | 61 | 62 | 63 | 64 |

65 | Facepager opened an external browser and is waiting for your login. 66 | Once you approve Facepager, this windows should automatically close. 67 |

68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /icons/add.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/add.png -------------------------------------------------------------------------------- /icons/addcolumn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/addcolumn.png -------------------------------------------------------------------------------- /icons/apis.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/apis.png -------------------------------------------------------------------------------- /icons/apply.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/apply.png -------------------------------------------------------------------------------- /icons/browser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/browser.png -------------------------------------------------------------------------------- /icons/clear.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/clear.png -------------------------------------------------------------------------------- /icons/collapse.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/collapse.png -------------------------------------------------------------------------------- /icons/delete.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/delete.png -------------------------------------------------------------------------------- /icons/expand.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/expand.png -------------------------------------------------------------------------------- /icons/export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/export.png -------------------------------------------------------------------------------- /icons/fetch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/fetch.png -------------------------------------------------------------------------------- /icons/help.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/help.png -------------------------------------------------------------------------------- /icons/icon_facepager.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/icon_facepager.icns -------------------------------------------------------------------------------- /icons/icon_facepager.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/icon_facepager.ico -------------------------------------------------------------------------------- /icons/icon_facepager.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/icon_facepager.png -------------------------------------------------------------------------------- /icons/icons.qrc: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | add.png 6 | delete.png 7 | export.png 8 | new.png 9 | save.png 10 | fetch.png 11 | icon_facepager.png 12 | presets.png 13 | collapse.png 14 | expand.png 15 | timer.png 16 | help.png 17 | toclip.png 18 | unpack.png 19 | apis.png 20 | addcolumn.png 21 | search.png 22 | browser.png 23 | more.png 24 | apply.png 25 | clear.png 26 | transfer.png 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /icons/more.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/more.png -------------------------------------------------------------------------------- /icons/new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/new.png -------------------------------------------------------------------------------- /icons/presets.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/presets.png -------------------------------------------------------------------------------- /icons/readme.md: -------------------------------------------------------------------------------- 1 | Add icons to Facepager: 2 | - Copy file into the icons directory (the directory of this file) 3 | - Add filename to icons.qrc 4 | - Open command line in the directory of pyside2-rcc (e.g. pyenv\Lib\site-packages\PySide2) 5 | - On command line execute (adjust path): 6 | `pyside2-rcc -py3 ../../../../icons/icons.qrc > ../../../../icons/icons.py` 7 | - Copy the new icons.py to src folder 8 | - Change linebreaks from CRLF to LF (workaround: will not load in builds otherwise) 9 | - Use icons by the following scheme ':/icons/filename.filetype' 10 | 11 | -------------------------------------------------------------------------------- /icons/save.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/save.png -------------------------------------------------------------------------------- /icons/search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/search.png -------------------------------------------------------------------------------- /icons/timer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/timer.png -------------------------------------------------------------------------------- /icons/toclip.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/toclip.png -------------------------------------------------------------------------------- /icons/transfer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/transfer.png -------------------------------------------------------------------------------- /icons/unpack.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/icons/unpack.png -------------------------------------------------------------------------------- /presets/Amazon-Get_bucket_content.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": "Amazon", 3 | "description": "Add bucket name as node or fetch buckets with another preset. Up to 1000 objects are returned. Use the prefix-parameter to get a subset with matching keys. You can see your buckets in the web browser as well: https://s3.console.aws.amazon.com\n\nBe aware of region settings. You get an error if the wrong region is set.\n\nRegister at Amazon AWS to obtain access key and secret. Reference: https://docs.aws.amazon.com/de_de/AmazonS3/latest/API/Welcome.html\n\n\n\n", 4 | "module": "Amazon", 5 | "name": "Get bucket content", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_uri": "", 11 | "basepath": "http://.s3.amazonaws.com", 12 | "format": "xml", 13 | "headers": {}, 14 | "key_paging": null, 15 | "nodedata": "listbucketresult.contents", 16 | "objectid": "key.$", 17 | "pages": 1, 18 | "paging_stop": null, 19 | "paging_type": "key", 20 | "param_paging": null, 21 | "params": { 22 | "list-type": "2" 23 | }, 24 | "redirect_uri": "", 25 | "region": "eu-central-1", 26 | "resource": "", 27 | "service": "s3", 28 | "token_uri": "", 29 | "verb": "GET" 30 | }, 31 | "speed": 200, 32 | "saveheaders": false, 33 | "columns": [ 34 | "lastmodified.$", 35 | "etag.$", 36 | "size.$", 37 | "storageclass.$" 38 | ] 39 | } -------------------------------------------------------------------------------- /presets/Amazon-Get_buckets.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": "Amazon", 3 | "description": "Add arbitrary node and fetch data.\n\nYou need to register at Amazon AWS to obtain access key and secret.\n\nReference: https://docs.aws.amazon.com/de_de/AmazonS3/latest/API/Welcome.html\n\n", 4 | "module": "Amazon", 5 | "name": "Get buckets", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_uri": "", 11 | "basepath": "http://s3.amazonaws.com", 12 | "format": "xml", 13 | "headers": {}, 14 | "key_paging": null, 15 | "nodedata": "listallmybucketsresult.buckets.bucket", 16 | "objectid": "name.$", 17 | "pages": 1, 18 | "paging_stop": null, 19 | "paging_type": "key", 20 | "param_paging": null, 21 | "params": {}, 22 | "redirect_uri": "", 23 | "region": "us-east-1", 24 | "resource": "", 25 | "service": "s3", 26 | "token_uri": "", 27 | "verb": "GET" 28 | }, 29 | "speed": 200, 30 | "saveheaders": false, 31 | "columns": [ 32 | "name.$", 33 | "creationdate.$" 34 | ] 35 | } -------------------------------------------------------------------------------- /presets/Bibliometrics-OpenAlex_1_Get_a_single_work.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "OpenAlex 1: Get a single work", 3 | "category": "Bibliometrics", 4 | "description": "This preset retrives a number of information about a single scholarly work from OpenAlex. To start add a DOI URL (e.g., https://doi.org/10.1007/978-3-658-37747-2) as a seed node. DOIs are not the only option, see [OpenAlex' documentation](https://docs.openalex.org/api-entities/works/work-object#ids) for all possible IDs. Once seed nodes are in place fetch data. \n\nSpecify the *select* parameter to narrow down your query. The preset at hand asks for id, doi, title, publication_year, authorships, and referenced_works of the stated work. \n\nIf you want to fetch other information about a single work such as, for example, the work's citation count, have a look at [OpenAlex's documentation on work-objects](https://docs.openalex.org/api-entities/works/work-object) and adjust the *select* parameter in the Query Setup accordingly. \n\nSee our [Getting Started with OpenAlex](https://github.com/strohne/Facepager/wiki/Getting-Started-with-OpenAlex) for a full walk through this preset. \n\nPlease note that each user is allowed a maximum of 100,000 API calls per day and 10 API calls per second. If you need a higher limit, you can register for [OpenAlex Premium](https://openalex.org/pricing).", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://api.openalex.org", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": {}, 18 | "key_paging": null, 19 | "nodedata": null, 20 | "objectid": null, 21 | "pages": 1, 22 | "paging_stop": null, 23 | "paging_type": "key", 24 | "param_paging": null, 25 | "params": { 26 | "": "", 27 | "select": "id, doi, title, publication_year, referenced_works, authorships" 28 | }, 29 | "proxy": "", 30 | "redirect_uri": "", 31 | "resource": "/works/", 32 | "scope": "", 33 | "token_uri": "", 34 | "verb": "GET" 35 | }, 36 | "speed": 200, 37 | "saveheaders": false, 38 | "timeout": 15, 39 | "maxsize": 5, 40 | "columns": [ 41 | "id", 42 | "doi", 43 | "title", 44 | "publication_year", 45 | "referenced_works", 46 | "authorships" 47 | ] 48 | } -------------------------------------------------------------------------------- /presets/Bibliometrics-OpenAlex_2_Get_author.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "OpenAlex 2: Get author", 3 | "category": "Bibliometrics", 4 | "description": "This preset fetches detailed information about an author from OpenAlex. To start add an OpenAlex author ID (e.g. A5088931518) or external IDs such as ORCID (e.g. https://orcid.org/0000-0003-1860-6695) as a seed note. Then fetch data. \n\nFor detailed instructions, see the step-by-step guide below.\n\n**1. Create a database:** Click *New Database* in the [Menu Bar](https://github.com/strohne/Facepager/wiki/Layout#menu-bar) of Facepager to create a blank database. Save it in a directory of your choice.\n**2. Setup the Generic module:** Apply the preset at hand. The [Generic module](https://github.com/strohne/Facepager/wiki/Query-Setup.-Generic) in the [Query Setup](https://github.com/strohne/Facepager/wiki/Layout#query-setup) will refresh automatically. Notice that the base path is now set to call OpenAlex' API and a node placeholder has been installed. Additionally, the *select* parameter determines what information Facepager will fetch from OpenAlex. Specify the input to narrow down your query. The preset at hand asks for id, orcid, display_name, works_count, cited_by_count, and last_known_institution of the stated author. Check out [OpenAlex' documentation on author-objects](https://docs.openalex.org/api-entities/authors/author-object) to get an overview over the broad range of options.\n**3. Add nodes:** Before fetching data, you will need to provide one or more seed nodes which will fill in said placeholder later on. For this, select *Add Nodes* in the Menu Bar. In the open dialogue box either add an OpenAlex ID (e.g. A5088931518) or external IDs such as ORCID (e.g. https://orcid.org/0000-0003-1860-6695 or in URN format orcid:0000-0003-1860-6695), ScopusID, Twitter, or Wikipedia. Check [OpenAlex' documentation](https://docs.openalex.org/api-entities/authors/author-object#ids) for an up-to-date list of all possibilities. Include as many nodes as you like here.\n**4. Fetch data:** Select one or more seed nodes and hit *Fetch Data* at the bottom of the Query Setup afterwards. Facepager will now fetch data on the basis of your setup. Once finished, you can inspect the data by expanding your seed node or clicking *Expand nodes* in the Menu Bar. For more detail, select a child node and review the raw data displayed in the [Data View](https://github.com/strohne/Facepager/wiki/Layout#data-view) to the right. If you want to fetch other information about an author such as, for example, the author's citation metrics or institution, have a look at [OpenAlex' documentation on author-objects](https://docs.openalex.org/api-entities/authors/author-object) and adjust the *select* parameter in the Query Setup accordingly.\n**5. Export data:** Expand all nodes and select the ones you want to export. Hit [*Export Data*](https://github.com/strohne/Facepager/wiki/Export-Data) to get a CSV-file. Notice the options provided by the export dialogue. You can open CSV files with Excel or any statistics software you like. \n\nPlease note that each user is allowed a maximum of 100,000 API calls per day and 10 API calls per second. If you need a higher limit, you can register for OpenAlex Premium. ", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://api.openalex.org", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": {}, 18 | "key_paging": null, 19 | "nodedata": null, 20 | "objectid": null, 21 | "pages": 1, 22 | "paging_stop": null, 23 | "paging_type": "key", 24 | "param_paging": null, 25 | "params": { 26 | "": "", 27 | "select": "id, orcid,\ndisplay_name, works_count,\ncited_by_count, last_known_institution" 28 | }, 29 | "proxy": "", 30 | "redirect_uri": "", 31 | "resource": "/authors/", 32 | "scope": "", 33 | "token_uri": "", 34 | "verb": "GET" 35 | }, 36 | "speed": 200, 37 | "saveheaders": false, 38 | "timeout": 15, 39 | "maxsize": 5, 40 | "columns": [ 41 | "id", 42 | "orcid", 43 | "display_name", 44 | "works_count", 45 | "cited_by_count", 46 | "last_known_institution.id" 47 | ] 48 | } -------------------------------------------------------------------------------- /presets/Bibliometrics-OpenCitations_Index_COCI_Metadata_from_doi.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "OpenCitations Index (COCI): Metadata from doi", 3 | "category": "Bibliometrics", 4 | "description": "[The OpenCitations Index](https://opencitations.net/index) – formerly known as the OpenCitations Index of Crossref open DOI-to-DOI citations (COCI) – is a bibliographic index recording and storing citations between publications. It pulls its data from various open scholarly sources as described on their website (see link above), thereby providing comprehensive and openly accessible bibliographic and citation data. \n\nEach citation is identified by an unique [Open Citation Identifier (OCI)](https://identifiers.org/oci). However, data can be fetched via various identifiers including DOIs. \n\nApply this preset to fetch metadate releated to a scholarly work. Add as many DOIs as seed nodes as you wish (e.g. , \"10.1111/j.1460-2466.2008.01401.x\").\n\nFor this specific preset we are still calling version 1.4.0 of the [REST API for COCI](https://opencitations.net/index/coci/api/v1#/metadata/%7Bdois%7D). As the operation \"/metadata/{dois}\" has not been carried to version 2.0.x of [the REST API for the OpenCitations Index](https://opencitations.net/index/api/v2). In this context OpenCitations states: \"we use certain legacy APIs for historical and compatibility reasons. While we encourage the use of our latest and more efficient V2 APIs, we understand that some applications may still rely on these older endpoints. Please note that legacy APIs may not receive updates or support, and their use may be phased out in the future. We recommend transitioning to our current V2 APIs for improved performance, security, and reliability.\"", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://opencitations.net/index/coci/api/v1", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": {}, 18 | "nodedata": null, 19 | "objectid": "doi", 20 | "offset_start": 1, 21 | "offset_step": 1, 22 | "pages": 1, 23 | "paging_type": "count", 24 | "param_paging": null, 25 | "params": { 26 | "": "" 27 | }, 28 | "proxy": "", 29 | "redirect_uri": "", 30 | "resource": "/metadata/", 31 | "scope": "", 32 | "token_uri": "", 33 | "verb": "GET" 34 | }, 35 | "speed": 200, 36 | "saveheaders": false, 37 | "timeout": 15, 38 | "maxsize": 5, 39 | "columns": [ 40 | "year", 41 | "author", 42 | "reference", 43 | "reference_count=reference|re:^.\\|;|length", 44 | "citation", 45 | "citation_count", 46 | "title", 47 | "source_title", 48 | "source_id", 49 | "volume", 50 | "issue", 51 | "page", 52 | "oa_link" 53 | ] 54 | } -------------------------------------------------------------------------------- /presets/Bibliometrics-OpenCitations_Index_COCI_citations.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "OpenCitations Index (COCI): citations", 3 | "category": "Bibliometrics", 4 | "description": "[The OpenCitations Index](https://opencitations.net/index) – formerly known as the OpenCitations Index of Crossref open DOI-to-DOI citations (COCI) – is a bibliographic index recording and storing citations between publications. It pulls its data from various open scholarly sources as described on their website (see link above), thereby providing comprehensive and openly accessible bibliographic and citation data. \n\nEach citation is identified by an unique [Open Citation Identifier (OCI)](https://identifiers.org/oci). However, data can be fetched via various identifiers including DOIs. Further, each item is a citation record. To get its bibliographic metadata use the \"OpenCitations Index (COCI): Metadata from doi\" preset.\n\nApply this preset to fetch the titles of all works that cite the seed work. Add as many DOIs as seed nodes as you wish (e.g. , \"10.1111/j.1460-2466.2008.01401.x\").\n\nHere we are calling version 2.0.x of [the REST API for the OpenCitations Index](https://opencitations.net/index/api/v2#/citations/%7Bid%7D).", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://opencitations.net/index/api/v2", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": {}, 18 | "nodedata": null, 19 | "objectid": "citing", 20 | "offset_start": 1, 21 | "offset_step": 1, 22 | "pages": 1, 23 | "paging_type": "count", 24 | "param_paging": null, 25 | "params": { 26 | "": "" 27 | }, 28 | "proxy": "", 29 | "redirect_uri": "", 30 | "resource": "/citations/doi:", 31 | "scope": "", 32 | "token_uri": "", 33 | "verb": "GET" 34 | }, 35 | "speed": 200, 36 | "saveheaders": false, 37 | "timeout": 15, 38 | "maxsize": 5, 39 | "columns": [ 40 | "oci", 41 | "citing", 42 | "cited", 43 | "creation", 44 | "timespan", 45 | "journal_sc", 46 | "author_sc", 47 | "", 48 | "" 49 | ] 50 | } -------------------------------------------------------------------------------- /presets/Bibliometrics-OpenCitations_Index_COCI_references.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "OpenCitations Index (COCI): references", 3 | "category": "Bibliometrics", 4 | "description": "[The OpenCitations Index](https://opencitations.net/index) – formerly known as the OpenCitations Index of Crossref open DOI-to-DOI citations (COCI) – is a bibliographic index recording and storing citations between publications. It pulls its data from various open scholarly sources as described on their website (see link above), thereby providing comprehensive and openly accessible bibliographic and citation data. \n\nEach citation is identified by an unique [Open Citation Identifier (OCI)](https://identifiers.org/oci). However, data can be fetched via various identifiers including DOIs. Further, each item is a citation record. To get its bibliographic metadata use the \"OpenCitations Index (COCI): Metadata from doi\" preset.\n\nApply this preset to fetch the references contained in the seed title. . Add as many DOIs as seed nodes as you wish (e.g. , \"10.1111/j.1460-2466.2008.01401.x\").\n\nHere we are calling version 2.0.x of [the REST API for the OpenCitations Index](https://opencitations.net/index/api/v2#/references/%7Bid%7D).", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "https://opencitations.net/index/api/v2", 8 | "resource": "/references/doi:", 9 | "params": { 10 | "": "" 11 | }, 12 | "extension": "", 13 | "headers": {}, 14 | "verb": "GET", 15 | "format": "json", 16 | "filename": "", 17 | "fileext": "", 18 | "pages": 1, 19 | "paging_type": "count", 20 | "param_paging": null, 21 | "offset_start": 1, 22 | "offset_step": 1, 23 | "nodedata": null, 24 | "objectid": "cited", 25 | "scope": "", 26 | "proxy": "", 27 | "auth_type": "Disable", 28 | "auth_uri": "", 29 | "redirect_uri": "", 30 | "token_uri": "", 31 | "auth": "disable", 32 | "auth_tokenname": "", 33 | "auth_prefix": "" 34 | }, 35 | "speed": 200, 36 | "saveheaders": false, 37 | "timeout": 15, 38 | "maxsize": 5, 39 | "columns": [ 40 | "creation", 41 | "timespan", 42 | "author_sc", 43 | "journal_sc", 44 | "oci", 45 | "citing", 46 | "cited" 47 | ] 48 | } -------------------------------------------------------------------------------- /presets/Bibliometrics-Open_Library_Author_metadata_Authors_API_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Open Library: Author metadata (Authors API)", 3 | "category": "Bibliometrics", 4 | "description": "[Open Library](https://openlibrary.org/) is an open project launched by the Internet Archive in 2006 that aims to provide \"one web page for every book ever published\". Each page contains detailed bibliographic information about the entry, including metadata such as publisher, publication date, and ISBN. The result is an exhaustive catalog of books searchable by title, author, or subject and, whenever possible, Open Library offers digital versions of its books for borrowing and reading. Due to its collaborative, wiki-style nature all users are welcome to contribute information to the catalog by editing book information or even adding new entries.\n\nApply this preset to fetch all available metadata about an author using the [Open Library Author API](https://openlibrary.org/dev/docs/api/authors). Start by adding an author's identifier key as seed note (e.g., \"OL234664A\"). You can also fetch information by adding the name of an author as a seed node (e.g., \"George R.R. Martin\"). Simply, use the Open Library Search API by applying our \"Open Library: Book metadata from title (Search API)\" preset and replace \"title:\" in the q parameter with \"author:\". Just make sure to adjust the Column Setup accordingly.\n\nPlease, see this [overview of Open Library's range of APIs](https://openlibrary.org/developers/api) to find the API that best suits your needs and adjust the preset accordingly. Generally, OpenLibrary recommends their powerful [Search API](https://openlibrary.org/dev/docs/api/search) and they have published some [tips for working with it](https://openlibrary.org/search/howto).\n\n**Note:** Please mind Open Library's bulk download policy. Open Library kindly asks to not use their APIs for bulk downloads. If you need a dump of complete data, please read about their [bulk download](https://openlibrary.org/data#downloads) options, or email them at openlibrary@archive.org. If you plan to make regular, frequent use of Open Library's APIs (e.g. multiple calls per minute), please edit the existing **header** to specify the **User-Agent string** with (a) the name of your application and (b) your contact email or phone number, so Open Library may contact you when we notice high request volume. Failing to include these headers may result in the application (Facepager) being blocked.", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://openlibrary.org", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": { 18 | "User-Agent": "Facepager/4.5.3 (youremail@example.com)" 19 | }, 20 | "nodedata": null, 21 | "objectid": "personal_name", 22 | "offset_start": 1, 23 | "offset_step": 1, 24 | "pages": 1, 25 | "paging_type": "count", 26 | "param_paging": null, 27 | "params": { 28 | "": "" 29 | }, 30 | "proxy": "", 31 | "redirect_uri": "", 32 | "resource": "/authors/.json", 33 | "scope": "", 34 | "token_uri": "", 35 | "verb": "GET" 36 | }, 37 | "speed": 200, 38 | "saveheaders": false, 39 | "timeout": 15, 40 | "maxsize": 5, 41 | "columns": [ 42 | "birth_date", 43 | "bio" 44 | ] 45 | } -------------------------------------------------------------------------------- /presets/Bibliometrics-Open_Library_Book_metadata_from_title_Search_API_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Open Library: Book metadata from title (Search API)", 3 | "category": "Bibliometrics", 4 | "description": "[Open Library](https://openlibrary.org/) is an open project launched by the Internet Archive in 2006 that aims to provide \"one web page for every book ever published\". Each page contains detailed bibliographic information about the entry, including metadata such as publisher, publication date, and ISBN. The result is an exhaustive catalog of books searchable by title, author, or subject and, whenever possible, Open Library offers digital versions of its books for borrowing and reading. Due to its collaborative, wiki-style nature all users are welcome to contribute information to the catalog by editing book information or even adding new entries.\n\nApply this preset to fetch all available metadata about a book or a book series using the [Open Library Search API](https://openlibrary.org/dev/docs/api/search). Start by adding a book title as seed note (e.g., \"A Song of Ice and Fire\"). This will return all works that contain the seed need in their title. Edit the fields parameter to specifiy the metadata returned by your query. \n\nPlease, see this [overview of Open Library's range of APIs](https://openlibrary.org/developers/api) to find the API that best suits your needs and adjust the preset accordingly. Generally, OpenLibrary recommends their powerful [Search API](https://openlibrary.org/dev/docs/api/search) and they have published some [tips for working with it](https://openlibrary.org/search/howto).\n\n**Note:** Please mind Open Library's bulk download policy. Open Library kindly asks to not use their APIs for bulk downloads. If you need a dump of complete data, please read about their [bulk download](https://openlibrary.org/data#downloads) options, or email them at openlibrary@archive.org. If you plan to make regular, frequent use of Open Library's APIs (e.g. multiple calls per minute), please edit the existing **header** to specify the **User-Agent string** with (a) the name of your application and (b) your contact email or phone number, so Open Library may contact you when we notice high request volume. Failing to include these headers may result in the application (Facepager) being blocked.", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://openlibrary.org", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": { 18 | "User-Agent": "Facepager/4.5.3 (youremail@example.com)" 19 | }, 20 | "nodedata": "docs", 21 | "objectid": "author_name.0", 22 | "offset_start": 1, 23 | "offset_step": 1, 24 | "pages": 1, 25 | "paging_type": "count", 26 | "param_paging": null, 27 | "params": { 28 | "fields": "*", 29 | "limit": "10", 30 | "q": "title:", 31 | "sort": "old" 32 | }, 33 | "proxy": "", 34 | "redirect_uri": "", 35 | "resource": "/search.json?", 36 | "scope": "", 37 | "token_uri": "", 38 | "verb": "GET" 39 | }, 40 | "speed": 200, 41 | "saveheaders": false, 42 | "timeout": 15, 43 | "maxsize": 5, 44 | "columns": [ 45 | "title", 46 | "key", 47 | "first_publish_year", 48 | "author_name.0", 49 | "author_key.0", 50 | "edition_count", 51 | "ebook_access", 52 | "isbn.0", 53 | "ratings_average" 54 | ] 55 | } -------------------------------------------------------------------------------- /presets/Crowdtangle-1_Get_lists.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "1 Get lists", 3 | "category": "Crowdtangle", 4 | "description": "CrowdTangle is a Facebook public insights tool that makes it simple to track, analyze, and report on what's going on with public social media material. \n\nThe access for CrowdTangle is available for academics and researchers in specific fields. Here you can find more information about the registration and apply for the access token: https://help.crowdtangle.com/en/articles/4302208-crowdtangle-for-academics-and-researchers. \n\nOnce you have the CrowdTangle Access you can start the data collection of public accounts and groups across Facebook and Instagram. From your CrowdTangle Dashboard you can create custom lists of public accounts and groups you want to monitor. \n\nWith this first preset you get the lists and saved searches/posts of your dashboard. Add your Access Token as seed node. \n\nDON'T FORGET: Before you can fetch the data, you must enter your CrowdTangle access token in the corresponding field \"access token\" for authentication.\n\nSee https://github.com/CrowdTangle/API/wiki/lists for more information\n\nAPI-Cheatsheet: https://help.crowdtangle.com/en/articles/3443476-api-cheat-sheet", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "param", 8 | "auth_tokenname": "token", 9 | "auth_type": "API key", 10 | "auth_uri": "", 11 | "basepath": "https://api.crowdtangle.com", 12 | "extension": "", 13 | "fileext": "", 14 | "filename": "", 15 | "format": "json", 16 | "headers": {}, 17 | "key_paging": "result.pagination.nextPage", 18 | "nodedata": "result.lists", 19 | "objectid": "id", 20 | "pages": 5, 21 | "paging_type": "url", 22 | "params": { 23 | "token": "" 24 | }, 25 | "proxy": "", 26 | "redirect_uri": "", 27 | "resource": "/lists", 28 | "scope": "", 29 | "token_uri": "", 30 | "verb": "GET" 31 | }, 32 | "speed": 200, 33 | "saveheaders": false, 34 | "timeout": 15, 35 | "maxsize": 5, 36 | "columns": [ 37 | "id", 38 | "title", 39 | "type", 40 | "result" 41 | ] 42 | } -------------------------------------------------------------------------------- /presets/Crowdtangle-2_Get_accounts_from_lists.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "2 Get accounts from lists", 3 | "category": "Crowdtangle", 4 | "description": "Add List ID as seed node. \n\nYou get the list ID from a list or a saved search/post at your CrowdTangle Dashboard (Preset 1) or you do a search with the search endpoint.\n\nWith this preset you can get the accounts for a given list. The list has to be of type list, saved searches and saved posts do not have associated accounts.\n\nThis preset fetches 10 accounts of a list. You can increase the count-parameter up to 100 to get more accounts.\n\nDON'T FORGET: Before you can fetch the data, you must enter your CrowdTangle access token in the corresponding field \"access token\" for authentication.\n\nSee https://github.com/CrowdTangle/API/wiki/List-Accounts for more information. \n\nAPI-Cheatsheet: https://help.crowdtangle.com/en/articles/3443476-api-cheat-sheet", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "param", 8 | "auth_tokenname": "token", 9 | "auth_type": "API key", 10 | "auth_uri": "", 11 | "basepath": "https://api.crowdtangle.com", 12 | "extension": "", 13 | "fileext": "", 14 | "filename": "", 15 | "format": "json", 16 | "headers": {}, 17 | "key_paging": "result.pagination.nextPage", 18 | "nodedata": "result.accounts", 19 | "objectid": "id", 20 | "pages": 5, 21 | "paging_type": "url", 22 | "params": { 23 | "": "", 24 | "count": "10" 25 | }, 26 | "proxy": "", 27 | "redirect_uri": "", 28 | "resource": "/lists//accounts", 29 | "scope": "", 30 | "token_uri": "", 31 | "verb": "GET" 32 | }, 33 | "speed": 200, 34 | "saveheaders": true, 35 | "timeout": 15, 36 | "maxsize": 5, 37 | "columns": [ 38 | "id", 39 | "name", 40 | "handle", 41 | "profileImage", 42 | "subscriberCount", 43 | "url", 44 | "platform", 45 | "platformId", 46 | "accountType", 47 | "pageAdminTopCountry", 48 | "pageDescription", 49 | "pageCreatedDate", 50 | "pageCategory", 51 | "verified" 52 | ] 53 | } -------------------------------------------------------------------------------- /presets/Crowdtangle-3_Get_posts_from_list.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "3 Get posts from list", 3 | "category": "Crowdtangle", 4 | "description": "Add list ID as seed node. \n\nYou get the list ID 1) with Preset 1 to get the ID from a list or a saved search/post from your CrowdTangle Dashboard 2) a search with the search endpoint or 3) by looking at the last part of the URL (the digits coming after the last slash /), when you select the list in your CrowdTangle Dashboard.\n\nThis preset fetches the last 10 posts on a page. To see more than 10 posts, set the count parameter up to 100. To see more than 100 posts increase the maximum pages. The API will max out at 10,000 posts. \n\nIf you want to search special accounts from your list, you can add the account handles or platform ids to the account parameter. These can be separated by commas to include multiple accounts. \n\nThe default sortBy Parameter for this preset is date. If you want to have other posts than the first 10 recently published posts, adjust the sortBy Parameter.\n\nDON'T FORGET: Before you can fetch the data, you must enter your CrowdTangle access token in the corresponding field \"access token\" for authentication.\n\nSee https://github.com/CrowdTangle/API/wiki/Posts for more parameter options.\n\nAPI-Cheatsheet: https://help.crowdtangle.com/en/articles/3443476-api-cheat-sheet", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "param", 8 | "auth_tokenname": "token", 9 | "auth_type": "API key", 10 | "auth_uri": "", 11 | "basepath": "https://api.crowdtangle.com", 12 | "extension": "", 13 | "fileext": "", 14 | "filename": "", 15 | "format": "json", 16 | "headers": {}, 17 | "key_paging": "result.pagination.nextPage", 18 | "nodedata": "result.posts", 19 | "objectid": "platformId", 20 | "pages": 1, 21 | "paging_type": "url", 22 | "params": { 23 | "count": "10", 24 | "listIds": "", 25 | "sortBy": "date" 26 | }, 27 | "proxy": "", 28 | "redirect_uri": "", 29 | "resource": "/posts", 30 | "scope": "", 31 | "token_uri": "", 32 | "verb": "GET" 33 | }, 34 | "speed": 200, 35 | "saveheaders": false, 36 | "timeout": 15, 37 | "maxsize": 5, 38 | "columns": [ 39 | "platformId", 40 | "platform", 41 | "date", 42 | "account.id", 43 | "account.name", 44 | "account.handle", 45 | "updated", 46 | "type", 47 | "message", 48 | "expandedLinks", 49 | "link", 50 | "postUrl", 51 | "subscriberCount", 52 | "score", 53 | "media", 54 | "statistics.actual.likeCount", 55 | "statistics.actual.shareCount", 56 | "statistics.actual.commentCount", 57 | "statistics.actual.loveCount", 58 | "statistics.actual.wowCount", 59 | "statistics.actual.hahaCount", 60 | "statistics.actual.sadCount", 61 | "statistics.actual.angryCount", 62 | "statistics.actual.thankfulCount", 63 | "statistics.actual.careCount", 64 | "statistics.expected.likeCount", 65 | "statistics.expected.shareCount", 66 | "statistics.expected.commentCount", 67 | "statistics.expected.loveCount", 68 | "statistics.expected.wowCount", 69 | "statistics.expected.hahaCount", 70 | "statistics.expected.sadCount", 71 | "statistics.expected.angryCount", 72 | "statistics.expected.thankfulCount", 73 | "statistics.expected.careCount", 74 | "account.profileImage", 75 | "account.subscriberCount", 76 | "account.url", 77 | "account.platform", 78 | "account.platformId", 79 | "account.accountType", 80 | "account.pageAdminTopCountry", 81 | "account.verified", 82 | "Id", 83 | "legacyid" 84 | ] 85 | } 86 | -------------------------------------------------------------------------------- /presets/Crowdtangle-4_Get_detail_data_about_posts.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "4 Get detail data about posts", 3 | "category": "Crowdtangle", 4 | "description": "Add the post IDs as seed node. Use the full _ ID format.\n\nPlease note that the ID format for Facebook and Instagram are different. For Instagram it's _, while for Facebook it's _. For example this URL\n https://www.facebook.com/81221197163/posts/10158939904512164 has a platformID of 81221197163_10158939904512164. \nInstagram does not expose the IDs in its URL as Facebook, you can pull the necessary IDs from previous presets.\n\nDON'T FORGET: Before you can fetch the data, you must enter your CrowdTangle access token in the corresponding field \"access token\" for authentication.\n\nSee https://github.com/CrowdTangle/API/wiki/Posts#get-postid for more information.\n\nAPI-Cheatsheet: https://help.crowdtangle.com/en/articles/3443476-api-cheat-sheet", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "param", 8 | "auth_tokenname": "token", 9 | "auth_type": "API key", 10 | "auth_uri": "", 11 | "basepath": "https://api.crowdtangle.com", 12 | "extension": "", 13 | "fileext": "", 14 | "filename": "", 15 | "format": "json", 16 | "headers": {}, 17 | "key_paging": "result.pagination.nextPage", 18 | "nodedata": "result.posts", 19 | "objectid": "platformId", 20 | "pages": 1, 21 | "paging_type": "url", 22 | "params": { 23 | "id": "" 24 | }, 25 | "proxy": "", 26 | "redirect_uri": "", 27 | "resource": "/posts/", 28 | "scope": "", 29 | "token_uri": "", 30 | "verb": "GET" 31 | }, 32 | "speed": 200, 33 | "saveheaders": false, 34 | "timeout": 15, 35 | "maxsize": 5, 36 | "columns": [ 37 | "platformId", 38 | "platform", 39 | "date", 40 | "updated", 41 | "type", 42 | "message", 43 | "expandedLinks", 44 | "link", 45 | "postUrl", 46 | "subscriberCount", 47 | "score", 48 | "media", 49 | "statistics.actual.likeCount", 50 | "statistics.actual.shareCount", 51 | "statistics.actual.commentCount", 52 | "statistics.actual.loveCount", 53 | "statistics.actual.wowCount", 54 | "statistics.actual.hahaCount", 55 | "statistics.actual.sadCount", 56 | "statistics.actual.angryCount", 57 | "statistics.actual.thankfulCount", 58 | "statistics.actual.careCount", 59 | "statistics.expected.likeCount", 60 | "statistics.expected.shareCount", 61 | "statistics.expected.commentCount", 62 | "statistics.expected.loveCount", 63 | "statistics.expected.wowCount", 64 | "statistics.expected.hahaCount", 65 | "statistics.expected.sadCount", 66 | "statistics.expected.angryCount", 67 | "statistics.expected.thankfulCount", 68 | "statistics.expected.careCount", 69 | "account.id", 70 | "account.name", 71 | "account.handle", 72 | "account.profileImage", 73 | "account.subscriberCount", 74 | "account.url", 75 | "account.platform", 76 | "account.platformId", 77 | "account.accountType", 78 | "account.pageAdminTopCountry", 79 | "account.verified", 80 | "videoLengthMS", 81 | "legacyId", 82 | "id" 83 | ] 84 | } -------------------------------------------------------------------------------- /presets/Crowdtangle-Get_posts_by_searchterms.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Get posts by searchterms", 3 | "category": "Crowdtangle", 4 | "description": "Add search terms as seed node. \n\nThis preset searches the entire, cross-platform CrowdTangle system of posts (Facebook and Instagram). If you want the data from a specific platform, set the platform parameter. \n\nPlease use a startDate. The collection will be much more quickly when the system only has to search a subset of dates for your data. \n\nThis preset fetches the last 100 posts for your search terms. To see more than 100 posts increase the maximum pages. The API will max out at 10,000 posts. \n\nThe default sortBy Parameter for this preset is date. If you want to have other posts than the first 100 recently published posts, adjust the sortBy Parameter. \n\nDON'T FORGET: Before you can fetch the data, you must enter your CrowdTangle access token in the corresponding field \"access token\" for authentication.\n\nSee https://github.com/CrowdTangle/API/wiki/Search for more parameter options. \n\nAPI Cheatsheet: https://help.crowdtangle.com/en/articles/3443476-api-cheat-sheet", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "param", 8 | "auth_tokenname": "token", 9 | "auth_type": "API key", 10 | "auth_uri": "", 11 | "basepath": "https://api.crowdtangle.com", 12 | "extension": "", 13 | "fileext": "", 14 | "filename": "", 15 | "format": "json", 16 | "headers": {}, 17 | "key_paging": "result.pagination.nextPage", 18 | "nodedata": "result.posts", 19 | "objectid": "platformId", 20 | "pages": 1, 21 | "paging_type": "url", 22 | "params": { 23 | "count": "100", 24 | "includeHistory": "true", 25 | "platforms": "instagram", 26 | "searchTerm": "", 27 | "sortBy": "", 28 | "startDate": "2021-08-01", 29 | "endDate": "2021-08-31" 30 | }, 31 | "proxy": "", 32 | "redirect_uri": "", 33 | "resource": "/posts/search", 34 | "scope": "", 35 | "token_uri": "", 36 | "verb": "GET" 37 | }, 38 | "speed": 200, 39 | "saveheaders": false, 40 | "timeout": 15, 41 | "maxsize": 5, 42 | "columns": [ 43 | "platformId", 44 | "platform", 45 | "date", 46 | "updated", 47 | "description", 48 | "postUrl", 49 | "subscriberCount", 50 | "score", 51 | "media.type", 52 | "media.url", 53 | "media.height", 54 | "media.width", 55 | "statistics.actual.favoriteCount", 56 | "statistics.actual.commentCount", 57 | "statistics.expected.favoriteCount", 58 | "statistics.expected.commentCount", 59 | "account.id", 60 | "account.name", 61 | "account.handle", 62 | "account.profileImage", 63 | "account.subscriberCount", 64 | "account.url", 65 | "account.platform", 66 | "account.platformId", 67 | "account.verified", 68 | "newId", 69 | "id" 70 | ] 71 | } 72 | -------------------------------------------------------------------------------- /presets/Facebook-1_Get_page_data_v20_0_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "1 Get page data (v20.0)", 3 | "category": "Facebook", 4 | "description": "*Please note: Due to changes in the Facebook API, you can no longer request page metadata with this preset. The special permissions needed can only be obtained by registering an own app and going through the app review (https://developers.facebook.com/docs/app-review/introduction).*\n\n*Facebook additionally introduced the New Page Experience, so that the API is more restricted for pages that have been migrated. See the New Pages Experience Overview: https://developers.facebook.com/docs/pages/new-pages-experience/overview/ *\n\nFetch the most basic information about a Facebook page.\n\n\n**Get started with the preset**\n\n1. After creating or opening a database, add the handle of a Facebook page (e.g. \"Uni.Greifswald.de\") as a seed node using the \"Add Nodes\" button.\n\n2. Load the preset by clicking \"Apply\" in the preset window.\n\n3. Login to Facebook in the Facebook module.\n\n4. Select the seed node and then click the \"Fetch Data\" button.\n\nLearn how to use presets in the wiki: https://github.com/strohne/Facepager/wiki/Presets\n\n\n**Further options**\n\nAbout fields: you can add fields to the fields parameter to fetch more data. For example, add \"likes\" to get to know which pages this page liked. See the Facebook API reference linked below for more options. Notice: If you add more fields, also add them in the column setup to make the data visible in the nodes view.\n\n**Facebook API documentation**\n\nBe aware that APIs are changing constantly. You can always find the latest information in the Facebook API Reference: https://developers.facebook.com/docs/graph-api/reference/page/.", 5 | "module": "Facebook", 6 | "options": { 7 | "auth": "param", 8 | "auth_prefix": "", 9 | "auth_tokenname": "access_token", 10 | "auth_type": "OAuth2", 11 | "auth_uri": "https://www.facebook.com/dialog/oauth", 12 | "basepath": "https://graph.facebook.com/v20.0", 13 | "extension": "", 14 | "key_paging": "paging.next", 15 | "nodedata": null, 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_type": "url", 19 | "params": { 20 | "": "", 21 | "fields": "name, about, location, category, talking_about_count, fan_count" 22 | }, 23 | "redirect_uri": "https://www.facebook.com/connect/login_success.html", 24 | "resource": "/", 25 | "scope": "", 26 | "token_uri": "" 27 | }, 28 | "speed": 200, 29 | "saveheaders": false, 30 | "timeout": 15, 31 | "maxsize": 5, 32 | "columns": [ 33 | "name", 34 | "about", 35 | "location.city", 36 | "category", 37 | "fan_count", 38 | "talking_about_count", 39 | "error.message" 40 | ] 41 | } -------------------------------------------------------------------------------- /presets/Facebook-2_Get_posts_v20_0_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "2 Get posts (v20.0)", 3 | "category": "Facebook", 4 | "description": "Fetch the last 20 posts of a Facebook page.\n\n\n**Get started with the preset**\n\n1. After creating or opening a database, add the handle of a Facebook page (e.g. \"Uni.Greifswald.de\") as a seed node using the \"Add Nodes\" button.\n\n2. Load the preset by clicking \"Apply\" in the preset window.\n\n3. Login to Facebook in the Facebook module.\n\n4. Select the seed node and then click the \"Fetch Data\" button.\n\nLearn how to use presets in the wiki: https://github.com/strohne/Facepager/wiki/Presets\n\n\n**Some further options**\n\n1. Change the limit-parameter: You can increase the limit-parameter or increade the maximum pages to get more posts. \n\n2. Filtering by date: To filter by date use the parameters \"since\" and \"until\". The until-parameter always has to be one day later, the day starts at 0:00am GMT. For example, set since to \"2019-03-21\" and until to \"2019-03-22\" to limit the result to 21st of March 2019. See https://developers.facebook.com/docs/graph-api/results for further information.\n\n3. Get other fields: If you want more data than message, created_time and updated_time adjust the fields-parameter. Try things such as: attachments, message_tags, story, permalink_url, shares. You will find more fiields in the API reference (see below).\n\n4. Get reactions: If you want data about reactions try adding fields from the reactions preset: comments.limit(0).summary(1), reactions.limit(0).summary(1), reactions.type(LIKE).limit(0).summary(1).as(like), reactions.type(LOVE).limit(0).summary(1).as(love), reactions.type(HAHA).limit(0).summary(1).as(haha), reactions.type(WOW).limit(0).summary(1).as(wow), reactions.type(SAD).limit(0).summary(1).as(sad), reactions.type(ANGRY).limit(0).summary(1).as(angry), likes.limit(0).summary(true) \n\n\n**Facebook API documentation**\n\nBe aware of the API limitations, Facebook provides only up to 600 posts per year for each page. For more information and query options see the API reference: https://developers.facebook.com/docs/graph-api/reference/v20.0/page/feed\n\nTo understand the data contained in a post see https://developers.facebook.com/docs/graph-api/reference/v20.0/post \n\nThe technique to query reactions along with the posts is called field expansion, see https://developers.facebook.com/docs/graph-api/guides/field-expansion", 5 | "module": "Facebook", 6 | "options": { 7 | "auth": "param", 8 | "auth_prefix": "", 9 | "auth_tokenname": "access_token", 10 | "auth_type": "OAuth2", 11 | "auth_uri": "https://www.facebook.com/dialog/oauth", 12 | "basepath": "https://graph.facebook.com/v20.0", 13 | "extension": "", 14 | "key_paging": "paging.next", 15 | "nodedata": "data", 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_type": "url", 19 | "params": { 20 | "": "", 21 | "fields": "message, from, created_time, updated_time", 22 | "limit": "20" 23 | }, 24 | "redirect_uri": "https://www.facebook.com/connect/login_success.html", 25 | "resource": "//posts", 26 | "scope": "", 27 | "token_uri": "" 28 | }, 29 | "speed": 200, 30 | "saveheaders": false, 31 | "timeout": 15, 32 | "maxsize": 5, 33 | "columns": [ 34 | "message", 35 | "created_time", 36 | "updated_time", 37 | "error.message" 38 | ] 39 | } -------------------------------------------------------------------------------- /presets/Facebook-3_Get_comments_v20_0_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "3 Get comments (v20.0)", 3 | "category": "Facebook", 4 | "description": "Fetch the last 20 comments of a post or the last 20 replies to a comment.\n\n\n**Get started with the preset**\n\n1. Before you can use this preset, you have to fetch posts or add Facebook IDs of some posts. See the preset named \"Get posts\".\n\n2. Load the preset by clicking the \"Apply\" button.\n\n3. Login to Facebook in the Facebook module\n\n4. Select post or comment nodes and click the button \"Fetch Data\".\n\n\n**Some hints**\n\n1. Replies: Replies are comments to comments. Thus, you can use the comment preset to get replies to the comments. Alternatively, set the filter parameter to \"stream\" to get replies along with the comments.\n\n2. About the author of comments: you cannot access data about the author of comments (if you are not the owner of the page).\n\n3. Adding fields: When adding fields to the fields-parameter also add them in the column setup to make the data visible in the nodes view.\n\n\n**Facebook API documentation**\n\nSee the reference of the comments edge for further information: https://developers.facebook.com/docs/graph-api/reference/v20.0/object/comments\n\nFor information about available fields see the Facebook API reference: https://developers.facebook.com/docs/graph-api/reference/v20.0/comment\n\n", 5 | "module": "Facebook", 6 | "options": { 7 | "auth": "param", 8 | "auth_prefix": "", 9 | "auth_tokenname": "access_token", 10 | "auth_type": "OAuth2", 11 | "auth_uri": "https://www.facebook.com/dialog/oauth", 12 | "basepath": "https://graph.facebook.com/v20.0", 13 | "extension": "", 14 | "key_paging": "paging.next", 15 | "nodedata": "data", 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_type": "url", 19 | "params": { 20 | "": "", 21 | "fields": "message, created_time, parent, comment_count, like_count", 22 | "filter": "stream", 23 | "summary": "1" 24 | }, 25 | "redirect_uri": "https://www.facebook.com/connect/login_success.html", 26 | "resource": "//comments", 27 | "scope": "", 28 | "token_uri": "" 29 | }, 30 | "speed": 200, 31 | "saveheaders": false, 32 | "timeout": 15, 33 | "maxsize": 5, 34 | "columns": [ 35 | "message", 36 | "created_time", 37 | "like_count", 38 | "comment_count", 39 | "error.message" 40 | ] 41 | } -------------------------------------------------------------------------------- /presets/Facebook-Detail_data_about_posts_v20_0_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Detail data about posts (v20.0)", 3 | "category": "Facebook", 4 | "description": "*Please note: Due to changes in the Facebook API, fetching post data with this preset is limited. The special permissions needed can only be obtained by registering an own app and going through the app review (https://developers.facebook.com/docs/app-review/introduction).*\n\nFetch information about a post.\n\n\n**Get started with the preset**\n\n1. Before you can use this preset, you have to fetch posts or add Facebook IDs of some posts. Full post IDs consist of the page ID, an underscore and the post ID, e.g. \"20373776304_10158930464676305\". You can fetch posts using the preset \"Get Facebook posts\".\n\n2. Load the preset by clicking the \"Apply\" button.\n\n3. Login to Facebook in the Facebook module\n\n4. Select the post nodes and click the button \"Fetch Data\". \n\n\n**Facebook API documentation**\n\nFor information about all available fields see https://developers.facebook.com/docs/graph-api/reference/v20.0/post ", 5 | "module": "Facebook", 6 | "options": { 7 | "auth": "param", 8 | "auth_prefix": "", 9 | "auth_tokenname": "access_token", 10 | "auth_type": "OAuth2", 11 | "auth_uri": "https://www.facebook.com/dialog/oauth", 12 | "basepath": "https://graph.facebook.com/v20.0", 13 | "extension": "", 14 | "key_paging": "paging.next", 15 | "nodedata": null, 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_type": "url", 19 | "params": { 20 | "": "", 21 | "fields": "from,likes.limit(0).summary(1),picture,attachments" 22 | }, 23 | "redirect_uri": "https://www.facebook.com/connect/login_success.html", 24 | "resource": "/", 25 | "scope": "", 26 | "token_uri": "" 27 | }, 28 | "speed": 200, 29 | "saveheaders": false, 30 | "timeout": 15, 31 | "maxsize": 5, 32 | "columns": [ 33 | "from.name", 34 | "error.message" 35 | ] 36 | } -------------------------------------------------------------------------------- /presets/Facebook-Download_images.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Download images", 3 | "category": "Facebook", 4 | "description": "The preset demonstrates how to download images or other kind of media. The principle is suitable for many different use cases. First, you need to get the URL of the image. For example, use the Facebook posts preset and make sure \"picture\" or \"attachments\" is in the fields list. In this case the URL can be found in the picture or attachements keys of the node.\n\nIn the base path set by this preset you find a Facepager placeholder referring to the key with the URL. Inspect your post data to figure out the right key. More information about placeholders can be found in the Facepager wiki: https://github.com/strohne/Facepager/wiki/URLs,-Placeholders,-Nodes-and-Keys\n\nBefore you fetch data make sure to select a download folder. The image is saved into this folder. Usually, `` is a good choice for the filename. The file extension will be guessed from the data if you set the custom file extension to ``.", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "", 8 | "resource": "", 9 | "params": {}, 10 | "headers": {}, 11 | "verb": "GET", 12 | "format": "file", 13 | "filename": "", 14 | "fileext": "", 15 | "pages": 1, 16 | "paging_type": "key", 17 | "key_paging": null, 18 | "paging_stop": null, 19 | "param_paging": null, 20 | "nodedata": null, 21 | "objectid": null, 22 | "scope": "", 23 | "proxy": "", 24 | "auth_type": "Disable", 25 | "auth_uri": "", 26 | "redirect_uri": "", 27 | "token_uri": "", 28 | "auth": "disable" 29 | }, 30 | "speed": 200, 31 | "headers": false, 32 | "columns": [ 33 | "type", 34 | "media.image.src", 35 | "url" 36 | ] 37 | } -------------------------------------------------------------------------------- /presets/Facebook-Get_page_insights_v20_0_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Get page insights (v20.0)", 3 | "category": "Facebook", 4 | "description": "*Facebook introduced the New Page Experience, so that the API is more restricted for pages that have been migrated. See the New Pages Experience Overview: https://developers.facebook.com/docs/pages/new-pages-experience/overview/*\n\nThis preset can be used by page admins to fetch page analytics.\n\n\n**Get started with the preset**\n\n1. You need to be admin of the Facebook page to get insights such as page impressions. \n\n2. Load the preset by clicking \"Apply\" in the preset window.\n\n3. To authorize, click the settings button (next to login) and set the Page ID (the name of your page, e.g. Uni.Greifswald.de). \n\n4. Login to Facebook in the Facebook module.\n\n5. Select the seed node and then click the \"Fetch Data\" button.\n\n\n**Facebook API documentation**\n\nSee the Facebook API reference for further options: https://developers.facebook.com/docs/graph-api/reference/page/insights/#availmetrics", 5 | "module": "Facebook", 6 | "options": { 7 | "auth": "param", 8 | "auth_prefix": "", 9 | "auth_tokenname": "access_token", 10 | "auth_type": "OAuth2", 11 | "auth_uri": "https://www.facebook.com/dialog/oauth", 12 | "basepath": "https://graph.facebook.com/v20.0", 13 | "extension": "", 14 | "key_paging": "paging.next", 15 | "nodedata": "data", 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_type": "url", 19 | "params": { 20 | "": "", 21 | "metric": "page_impressions" 22 | }, 23 | "redirect_uri": "https://www.facebook.com/connect/login_success.html", 24 | "resource": "//insights", 25 | "scope": "", 26 | "token_uri": "" 27 | }, 28 | "speed": 200, 29 | "saveheaders": false, 30 | "timeout": 15, 31 | "maxsize": 5, 32 | "columns": [ 33 | "name", 34 | "period", 35 | "values.*.value", 36 | "values.*.end_time", 37 | "title", 38 | "description", 39 | "id" 40 | ] 41 | } -------------------------------------------------------------------------------- /presets/Facebook-Get_reactions_v20_0_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Get reactions (v20.0)", 3 | "category": "Facebook", 4 | "description": "*Please note: Due to changes in the Facebook API, fetching reactions with this preset is limited. The special permissions needed can only be obtained by registering an own app and going through the app review (https://developers.facebook.com/docs/app-review/introduction).*\n\nFetch all different types of reactions in one call, instead of querying every single edge.\n\n**Get started with the preset**\n\n1. Before you can use this preset, you have to fetch posts or add Facebook IDs of some posts. Full post IDs consist of the page ID, an underscore and the post ID, e.g. \"20373776304_10158930464676305\". You can fetch posts using the preset \"Get posts\".\n\n2. Load the preset by clicking \"Apply\" in the preset window.\n\n3. Login to Facebook in the Facebook module.\n\n4. Select the seed node and then click the \"Fetch Data\" button.\n\n\n**Facebook API documentation**\n\nBe aware that APIs are changing constantly. You can always find the latest information about permissions, available fields etc. in the Facebook API Reference: https://developers.facebook.com/docs/graph-api/reference/v20.0/object/reactions", 5 | "module": "Facebook", 6 | "options": { 7 | "basepath": "https://graph.facebook.com/v20.0", 8 | "resource": "/", 9 | "params": { 10 | "": "", 11 | "fields": "comments.limit(0).summary(1),\nshares, reactions.limit(0).summary(1),\nreactions.type(LIKE).limit(0).summary(1).as(like),\nreactions.type(LOVE).limit(0).summary(1).as(love),\nreactions.type(HAHA).limit(0).summary(1).as(haha),\nreactions.type(WOW).limit(0).summary(1).as(wow),\nreactions.type(SAD).limit(0).summary(1).as(sad),\nreactions.type(ANGRY).limit(0).summary(1).as(angry)" 12 | }, 13 | "extension": "", 14 | "pages": 10, 15 | "paging_type": "url", 16 | "key_paging": "paging.next", 17 | "nodedata": null, 18 | "objectid": "id", 19 | "scope": "", 20 | "auth_type": "OAuth2", 21 | "auth_uri": "https://www.facebook.com/dialog/oauth", 22 | "redirect_uri": "https://www.facebook.com/connect/login_success.html", 23 | "token_uri": "", 24 | "auth": "param", 25 | "auth_prefix": "", 26 | "auth_tokenname": "access_token" 27 | }, 28 | "speed": 200, 29 | "saveheaders": false, 30 | "timeout": 15, 31 | "maxsize": 5, 32 | "columns": [ 33 | "comments.summary.total_count", 34 | "shares.count", 35 | "reactions.summary.total_count", 36 | "like.summary.total_count", 37 | "love.summary.total_count", 38 | "wow.summary.total_count", 39 | "haha.summary.total_count", 40 | "sad.summary.total_count", 41 | "angry.summary.total_count" 42 | ] 43 | } -------------------------------------------------------------------------------- /presets/Facebook-Pages_a_page_likes_v20_0_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Pages a page likes (v20.0)", 3 | "category": "Facebook", 4 | "description": "*Please note: This endpoint is not supported for Pages that have been migrated to Facebook's [New Pages Experience](https://developers.facebook.com/docs/pages/new-pages-experience).*\n\nGet the pages a page likes.\n\n\n**Get started with the preset**\n1. After creating or opening a database, add page IDs as seed nodes using the \"Add Nodes\" button, e.g. \"116949331668018\". You can get page IDs using the preset \"Get page data\".\n\n2. Load the preset by clicking \"Apply\" in the preset window.\n\n3. Login to Facebook in the Facebook module.\n\n4. Select the seed node and then click the \"Fetch Data\" button.\n\n**Facebook documentation**\n\nSee https://developers.facebook.com/docs/graph-api/reference/page/likes/ for more information about the endpoint. \n", 5 | "module": "Facebook", 6 | "options": { 7 | "auth": "param", 8 | "auth_prefix": "", 9 | "auth_tokenname": "access_token", 10 | "auth_uri": "https://www.facebook.com/dialog/oauth", 11 | "basepath": "https://graph.facebook.com/v20.0", 12 | "extension": "", 13 | "key_paging": "paging.next", 14 | "nodedata": null, 15 | "objectid": null, 16 | "pages": 1, 17 | "paging_type": "url", 18 | "params": { 19 | "": "", 20 | "fields": "id,about,bio,category,company_overview,description,fan_count,general_info,keywords,link,location,members,name,release_date,start_info,talking_about_count,username,website,were_here_count,written_by" 21 | }, 22 | "redirect_uri": "https://www.facebook.com/connect/login_success.html", 23 | "resource": "//likes", 24 | "scope": "", 25 | "token_uri": "" 26 | }, 27 | "speed": 200, 28 | "saveheaders": false, 29 | "timeout": 15, 30 | "maxsize": 5, 31 | "columns": [ 32 | "id", 33 | "about", 34 | "category", 35 | "fan_count", 36 | "link", 37 | "location.city", 38 | "location.country", 39 | "location.latitude", 40 | "location.longitude", 41 | "location.street", 42 | "location.zip", 43 | "name", 44 | "start_info.type", 45 | "start_info.date.year", 46 | "start_info.date.month", 47 | "talking_about_count", 48 | "username", 49 | "website", 50 | "were_here_count" 51 | ] 52 | } -------------------------------------------------------------------------------- /presets/Files-Download_Google_Cloud_Storage_buckets.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "This preset works at best together with the preset \"Get information about Google Cloud Storage bucket\", which gets bucket data and creates proper nodes for the files in the bucket. Alternatively you can add names of objects (=filename inside of the bucket) as nodes and set -parameter to the name of the bucket.\n\nBefore fetching data create a folder on your computer and select it in the settings.\n\n__Authorization notes__\n\nYou need to register an app at Google Cloud Platform: https://console.cloud.google.com. Enable Cloud Storage JSON API and create OAuth2-Credentials. In Facepager, open the authorization settings (next to Login-button) and provide Client ID and Client Secret fields. Then you are ready to login with Facepager, which gets you an access token. Make sure the header option or param option is selected in the box next to the access token.\n\nAlternatively, you can get a request token from https://developers.google.com/oauthplayground/. Add request token as header: \"Authorization: Bearer [OAUTH2_TOKEN]\". Attention: the token is preceded by the string \"Bearer \".\n", 3 | "module": "Files", 4 | "category": "Google Cloud Platform", 5 | "speed": 200, 6 | "options": { 7 | "verb": "GET", 8 | "resource": "/storage/v1/b//o/", 9 | "objectid": "filename", 10 | "headers": {}, 11 | "token_uri": "https://accounts.google.com/o/oauth2/token", 12 | "basepath": "https://www.googleapis.com", 13 | "auth": "param", 14 | "filename": "", 15 | "fileext": "", 16 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 17 | "querytype": "Files:storage/v1/b//o/", 18 | "redirect_uri": "https://localhost", 19 | "nodedata": null, 20 | "scope": "https://www.googleapis.com/auth/devstorage.full_control", 21 | "params": { 22 | "alt": "media", 23 | "": "" 24 | } 25 | }, 26 | "columns": [ 27 | "kind", 28 | "contentType", 29 | "name", 30 | "timeCreated", 31 | "generation", 32 | "md5Hash", 33 | "bucket", 34 | "updated", 35 | "crc32c", 36 | "metageneration", 37 | "mediaLink", 38 | "timeStorageClassUpdated", 39 | "etag", 40 | "size", 41 | "id", 42 | "selfLink", 43 | "storageClass" 44 | ], 45 | "name": "Download files from Google Cloud Storage buckets" 46 | } 47 | -------------------------------------------------------------------------------- /presets/GPT_3_from_Open_AI_-GTP_3_Completion.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "GTP 3 Completion", 3 | "category": "GPT 3 from Open AI ", 4 | "description": "Text completion with GPT-3 lets you predict what comes next after a given text. Add a prompt as a seed node, enter your access token (see below) and fetch data.\n\n**Authorization**\n \nOpenAI\u2019s API provides access to GPT-3. In order to use the OpenAI API, you need to register on https://openai.com/api/. After registration, you will find a secret API key in your account settings (https://beta.openai.com/account/api-keys). In Facepager, enter the key into the access token field. *Note*: You need to pay for using the API, however, as a new user you get free credits for trial purposes.\n\n\n**Get started with the preset**\n \n1. After creating or opening a database, add a seed node with your prompt, e.g. \u201cSay this is a test\u201d using the \"Add Nodes\" button. \n\n2. Load the preset by clicking \"Apply\" in the preset window. \n\n3. Login to OpenAI, find your secret API key and put it into the access token field of Facepager.\n\n4. Select the seed node and finally click \u201cFetch Data\u201d.\n\n**Further Options**\n\nWithin the payload field, you can customize the query.\n\n*prompt*: The placeholder is replaced by you node data. Thus, add your prompts as seed nodes.\n\n*temperature*: Defines the sampling temperature. Higher values indicate riskier (or more creative) predictions. \n\n*max_tokens*: The maximal tokens (=words) the model returns. Each model has a context length that cannot be exceed (e.g. 4000 tokens with the Davinci model). \n\nGTP 3 is powered by four models for different purposes: ada, davinci, curie, babbage, see https://beta.openai.com/docs/engines/overview. This preset uses Davinci as defined in the base path.\n\nFor more options see the API documentation: https://beta.openai.com/docs/api-reference/completions.\n", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "https://api.openai.com/v1", 8 | "resource": "/engines/text-davinci-002/completions", 9 | "params": {}, 10 | "extension": "", 11 | "headers": { 12 | "Content-Type": "application/json" 13 | }, 14 | "verb": "POST", 15 | "format": "json", 16 | "encoding": "", 17 | "payload": "{\n \"prompt\": \"\", \n \"temperature\": 0, \n \"max_tokens\": 20\n}", 18 | "filename": "", 19 | "fileext": "", 20 | "pages": 1, 21 | "paging_type": "url", 22 | "key_paging": "result.pagination.nextPage", 23 | "nodedata": "choices", 24 | "objectid": "text", 25 | "scope": "", 26 | "proxy": "", 27 | "auth_type": "OAuth2", 28 | "auth_uri": "", 29 | "redirect_uri": "", 30 | "token_uri": "", 31 | "auth": "header", 32 | "auth_tokenname": "Authorization", 33 | "auth_prefix": "Bearer " 34 | }, 35 | "speed": 200, 36 | "saveheaders": false, 37 | "timeout": 15, 38 | "maxsize": 5, 39 | "columns": [ 40 | "text", 41 | "index", 42 | "logprobs", 43 | "finish_reason" 44 | ] 45 | } -------------------------------------------------------------------------------- /presets/GPT_3_from_Open_AI_-GTP_3_Text_Classification_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "GTP 3 Text Classification ", 3 | "category": "GPT 3 from Open AI ", 4 | "description": "With GPT 3 you can perform text classification, i.e. predict the most likely label for a query. Add your prompts as seed nodes (e.g. \"I love you\"), enter the access token (see below) and click the fetch data button.\n\n\n**Authorization**\n \nOpenAI\u2019s API provides access to GPT-3. In order to use the OpenAI\u2019 API, you need to register on https://openai.com/api/. After the registration, you will find a secret API key in your account settings (https://beta.openai.com/account/api-keys). In Facepager, enter the key into the access token field. *Note*: You need to pay for using the API, however, as a new user you get free credits for trial purposes.\n\n\n**Get started with the preset** \n\n1. After creating or opening a database, add a seed node with your prompt (e.g. \u201cI love you\u201d) using the \"Add Nodes\" button. \n\n2. Load the preset by clicking \"Apply\" in the preset window. \n\n3. Login to OpenAI, find your API key and enter it into the access token field of Facepager.\n\n4. Select the seed node and then click \u201cFetch Data\u201d. \n\n**Further Options**\n\nWithin the payload field, you can customize the query.\n\n*examples*: Defines labeled examples, which are the classification basis for the model. Adopt the syntax used in the preset. \n\n*query*: Define the prompt, for example \u201cI love you\u201d. This field is required. You can use `` as a placeholder, to refer to node data. \n\n*model*: Defines the completion model, GTP 3 provides four models: ada, davinci, curie, babbage.See https://beta.openai.com/docs/engines/overview. This field is required, the preset uses the curie model. \n\nFor more options see the API documentation: https://beta.openai.com/docs/api-reference/classifications", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "header", 8 | "auth_prefix": "Bearer ", 9 | "auth_tokenname": "Authorization", 10 | "auth_type": "OAuth2", 11 | "auth_uri": "", 12 | "basepath": "https://api.openai.com/v1", 13 | "encoding": "", 14 | "extension": "", 15 | "fileext": "", 16 | "filename": "", 17 | "format": "json", 18 | "headers": { 19 | "Content-Type": "application/json" 20 | }, 21 | "key_paging": null, 22 | "nodedata": null, 23 | "objectid": null, 24 | "pages": 1, 25 | "paging_stop": null, 26 | "paging_type": "key", 27 | "param_paging": null, 28 | "params": {}, 29 | "payload": "{\n \"query\": \"\",\n \"labels\": [\"Positive\", \"Negative\", \"Neutral\"],\n \"examples\": [\n [\"A happy moment\", \"Positive\"],\n [\"I am sad.\", \"Negative\"],\n [\"I am feeling awesome\", \"Positive\"]\n ],\n \"temperature\": 0,\n \"logprobs\":1,\n \"search_model\": \"ada\",\n \"model\": \"curie\",\n \"return_prompt\":false\n}\n", 30 | "proxy": "", 31 | "redirect_uri": "", 32 | "resource": "/classifications", 33 | "scope": "", 34 | "token_uri": "", 35 | "verb": "POST" 36 | }, 37 | "speed": 200, 38 | "saveheaders": false, 39 | "timeout": 15, 40 | "maxsize": 5, 41 | "columns": [ 42 | "label", 43 | "object", 44 | "model" 45 | ] 46 | } -------------------------------------------------------------------------------- /presets/Generic-Asynchronous_Speech_Recognition_using_Google_Cloud_Storage.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "At first, you need to upload your audio file to a Google Cloud Storage bucket. The encoding must be FLAC, see https://cloud.google.com/speech-to-text/docs/reference/rest/v1/RecognitionConfig#AudioEncoding for other options.\n\nSecond, get the bucket contents with the preset \"Get contents of bucket from Google Cloud Storage\". Alternatively, you can adjust the uri-setting in the payload.\n\nFetching data starts asynchronous processing at Google and gives you backk an ID of the operation. Last step is to poll the status of the operation with the preset \"Result of asyncronous speech recognition\".\n\nMaximum length of audio is about 180 minutes. Be aware of prices, see https://cloud.google.com/speech-to-text/quotas\n\n__Authorization notes__\n\nYou need to register an app at Google Cloud Platform: https://console.cloud.google.com. Enable Cloud Storage JSON API and create OAuth2-Credentials. In Facepager, open the authorization settings (next to Login-button) and provide Client ID and Client Secret fields. Then you are ready to login with Facepager, which gets you an access token. Make sure the header option or param option is selected in the box next to the access token.\n\nAlternatively, you can get a request token from https://developers.google.com/oauthplayground/. Add request token as header: \"Authorization: Bearer [OAUTH2_TOKEN]\". Attention: the token is preceded by the string \"Bearer \".", 3 | "module": "Generic", 4 | "category": "Google Cloud Platform", 5 | "speed": 200, 6 | "options": { 7 | "redirect_uri": "https://localhost", 8 | "resource": "/speech:longrunningrecognize", 9 | "objectid": "name", 10 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 11 | "token_uri": "https://accounts.google.com/o/oauth2/token", 12 | "basepath": "https://speech.googleapis.com/v1", 13 | "auth": "header", 14 | "headers": {}, 15 | "verb": "POST", 16 | "params": {}, 17 | "scope": "https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/devstorage.read_write https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/cloud-language", 18 | "nodedata": null, 19 | "payload": "{\n\"config\": {\n \"encoding\":\"FLAC\",\n \"languageCode\": \"en-US\",\n \"enableWordTimeOffsets\": false,\n maxAlternatives:2\n },\n\"audio\":{\n \"uri\":\"gs:///\"\n }\n}" 20 | }, 21 | "columns": [ 22 | "error.message", 23 | "name" 24 | ], 25 | "name": "Asynchronous speech recognition using cloud storage" 26 | } 27 | -------------------------------------------------------------------------------- /presets/Generic-Extract_links_from_website.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Extract links from a webpage", 3 | "category": "Scraping", 4 | "description": "This preset gets the HTML source code from a website and extracts all the links.\n\nAdd the URL of a website as a node. Adjust the download folder in the Query Setup. When you fetch the data, the file with the HTML source code of the webpage will be stored in this folder. Simultaneously, the links from that website appear in the Data View as new nodes. ", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "", 8 | "resource": "", 9 | "params": {}, 10 | "extension": "", 11 | "headers": { 12 | "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 13 | "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36" 14 | }, 15 | "verb": "GET", 16 | "format": "links", 17 | "filename": "", 18 | "fileext": "", 19 | "pages": 1, 20 | "paging_type": "key", 21 | "key_paging": null, 22 | "paging_stop": null, 23 | "param_paging": null, 24 | "nodedata": "links", 25 | "objectid": "url", 26 | "scope": "", 27 | "proxy": "", 28 | "auth_type": "Disable", 29 | "auth_uri": "", 30 | "redirect_uri": "", 31 | "token_uri": "", 32 | "auth": "disable", 33 | "auth_tokenname": "", 34 | "auth_prefix": "" 35 | }, 36 | "speed": 200, 37 | "saveheaders": false, 38 | "timeout": 15, 39 | "columns": [ 40 | "url_domain_normalized", 41 | "text" 42 | ] 43 | } -------------------------------------------------------------------------------- /presets/Generic-Get_Bucket_Contents_from_Google Cloud Storage.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Add bucket name as node, e.g. \"myfancyusername-myfancybucket\".\n\nSee https://cloud.google.com/storage/docs/getting-bucket-information for more information about getting bucket information.\n\n__Authorization notes__\n\nYou need to register an app at Google Cloud Platform: https://console.cloud.google.com. Enable Cloud Storage JSON API and create OAuth2-Credentials. In Facepager, open the authorization settings (next to Login-button) and provide Client ID and Client Secret fields. Then you are ready to login with Facepager, which gets you an access token. Make sure the header option or param option is selected in the box next to the access token.\n\nAlternatively, you can get a request token from https://developers.google.com/oauthplayground/. Add request token as header: \"Authorization: Bearer [OAUTH2_TOKEN]\". Attention: the token is preceded by the string \"Bearer \".\n", 3 | "module": "Generic", 4 | "category": "Google Cloud Platform", 5 | "speed": 200, 6 | "options": { 7 | "redirect_uri": "https://localhost", 8 | "resource": "/b//o", 9 | "objectid": "name", 10 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 11 | "token_uri": "https://accounts.google.com/o/oauth2/token", 12 | "basepath": "https://www.googleapis.com/storage/v1", 13 | "auth": "header", 14 | "headers": {}, 15 | "verb": "GET", 16 | "params": {}, 17 | "scope": "https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/devstorage.read_write https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/cloud-language", 18 | "nodedata": "items", 19 | "folder": "C:\\Users\\Jakob\\Facepager\\audio", 20 | "querytype": "Generic:https://www.googleapis.com/storage/v1/b//o" 21 | }, 22 | "columns": [ 23 | "error.message", 24 | "id", 25 | "selfLink", 26 | "updated" 27 | ], 28 | "name": "Get contents of bucket from Google Cloud Storage" 29 | } 30 | -------------------------------------------------------------------------------- /presets/Generic-Get_publications_from_arxiv_.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": "arxiv", 3 | "columns": [ 4 | "author.name.$", 5 | "published.$", 6 | "title.$", 7 | "link.0.@href" 8 | ], 9 | "description": "This presets demonstrates how to fetch XML data, e.g. RSS feeds. Add search term as node, for example \"abs:network\" to search the term \"network\" in all abstracts. \n\nDocumentation of the API: https://arxiv.org/help/api/user-manual", 10 | "headers": false, 11 | "module": "Generic", 12 | "name": "Get publications from arxiv.org", 13 | "options": { 14 | "auth": "disable", 15 | "auth_type": "Disable", 16 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 17 | "basepath": "http://export.arxiv.org/api/query", 18 | "format": "xml", 19 | "headers": {}, 20 | "key_paging": null, 21 | "nodedata": "feed.entry", 22 | "objectid": "id.$", 23 | "pages": 1, 24 | "param_paging": null, 25 | "params": { 26 | "search_query": "" 27 | }, 28 | "redirect_uri": "https://localhost", 29 | "resource": "", 30 | "scope": "https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/devstorage.read_write https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/cloud-language", 31 | "token_uri": "https://accounts.google.com/o/oauth2/token", 32 | "verb": "GET" 33 | }, 34 | "speed": 200 35 | } -------------------------------------------------------------------------------- /presets/Generic-Google_Maps_API_Geocoding.3_9.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "By geocoding you get the latitude and longitude of an address. The Google Maps API offers geocoding (as well as reverse geocoding).\n\nYou need an Google API key, see https://developers.google.com/maps/documentation/geocoding/start\n\nAdd addresses as node, enter your API key in the parameters and fetch data.\n\nBe aware of request limits (at the time of this writing 2500 requests per day and 50 requests per second): https://developers.google.com/maps/documentation/geocoding/usage-limits", 3 | "module": "Generic", 4 | "category": "Google Maps", 5 | "speed": 200, 6 | "options": { 7 | "resource": "/maps/api/geocode/json", 8 | "objectid": "formatted_address", 9 | "basepath": "https://maps.googleapis.com", 10 | "headers": {}, 11 | "params": { 12 | "key": "YOUR_GOOGLE_API_KEY_HERE", 13 | "address": "" 14 | }, 15 | "nodedata": "results" 16 | }, 17 | "columns": [ 18 | "name" 19 | ], 20 | "name": "Geocoding" 21 | } 22 | -------------------------------------------------------------------------------- /presets/Generic-Result_of_asychronous_speech_recognition.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "The preset \"Start speech recognition\" gives you back an operation ID. Wait some seconds. Then poll the result of the operation with this preset.\n\nSee https://cloud.google.com/speech-to-text/docs/async-recognize for further information.\n\n__Authorization notes__\n\nYou need to register an app at Google Cloud Platform: https://console.cloud.google.com. Enable Cloud Storage JSON API and create OAuth2-Credentials. In Facepager, open the authorization settings (next to Login-button) and provide Client ID and Client Secret fields. Then you are ready to login with Facepager, which gets you an access token. Make sure the header option or param option is selected in the box next to the access token.\n\nAlternatively, you can get a request token from https://developers.google.com/oauthplayground/. Add request token as header: \"Authorization: Bearer [OAUTH2_TOKEN]\". Attention: the token is preceded by the string \"Bearer \".", 3 | "module": "Generic", 4 | "category": "Google Cloud Platform", 5 | "speed": 200, 6 | "options": { 7 | "redirect_uri": "https://localhost", 8 | "resource": "/operations/", 9 | "objectid": "alternatives.0.transcript", 10 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 11 | "token_uri": "https://accounts.google.com/o/oauth2/token", 12 | "basepath": "https://speech.googleapis.com/v1", 13 | "auth": "header", 14 | "headers": {}, 15 | "verb": "GET", 16 | "params": {}, 17 | "scope": "https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/devstorage.read_write https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/cloud-language", 18 | "nodedata": "response.results" 19 | }, 20 | "columns": [ 21 | "error.message", 22 | "alternatives.0.transcript", 23 | "alternatives.0.confidence", 24 | "alternatives.1.transcript", 25 | "alternatives.1.confidence" 26 | ], 27 | "name": "Result of asynchronous speech recognition" 28 | } 29 | -------------------------------------------------------------------------------- /presets/Generic-Scrape_Wikipedia_tables.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": "Scraping", 3 | "columns": [ 4 | "text|xpath:string(*[1])", 5 | "text|xpath:string(*[2])", 6 | "text|xpath:string(*[3])", 7 | "text|xpath:string(*[4])", 8 | "text|xpath:string(*[5])", 9 | "text|xpath:string(*[6])", 10 | "text|xpath://a/@href", 11 | "text|xpath://a/text()" 12 | ], 13 | "description": "Add the URL of a webpage containing a table as seed node, select the node, and fetch data, e.g. \nhttps://en.wikipedia.org/wiki/List_of_countries_and_dependencies_by_population\n\nAll tables and their rows are scraped. If a page contains multiple tables and you are interested in only one specific table, adjust the Key to extract-field. For example, the key \"text|xpath://table[3]//tr\" extracts only rows in the third table. \n\nThe column setup defines what data you see in the table. XPath expressions are used to extract the data. The preset shows the first six table columns followed by two columns with all contained links and link texts. You can adjust the setup according to your needs and the structure of the table. Inspect the source code of the table with you browser in order to understand the structure. The following examples may help you with finding XPath expressions.\n\n- Extract the HTML code of the second td-element: text|xpath://td[2]\n\n- Extract all text of the second td-element: text|xpath:string(//td[2])\n\n- Extract all text nodes of the second td-element. Multiple nodes are pasted together with a semicolon by Facepager: text|xpath://td[2]/text()\n\n- Extract the URL of all image tags: text|xpath://img/@src\n\nHints:\n- You can blank out the Key to extract-field and use the Extract Data-button after fecthing the complete source code of the page.\n\n- Some tables in webpages are loaded by JavaScript (using HttpRequests / Ajax). In this case, you need to figure out the URL of the page containing the table. The network analysis tab in the developer console of your browser (press F12 in firefox) is a relatively convenient way.\n\n- If tables are split over several pages, figure out which parts in the URL change between the pages. You can use the paging mechanism of Facepager to set the parameter and automatically download all pages.\n\n- To address an element no matter where it is in the source code, use an relative XPath expression. For example, \"xpath://td\" gets all td elements. \n\n- An asterisk * in XPath expressions denotes any element. For example,\"*\" adresses all direct child elements of the root element.\n\n- In XPath, the current element is denoted by a dot. For example, \"./*/img\" adresses all img elements that are direct children to the current element. \n\n- See https://github.com/strohne/Facepager/wiki/The-Basic-Concept#webscraping-with-css-and-xpath for an explanation of XPath and CSS in Facepager.\n\nOther pages you could try to scrape:\nhttps://en.wikipedia.org/wiki/List_of_news_television_channels\nhttps://en.wikipedia.org/wiki/List_of_most_expensive_films\nhttps://en.wikipedia.org/wiki/List_of_micronations\nhttps://en.wikipedia.org/wiki/List_of_countries_by_population_(United_Nations)\n", 14 | "headers": false, 15 | "module": "Generic", 16 | "name": "Scrape Wikipedia tables", 17 | "options": { 18 | "auth": "disable", 19 | "auth_type": "Disable", 20 | "auth_uri": "", 21 | "basepath": "", 22 | "fileext": "", 23 | "filename": "", 24 | "format": "text", 25 | "headers": {}, 26 | "key_paging": null, 27 | "nodedata": "text|xpath://table//tr", 28 | "objectid": "text|xpath:string(td[1])", 29 | "offset_start": 1, 30 | "offset_step": 1, 31 | "pages": 1, 32 | "paging_type": "key", 33 | "param_paging": null, 34 | "params": {}, 35 | "redirect_uri": "", 36 | "resource": "", 37 | "scope": "", 38 | "token_uri": "", 39 | "verb": "GET" 40 | }, 41 | "speed": 200 42 | } -------------------------------------------------------------------------------- /presets/Generic-Synchrounous_Speech_Recognition.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Select the folder where the file resides (below the payload field). In the dialog make sure to check the option \"Add files as nodes\". Alternatively, you can manually add filenames as nodes, e.g. \"amazing-speech.flac\". \n\nFiles need to be FLAC-encoded, 16bit, 16000Hz. See https://cloud.google.com/speech-to-text/docs/reference/rest/v1/RecognitionConfig#AudioEncoding for other options.\n\nMaximum length is about 1 minute, see https://cloud.google.com/speech-to-text/quotas. Be aware of prices.\n\nSee https://cloud.google.com/speech-to-text/docs/sync-recognize for further information.\n\n__Authorization notes__\n\nYou need to register an app at Google Cloud Platform: https://console.cloud.google.com. Enable Cloud Storage JSON API and create OAuth2-Credentials. In Facepager, open the authorization settings (next to Login-button) and provide Client ID and Client Secret fields. Then you are ready to login with Facepager, which gets you an access token. Make sure the header option or param option is selected in the box next to the access token.\n\nAlternatively, you can get a request token from https://developers.google.com/oauthplayground/. Add request token as header: \"Authorization: Bearer [OAUTH2_TOKEN]\". Attention: the token is preceded by the string \"Bearer \".", 3 | "module": "Generic", 4 | "category": "Google Cloud Platform", 5 | "speed": 200, 6 | "options": { 7 | "redirect_uri": "https://localhost", 8 | "resource": "/speech:recognize", 9 | "objectid": "alternatives.0.transcript", 10 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 11 | "token_uri": "https://accounts.google.com/o/oauth2/token", 12 | "basepath": "https://speech.googleapis.com/v1", 13 | "auth": "header", 14 | "querytype": "Generic:https://speech.googleapis.com/v1/speech:recognize", 15 | "headers": {}, 16 | "verb": "POST", 17 | "params": {}, 18 | "scope": "https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/devstorage.read_write https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/cloud-language", 19 | "nodedata": "results", 20 | "payload": "{\n\"config\": {\n \"encoding\":\"FLAC\",\n \"languageCode\": \"en-US\",\n \"enableWordTimeOffsets\": false,\n maxAlternatives:2\n },\n\"audio\":{\n \"content\":\"\"\n }\n}" 21 | }, 22 | "columns": [ 23 | "error.message", 24 | "alternatives.0.confidence", 25 | "alternatives.0.transcript" 26 | ], 27 | "name": "Synchronous speech recognition" 28 | } 29 | -------------------------------------------------------------------------------- /presets/Generic-Upload_Google_Cloud_Storage_buckets.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Select the folder where the files reside (below the payload field). In the dialog make sure to check the option \"Add files as nodes\". Alternatively, you can manually add filenames as nodes, e.g. \"amazing-speech.flac\".\n\nSet parameter to the name of you bucket, e.g. \"myfancyusername-myfancybucket\".\n\nSet the correct content-type for the file. See https://en.wikipedia.org/wiki/Media_type for information about content types.\n\nCheck the result at https://console.cloud.google.com/storage/browser.\n\n__Authorization notes__\n\nYou need to register an app at Google Cloud Platform: https://console.cloud.google.com. Enable Cloud Storage JSON API and create OAuth2-Credentials. In Facepager, open the authorization settings (next to Login-button) and provide Client ID and Client Secret fields. Then you are ready to login with Facepager, which gets you an access token. Make sure the header option or param option is selected in the box next to the access token.\n\nAlternatively, you can get a request token from https://developers.google.com/oauthplayground/. Add request token as header: \"Authorization: Bearer [OAUTH2_TOKEN]\". Attention: the token is preceded by the string \"Bearer \".\n", 3 | "module": "Generic", 4 | "category": "Google Cloud Platform", 5 | "speed": 200, 6 | "options": { 7 | "redirect_uri": "https://localhost", 8 | "resource": "/b//o", 9 | "objectid": "name", 10 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 11 | "token_uri": "https://accounts.google.com/o/oauth2/token", 12 | "basepath": "https://www.googleapis.com/upload/storage/v1", 13 | "auth": "header", 14 | "querytype": "Generic:https://www.googleapis.com/upload/storage/v1/b//o", 15 | "headers": { 16 | "Content-Type": "audio/x-flac" 17 | }, 18 | "verb": "POST", 19 | "params": { 20 | "uploadType": "media", 21 | "": "myfancyusername-myfancybucket", 22 | "name": "" 23 | }, 24 | "scope": "https://www.googleapis.com/auth/devstorage.full_control https://www.googleapis.com/auth/devstorage.read_write https://www.googleapis.com/auth/cloud-platform https://www.googleapis.com/auth/cloud-language", 25 | "nodedata": "", 26 | "payload": "" 27 | }, 28 | "columns": [ 29 | "error.message", 30 | "id", 31 | "selfLink", 32 | "updated" 33 | ], 34 | "name": "Upload files to Google Cloud Storage bucket" 35 | } 36 | -------------------------------------------------------------------------------- /presets/Generic-Wikipedia_Full_text_search.3_9.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Add search terms as nodes, for example \"Election\", then fetch data from the English Wikipedia. You can change the basepath to query other languages. For example, https://de.wikipedia.org/w/api.php uses the German Wikipedia. \n\nThis gives you the first three results. Change the srlimit-parameter to get more or less. \n\nPossible parameters of the search function: https://www.mediawiki.org/w/api.php?action=help&modules=query%2Bsearch\n\nGeneral introduction to Wikipedia API: https://www.mediawiki.org/wiki/API:Main_page\n\nThe search is part of the action=query module of Wikipedia: https://www.mediawiki.org/wiki/API:Query\n", 3 | "module": "Generic", 4 | "category": "Wikipedia", 5 | "speed": 200, 6 | "options": { 7 | "resource": "", 8 | "objectid": "pageid", 9 | "basepath": "https://en.wikipedia.org/w/api.php", 10 | "headers": {}, 11 | "nodedata": "query.search", 12 | "params": { 13 | "srinfo": "totalhits|suggestion|rewrittenquery", 14 | "srlimit": "3", 15 | "format": "json", 16 | "srsearch": "", 17 | "list": "search", 18 | "srwhat": "text", 19 | "action": "query" 20 | } 21 | }, 22 | "columns": [ 23 | "pageid", 24 | "title", 25 | "timestamp", 26 | "wordcount", 27 | "size", 28 | "query.searchinfo.totalhits" 29 | ], 30 | "name": "Wikipedia: Full text search" 31 | } -------------------------------------------------------------------------------- /presets/Generic-Wikipedia_Metadata.3_9.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "Add titles of articles as nodes, for example \"Election\", then fetch data from the English Wikipedia. You can change the basepath to query other languages. For example, https://de.wikipedia.org/w/api.php uses the German Wikipedia.\n\nThe metadata includes page views and date of the last revision. Redirects are automatically resolved.\n\nGeneral introduction to Wikipedia API: https://www.mediawiki.org/wiki/API:Main_page\n\nThe action=query module of Wikipedia is probably what you want to use: https://www.mediawiki.org/wiki/API:Query\n\nPossible values of the prop-parameter: https://www.mediawiki.org/w/api.php?action=help&modules=query\n\n", 3 | "module": "Generic", 4 | "category": "Wikipedia", 5 | "speed": 200, 6 | "options": { 7 | "resource": "", 8 | "objectid": "query.pages.*.pageid", 9 | "basepath": "https://en.wikipedia.org/w/api.php", 10 | "headers": {}, 11 | "params": { 12 | "redirects": "1", 13 | "format": "json", 14 | "rvprop": "timestamp|user|userid", 15 | "prop": "info|pageviews|revisions", 16 | "titles": "", 17 | "pvipdays": "7", 18 | "action": "query" 19 | }, 20 | "nodedata": null 21 | }, 22 | "columns": [ 23 | "query.redirects.0.from", 24 | "query.redirects.0.to", 25 | "query.pages.*.title", 26 | "query.pages.*.pageid", 27 | "query.pages.*.touched", 28 | "query.pages.*.pageviews.*", 29 | "query.pages.*.length", 30 | "query.pages.*.pagelanguage", 31 | "query.pages.*.lastrevid", 32 | "query.pages.*.revisions.*.timestamp", 33 | "query.pages.*.revisions.*.userid", 34 | "query.pages.*.revisions.*.user" 35 | ], 36 | "name": "Wikipedia: Meta data about articles" 37 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-Culture_Knowledge_Graph.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Culture Knowledge Graph", 3 | "category": "Knowledge Graph", 4 | "type": "pipeline", 5 | "pipeline": null, 6 | "description": "This pipeline demonstrates how to run presets one after another. Add a year (e.g. 1851) as a seed node, then click \"Run Pipeline\". \n\n1. In the first step, GND IDs are retrieved from the Culture Knowledge Graph using the first preset in this pipeline. \n\n2. After GND IDs were retrieved, a second preset is used to query names from the Culture Knowledge graph. \n\nSee the presets for further information. In the Getting Started with Culture Knowledge Graph you fin a walk through the presets. \n", 7 | "module": null, 8 | "options": null, 9 | "speed": null, 10 | "saveheaders": null, 11 | "timeout": null, 12 | "maxsize": null, 13 | "columns": null 14 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-DBPedia_Scientists_who_have_birhtday_today.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DBPedia: Scientists who have birhtday today", 3 | "category": "Knowledge Graph", 4 | "description": "*This preset requires at least Facepager v4.6.* \n\nGet the names, descriptions, birthdays and if available thumbnail images of scientists who have birthday today. The results are limited to English and to the 10 oldest scientists. \n\nAdd an arbitrary node and fetch data. The node name doesn't matter as it is not used in the query. Suggestion: add dates as node names and adjust the query to get scientists who have birthday at a specific date.\n\nDBPedia is like Wikidata a semantic web version of Wikipedia. On the semantic web data is organizied according to the Resource Decription Framework (RDF). The preset retrieves data from the SPARQL endpoint of DBPedia. DBPedia mainly consists of content scrapted from the info boxes of Wikipedia. With limiting to english names and descriptions the results should reflect the English Wikipedia.\n\nSee https://wiki.dbpedia.org/ for details about DBPedia. If you are not familiar with semantic web technology consider reading Wikipedia articles about the Resource Description Framework and about SPARQL.\n\nThe example query was taken from an OpenHPI course: https://open.hpi.de/courses/semanticweb2017/\n\nPay attention on how the SPARQL query is formulated. Since angle brackets signify placeholders in Facepager they have to be masked with a backslash. If you need a backslash in your query, mask it by another backslash. To use the name of a node or data of previously fetched nodes in your query user placeholders (see the Facepager help).\n", 5 | "module": "SPARQL", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "http://dbpedia.org/sparql", 13 | "extension": "", 14 | "format": "json", 15 | "headers": { 16 | "User-Agent": "FACEPAGERBOT/4.6 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.6" 17 | }, 18 | "key_paging": null, 19 | "nodedata": "results.bindings", 20 | "objectid": null, 21 | "offset_start": 1, 22 | "offset_step": 1, 23 | "paging_stop": null, 24 | "paging_type": null, 25 | "param_paging": null, 26 | "params": { 27 | "format": "json", 28 | "query": "PREFIX rdf: \\\nPREFIX dbo: \\\nPREFIX rdfs: \\\nPREFIX dc: \\\n\nSelect distinct ?birthdate ?thumbnail ?scientist ?name ?description WHERE {\n?scientist rdf:type dbo:Scientist ;\n dbo:birthDate ?birthdate ;\n rdfs:label ?name ;\n rdfs:comment ?description \n FILTER ((lang(?name)=\"en\")&&(lang(?description)=\"en\")&&(STRLEN(STR(?birthdate))>6)&&(SUBSTR(STR(?birthdate),6)=SUBSTR(STR(bif:curdate('')),6))) .\n OPTIONAL { ?scientist dbo:thumbnail ?thumbnail . }\n} ORDER BY ?birthdate LIMIT 10" 29 | }, 30 | "redirect_uri": "", 31 | "resource": "", 32 | "token_uri": "" 33 | }, 34 | "speed": 200, 35 | "saveheaders": false, 36 | "timeout": 15, 37 | "maxsize": 5, 38 | "columns": [ 39 | "name.value", 40 | "description.value", 41 | "birthdate.value", 42 | "thumbnail.value" 43 | ] 44 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-Getty_Fetch_triples_of_geographical_names.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Getty: Fetch triples of geographical names", 3 | "category": "Knowledge Graph", 4 | "type": "preset", 5 | "pipeline": null, 6 | "description": "This preset fetches all triples (turtle) associated with one or more entries from the [Getty Thesaurus of Geographic Names Online (TGN)](https://www.getty.edu/research/tools/vocabularies/tgn/). To start add a Getty ID as seed node, e.g., \"7005903\" for triples associated with Milan.\n\nSee our [Getting Started with Triple Parsing](https://github.com/strohne/Facepager/wiki/Getting-Started-with-Triple-Parsing) for a full walk through this preset.\n", 7 | "module": "Generic", 8 | "options": { 9 | "auth": "disable", 10 | "auth_prefix": "", 11 | "auth_tokenname": "", 12 | "auth_type": "Disable", 13 | "auth_uri": "", 14 | "basepath": "https://vocab.getty.edu/tgn/.ttl", 15 | "extension": "", 16 | "fileext": "", 17 | "filename": "", 18 | "format": "ttl", 19 | "headers": {}, 20 | "key_paging": null, 21 | "nodedata": "triples", 22 | "objectid": null, 23 | "pages": 1, 24 | "paging_stop": null, 25 | "paging_type": "key", 26 | "param_paging": null, 27 | "params": {}, 28 | "proxy": "", 29 | "redirect_uri": "", 30 | "resource": "", 31 | "scope": "", 32 | "token_uri": "", 33 | "verb": "GET" 34 | }, 35 | "speed": 200, 36 | "saveheaders": false, 37 | "timeout": 15, 38 | "maxsize": 5, 39 | "columns": [ 40 | "subject", 41 | "predicate", 42 | "object" 43 | ] 44 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-NFDI4Culture_1_Get_GND_IDs_of_Ferdinand_Gregorovius_s_addressees.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "1 Get GND IDs of Ferdinand Gregorovius's addressees", 3 | "category": "Knowledge Graph", 4 | "pipeline" : "Culture Knowledge Graph", 5 | "description": "*This preset requires at least Facepager v4.6.* \n\nThis preset retrieves GND IDs of Ferdinand Gregorovius' addressees from the Culture Knowledge Graph. To start add a year (e.g. 1851) as a seed node. Then fetch data. \n\nAt the heart of the preset lies a SPARQL query. Making an effort to learn the basics of SPARQL's syntax will allow you to literally understand any such query and, more crucially, let's you write your own queries, thus empowering you to answer (research) questions of your interest. A great resources to start with is, for example, [Wikidata's introduction to SPARQL](https://www.wikidata.org/wiki/Wikidata:SPARQL_tutorial). For now, it will suffice to understand that the query returns the url, label, and date of any letter Gregorovius wrote during the specified year as well as the GND IDs of all persons addressed or mention in each letter. \n\nSee our [Getting Started with Culture Knowledge Graph](https://github.com/strohne/Facepager/wiki/Getting-Started-with-KnowledgeGraph) for a walk through this preset. \n\n**DISCLAIMER:** The Culture Knowledge Graph is work in progress. Currently, only a handful of datasets are accessible through its SPARQL endpoint using the NFDI4Culture Ontology. The User-Policy and guidelines of the Culture Knowledge Graph are being worked on as well. For the time being we advise you to be mindful of potential query limits.", 6 | "module": "SPARQL", 7 | "options": { 8 | "auth": "disable", 9 | "auth_prefix": "", 10 | "auth_tokenname": "", 11 | "auth_type": "Disable", 12 | "auth_uri": "", 13 | "basepath": "https://nfdi4culture.de/sparql", 14 | "extension": "", 15 | "format": "json", 16 | "headers": { 17 | "User-Agent": "FACEPAGERBOT/4.6 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.6" 18 | }, 19 | "key_paging": null, 20 | "nodedata": "results.bindings", 21 | "objectid": "gnd_id.value", 22 | 23 | "offset_start": 1, 24 | "offset_step": 1, 25 | "paging_stop": null, 26 | "paging_type": null, 27 | "param_paging": null, 28 | "params": { 29 | "format": "json", 30 | "query": "PREFIX cto: \\\nPREFIX schema: \\\nPREFIX rdf: \\\nPREFIX rdfs: \\\nPREFIX n4c: \\\n\nSELECT ?letter ?letterLabel ?date ?gnd_id\nWHERE {\n n4c:E5378 schema:dataFeedElement/schema:item ?letter .\n ?letter rdfs:label ?letterLabel ;\n cto:creationDate ?date ;\n cto:relatedPerson ?gnd_url .\n FILTER(YEAR(?date) = ).\n BIND(REPLACE(STR(?gnd_url), \".*://.*/(.*?)\", \"$1\") AS ?gnd_id)\n} LIMIT 100" 31 | }, 32 | "redirect_uri": "", 33 | "resource": "", 34 | "token_uri": "" 35 | }, 36 | "speed": 200, 37 | "saveheaders": false, 38 | "timeout": 15, 39 | "maxsize": 5, 40 | "columns": [ 41 | "date.value", 42 | "gnd_id.value", 43 | "letterLabel.value" 44 | ] 45 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-NFDI4Culture_2_Translate_GND_ID_to_a_person_s_name.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "2 Translate GND ID to a person's name (Entity Facts API; German National Library)", 3 | "category": "Knowledge Graph", 4 | "pipeline" : "Culture Knowledge Graph", 5 | "description": "This preset builds upon the first NFDI4Culture preset. It translates the resulting GND IDs of Ferdinand Gregorovius' addressees into the persons' preferred names (from a historic perspective) for further use. To use it as a standalone, start by adding a seed note containing any GND ID (e.g., 118623222 will fetch information about J.R.R. Tolkien). Generally, you can use this preset to obtain any data accessible through the [Entity Facts API](https://www.dnb.de/EN/Professionell/Metadatendienste/Datenbezug/Entity-Facts/entityFacts_node.html#doc250704bodyText8) of the German National Library. To get an overview of the possibilities, please, refer to the [library's online sheet](https://wiki.dnb.de/pages/viewpage.action?pageId=134055670). \n\nSee our [Getting Started with Culture Knowledge Graph](https://github.com/strohne/Facepager/wiki/Getting-Started-with-CultureKnowledgeGraph) for a full walk through this preset. \n\nPlease, mind the German National Library's [Terms of Use](https://www.dnb.de/EN/Professionell/Metadatendienste/Datenbezug/geschaeftsmodell.html).", 6 | "module": "Generic", 7 | "options": { 8 | "auth": "disable", 9 | "auth_prefix": "", 10 | "auth_tokenname": "", 11 | "auth_type": "Disable", 12 | "auth_uri": "", 13 | "basepath": "https://hub.culturegraph.org", 14 | "extension": "", 15 | "fileext": "", 16 | "filename": "", 17 | "format": "json", 18 | "headers": {}, 19 | "nodedata": null, 20 | "objectid": null, 21 | "offset_start": 0, 22 | "offset_step": 1, 23 | "pages": 1, 24 | "paging_type": "count", 25 | "param_paging": null, 26 | "params": { 27 | "": "", 28 | "key": "preferredName" 29 | }, 30 | "proxy": "", 31 | "redirect_uri": "", 32 | "resource": "/entityfacts/", 33 | "scope": "", 34 | "token_uri": "", 35 | "verb": "GET" 36 | }, 37 | "speed": 200, 38 | "saveheaders": false, 39 | "timeout": 15, 40 | "maxsize": 5, 41 | "columns": [ 42 | "date.value", 43 | "gnd_id.value", 44 | "preferredName", 45 | "letterLabel.value" 46 | ] 47 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-Wikidata_Get_gender_of_solo_artist_band_members.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wikidata: Get gender of solo artist/band members", 3 | "category": "Knowledge Graph", 4 | "description": "*This preset requires at least Facepager v4.6.* \n\nThis preset fetches a list of musical artists and their sex or gender as provided by Wikidata. To start add the English name (label) of an arbitrary musical solo artist or band as seed node, e.g. \"Peter Maffay\" or \"Coldplay\". Then fetch data. \n\nThe query automatically extracts names of band members on the basis of the group's name, e.g. \"Coldplay\". \n\nSee our [Getting Started with Wikidata (artists gender)](https://github.com/strohne/Facepager/wiki/Getting-Started-with-Wikidata-(artists-gender)) for a full walk through this preset.\n\nSee https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nAlways ensure that you comply with Wikidata's User-Agent policy and be mindful of potential query limits.\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.\n\nPagination is handled by the placeholder.", 5 | "module": "SPARQL", 6 | "options": { 7 | "basepath": "https://query.wikidata.org/sparql", 8 | "extension": "", 9 | "paging_type": null, 10 | "key_paging": null, 11 | "paging_stop": null, 12 | "param_paging": null, 13 | "offset_start": 1, 14 | "offset_step": 1, 15 | "nodedata": "results.bindings", 16 | "objectid": null, 17 | "auth_type": "Disable", 18 | "auth_uri": "", 19 | "redirect_uri": "", 20 | "token_uri": "", 21 | "auth": "disable", 22 | "auth_prefix": "", 23 | "auth_tokenname": "", 24 | "resource": "", 25 | "format": "json", 26 | "headers": { 27 | "User-Agent": "FACEPAGERBOT/4.6 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.6" 28 | }, 29 | "params": { 30 | "query": "SELECT distinct ?entityLabel ?genderLabel WHERE {\n \n # Search for solo artist or band\n ?entity rdfs:label \"\"@en.\n \n # Check if ?entity is solo artist or band (musical group)\n {\n # If solo artist, get sex or gender\n ?entity wdt:P31 wd:Q5. \n ?entity wdt:P21 ?gender. \n }\n UNION\n {\n # If band (musical group), first get all members then their sex or gender \n ?entity wdt:P31 wd:Q215380. \n ?entity wdt:P527 ?member. \n ?member wdt:P21 ?gender. \n ?member rdfs:label ?entityLabel.\n FILTER(LANG(?entityLabel) = \"en\")\n }\n SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". }\n}", 31 | "format": "json" 32 | } 33 | }, 34 | "speed": 200, 35 | "saveheaders": false, 36 | "timeout": 15, 37 | "maxsize": 5, 38 | "columns": [ 39 | "entityLabel.value", 40 | "genderLabel.value" 41 | ] 42 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-Wikidata_Get_list_of_biblical_characters.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wikidata: Get list of biblical characters", 3 | "category": "Knowledge Graph", 4 | "description": "*This preset requires at least Facepager v4.6.* \n\nThis preset gets a list of biblical characters, along with the sections in which they occur, from WikiData. Add an arbitrary seed node and fetch data.\n\nSee https://www.wikidata.org/wiki/Wikidata:Lists/List_of_biblical_characters\n\nYou can try out different query on https://query.wikidata.org/ (paste the value of the query parameter to get started).\n\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, if your query contains angle brackets, escape them with a backslash. Pagination is handled by the placeholder.", 5 | "module": "SPARQL", 6 | "options": { 7 | "basepath": "https://query.wikidata.org/sparql", 8 | "extension": "", 9 | "paging_type": "count", 10 | "key_paging": null, 11 | "paging_stop": null, 12 | "param_paging": "", 13 | "offset_start": 0, 14 | "offset_step": 10, 15 | "nodedata": "results.bindings", 16 | "objectid": "figure.value", 17 | "auth_type": "Disable", 18 | "auth_uri": "", 19 | "redirect_uri": "", 20 | "token_uri": "", 21 | "auth": "disable", 22 | "auth_prefix": "", 23 | "auth_tokenname": "", 24 | "resource": "", 25 | "format": "json", 26 | "headers": { 27 | "User-Agent": "FACEPAGERBOT/4.6 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.6" 28 | }, 29 | "params": { 30 | "query": "SELECT DISTINCT ?figure ?figureLabel ?figureDescription ?section ?sectionLabel ?sectionDescription WHERE {\n\n ?figure wdt:P31 wd:Q12405827 .\n ?figure wdt:P1441 ?section .\n wd:Q1845 (wdt:P527)* ?chapter . \n wd:Q1845 (wdt:P527)* ?chapter .\n \n SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\".} \n}\nLIMIT 10", 31 | "format": "json" 32 | } 33 | }, 34 | "speed": 200, 35 | "saveheaders": false, 36 | "timeout": 15, 37 | "maxsize": 5, 38 | "columns": [ 39 | "figureLabel.value", 40 | "figureDescription.value", 41 | "sectionLabel.value", 42 | "sectionDescription.value", 43 | "figure.value", 44 | "section.value" 45 | ] 46 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-Wikidata_Which_politicans_did_not_die_on_natural_causes.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wikidata: Which politicans did not die on natural causes?", 3 | "category": "Knowledge Graph", 4 | "description": "*This preset requires at least Facepager v4.6.* \n\nThis preset fetches a list of politicians from WikiData that did not die on natural causes. Add an arbitrary node as seed node, e.g. \"politicians\".\n\nSee https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.\n\nPagination is handled by the placeholder.", 5 | "module": "SPARQL", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://query.wikidata.org/sparql", 13 | "extension": "", 14 | "format": "json", 15 | "headers": { 16 | "User-Agent": "FACEPAGERBOT/4.6 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.6" 17 | }, 18 | "key_paging": null, 19 | "nodedata": "results.bindings", 20 | "objectid": "politician.value", 21 | "offset_start": 1, 22 | "offset_step": 1, 23 | "paging_stop": null, 24 | "paging_type": null, 25 | "param_paging": null, 26 | "params": { 27 | "format": "json", 28 | "query": "PREFIX wd: \\ \nPREFIX wdt: \\\n\nSELECT DISTINCT ?politician ?cause ?politicianLabel ?causeLabel WHERE {\n ?politician wdt:P106 wd:Q82955 . # find items that have \"occupation (P106): politician (Q82955)\"\n ?politician wdt:P509|wdt:P1196 ?cause . # with a P509 (cause of death) or P1196 (manner of death) claim\n ?cause wdt:P279* wd:Q149086 . # ... where the cause is a subclass of (P279*) homicide (Q149086)\n # ?politician wdt:P39 wd:Q11696 . # Uncomment this line to include only U.S. Presidents \n SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\" }\n} LIMIT 10" 29 | }, 30 | "redirect_uri": "", 31 | "resource": "", 32 | "token_uri": "" 33 | }, 34 | "speed": 200, 35 | "saveheaders": false, 36 | "timeout": 15, 37 | "maxsize": 5, 38 | "columns": [ 39 | "politicianLabel.value", 40 | "causeLabel.value", 41 | "politician.value", 42 | "cause.value" 43 | ] 44 | } -------------------------------------------------------------------------------- /presets/Knowledge_Graph-Wikidata_Writers_Network.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wikidata: Writers Network", 3 | "category": "Knowledge Graph", 4 | "description": "*This preset requires at least Facepager v4.6.* \n\nThis preset fetches a list of German writers as well as cities relevant to them based on literary movements. To start add a Q-identifier of an arbitrary literary movement as seed node, e.g. \"Q37068\" for romanticism. Then fetch data. \n\nSee our [Getting Started with Wikidata (writers network)](https://github.com/strohne/Facepager/wiki/Getting-Started-with-Wikidata-(writers-network)) for a full walk through this preset.\n\nCheck https://www.wikidata.org/ for further information. You can try out different queries on https://query.wikidata.org/ (paste the value of the query parameter).\n\nAlways ensure that you comply with [Wikidata's User-Agent policy](https://foundation.wikimedia.org/wiki/Policy:User-Agent_policy) and be mindful of potential [query limits](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#Query_limits).\n\nPlease be aware that expressions in angle brackets are handled as placeholder by Facepager. Thus, the brackets of the prefixes have to be escaped with a backslash.\n\nPagination is handled by the placeholder.", 5 | "module": "SPARQL", 6 | "options": { 7 | "basepath": "https://query.wikidata.org/sparql", 8 | "extension": "", 9 | "paging_type": null, 10 | "key_paging": null, 11 | "paging_stop": null, 12 | "param_paging": null, 13 | "offset_start": 1, 14 | "offset_step": 1, 15 | "nodedata": "results.bindings", 16 | "objectid": null, 17 | "auth_type": "Disable", 18 | "auth_uri": "", 19 | "redirect_uri": "", 20 | "token_uri": "", 21 | "auth": "disable", 22 | "auth_prefix": "", 23 | "auth_tokenname": "", 24 | "resource": "", 25 | "format": "json", 26 | "headers": { 27 | "User-Agent": "FACEPAGERBOT/4.6 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.6" 28 | }, 29 | "params": { 30 | "query": "SELECT DISTINCT ?writer ?writerLabel ?place ?placeLabel ?movement\nWHERE\n{\n # get all German writers of a literary movement\n ?writer wdt:P135 wd:;\n wdt:P106 wd:Q36180;\n wdt:P6886 wd:Q188.\n \n wd: rdfs:label ?movement.\n filter(lang(?movement) = \"en\")\n \n # relevant places\n ?writer wdt:P19|wdt:P20|(wdt:P69/wdt:P131)|(wdt:P108/wdt:P131) ?place.\n\n SERVICE wikibase:label { bd:serviceParam wikibase:language \"[AUTO_LANGUAGE],en\". }\n \n}", 31 | "format": "json" 32 | } 33 | }, 34 | "speed": 200, 35 | "saveheaders": false, 36 | "timeout": 15, 37 | "maxsize": 5, 38 | "columns": [ 39 | "writerLabel.value", 40 | "placeLabel.value", 41 | "movement.value" 42 | ] 43 | } -------------------------------------------------------------------------------- /presets/Mastodon-Get_public_timeline.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Get public timeline", 3 | "category": "Mastodon", 4 | "description": "This is a simple preset that serves as a starting point to dive into the mastodon fediverse.\n\nFor retrieving data from Mastodon, you first have to decide for an instance such as sciences.social or mastodon.social. Adjust the base path accordingly.\n\nTo get the public timeline, add \"public\" as a seed not and fetch the data. Each response should give you up to 10 posts. You can get more posts by increasing the maximum pages (requires Facepager 4.6 which will be published soon).\n\nFor further information, see https://docs.joinmastodon.org/client/public/", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "https://sciences.social/api/v1", 8 | "resource": "/timelines/", 9 | "params": { 10 | "limit": "10" 11 | }, 12 | "extension": "", 13 | "headers": {}, 14 | "verb": "GET", 15 | "format": "json", 16 | "filename": "", 17 | "fileext": "", 18 | "pages": 10, 19 | "paging_type": "key", 20 | "key_paging": "*.id|last", 21 | "paging_stop": null, 22 | "param_paging": "max_id", 23 | "nodedata": null, 24 | "objectid": "id", 25 | "scope": "identity read mysubreddits", 26 | "proxy": "", 27 | "auth_type": "Disable", 28 | "auth_uri": "", 29 | "redirect_uri": "", 30 | "token_uri": "", 31 | "auth": "disable", 32 | "auth_tokenname": "", 33 | "auth_prefix": "" 34 | }, 35 | "speed": 200, 36 | "saveheaders": false, 37 | "timeout": 15, 38 | "maxsize": 5, 39 | "columns": [ 40 | "created_at", 41 | "account.acct", 42 | "content", 43 | "replies_count", 44 | "reblogs_count", 45 | "favourites_count", 46 | "uri" 47 | ] 48 | } -------------------------------------------------------------------------------- /presets/Reddit-1_Get_posts_in_a_subreddit.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "1_Get posts in a subreddit", 3 | "category": "Reddit", 4 | "description": "Add the name of the subreddit as seed node, e.g. \"funnycats\" for https://www.reddit.com/r/funnycats/\n\n\nSee https://github.com/reddit-archive/reddit/wiki/JSON for an overview about the fields. See https://www.reddit.com/dev/api for information about the API.\n\nSee https://github.com/reddit-archive/reddit/wiki/API for information about setting the user agent.\n", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "https://www.reddit.com/api/v1/authorize?state=mystate&duration=temporary", 12 | "basepath": "https://www.reddit.com", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": { 18 | "User-Agent": "MyApp/0.1 by Meself" 19 | }, 20 | "key_paging": "data.after", 21 | "nodedata": "data.children", 22 | "objectid": "data.id", 23 | "pages": 1, 24 | "paging_stop": null, 25 | "paging_type": "key", 26 | "param_paging": "after", 27 | "params": {}, 28 | "proxy": "", 29 | "redirect_uri": "https://localhost", 30 | "resource": "/r//.json", 31 | "scope": "identity read mysubreddits", 32 | "token_uri": "https://www.reddit.com/api/v1/access_token", 33 | "verb": "GET" 34 | }, 35 | "speed": 200, 36 | "saveheaders": false, 37 | "timeout": 15, 38 | "maxsize": 5, 39 | "columns": [ 40 | "kind", 41 | "data.title", 42 | "data.created_utc|timestamp", 43 | "data.num_comments", 44 | "data.ups", 45 | "data.downs", 46 | "data.url", 47 | "data.is_video", 48 | "data.name", 49 | "data.subreddit_id", 50 | "data.subreddit", 51 | "data.permalink" 52 | ] 53 | } -------------------------------------------------------------------------------- /presets/Reddit-2_Get_comments.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "2_Get comments", 3 | "category": "Reddit", 4 | "description": "First fetch posts, then comments. \n\nThis preset is a first start and still is experimental. Feel free to improve it.\n\nSee the posts preset for further information about the API.", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "https://www.reddit.com/api/v1/authorize?state=mystate&duration=temporary", 12 | "basepath": "https://www.reddit.com", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": { 18 | "User-Agent": "MyApp/0.1 by Meself" 19 | }, 20 | "key_paging": "data.after", 21 | "nodedata": "1.data.children", 22 | "objectid": "data.id", 23 | "pages": 1, 24 | "paging_stop": null, 25 | "paging_type": "key", 26 | "param_paging": "after", 27 | "params": { 28 | "count": "5" 29 | }, 30 | "proxy": "", 31 | "redirect_uri": "https://localhost", 32 | "resource": "/r//comments//.json", 33 | "scope": "identity read mysubreddits", 34 | "token_uri": "https://www.reddit.com/api/v1/access_token", 35 | "verb": "GET" 36 | }, 37 | "speed": 200, 38 | "saveheaders": false, 39 | "timeout": 15, 40 | "maxsize": 5, 41 | "columns": [ 42 | "kind", 43 | "data.body", 44 | "data.created_utc|timestamp", 45 | "data.author", 46 | "data.ups", 47 | "data.downs", 48 | "data.score", 49 | "data.name", 50 | "data.subreddit_id", 51 | "data.subreddit", 52 | "data.permalink" 53 | ] 54 | } -------------------------------------------------------------------------------- /presets/Reddit-3_Get_posts_in_a_subreddit_via_pushshift_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "3 Get posts in a subreddit (via pushshift)", 3 | "category": "Reddit", 4 | "description": "This preset fetches submissions from a subreddit. Add the name of the subreddit as seed node and you'll get the most recent submissions from the particular subreddits.\n\n**Parameter Options**\n\n1. Number of results: One request gives you back the 250 most recent submissions matching the query. Increase the maximum pages parameter to get more.\n\n2. Order of results: With the \"sort_type\" parameter you can change the result order between \"score\", \"num_comments\" and \"created_utc\" (the latter is the default). Set the \"sort\" parameter to \"asc\" in order to sort the results ascending instead of the default descending. \n\n3. Filter by date: To filter by date use the parameters \"after\" and \"before\". You can use an epoch timestamp (e.g. \"1669040608\") or a number followed by the characters s,m,h,d, which stand for second, minute, hour and day (e.g. \"30d\") to limit the timeframe. \n\nTake a look at the documentation on https://github.com/pushshift/api#search-parameters-for-submissions for further options.\n", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "https://www.reddit.com/api/v1/authorize?state=mystate&duration=temporary", 12 | "basepath": "https://api.pushshift.io/reddit", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": {}, 18 | "key_paging": "data.249.created_utc", 19 | "nodedata": "data", 20 | "objectid": "id", 21 | "pages": 1, 22 | "paging_stop": null, 23 | "paging_type": "key", 24 | "param_paging": "before", 25 | "params": { 26 | "size": "250", 27 | "subreddit": "" 28 | }, 29 | "proxy": "", 30 | "redirect_uri": "https://localhost", 31 | "resource": "/search/submission/", 32 | "scope": "identity read mysubreddits", 33 | "token_uri": "https://www.reddit.com/api/v1/access_token", 34 | "verb": "GET" 35 | }, 36 | "speed": 200, 37 | "saveheaders": false, 38 | "timeout": 15, 39 | "maxsize": 5, 40 | "columns": [ 41 | "author", 42 | "author_fullname", 43 | "created_utc", 44 | "domain", 45 | "full_link", 46 | "id", 47 | "num_comments", 48 | "permalink", 49 | "selftext", 50 | "subreddit", 51 | "subreddit_id", 52 | "subreddit_subscribers", 53 | "subreddit_type", 54 | "title", 55 | "url" 56 | ] 57 | } -------------------------------------------------------------------------------- /presets/Reddit-4_Get_comments_via_pushshift_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "4 Get comments (via pushshift)", 3 | "category": "Reddit", 4 | "description": "Use the preset to retrieve all comments of a submission (that's how posts are called on reddit). \n\nAdd a submission ID as seed node. You can fetch submission IDs with the posts presets.\n\n**Documentation**\nhttps://github.com/pushshift/api#get-all-comment-ids-for-a-particular-submission ", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "https://www.reddit.com/api/v1/authorize?state=mystate&duration=temporary", 12 | "basepath": "https://api.pushshift.io/reddit", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "json", 17 | "headers": {}, 18 | "key_paging": "data.249.created_utc", 19 | "nodedata": "data", 20 | "objectid": "id", 21 | "pages": 10, 22 | "paging_stop": null, 23 | "paging_type": "key", 24 | "param_paging": "before", 25 | "params": { 26 | "link_id": "", 27 | "q": "*", 28 | "size": "250" 29 | }, 30 | "proxy": "", 31 | "redirect_uri": "https://localhost", 32 | "resource": "/search/comment/", 33 | "scope": "identity read mysubreddits", 34 | "token_uri": "https://www.reddit.com/api/v1/access_token", 35 | "verb": "GET" 36 | }, 37 | "speed": 200, 38 | "saveheaders": false, 39 | "timeout": 15, 40 | "maxsize": 5, 41 | "columns": [ 42 | "created_utc", 43 | "body", 44 | "title", 45 | "author", 46 | "author_fullname", 47 | "created_utc", 48 | "domain", 49 | "full_link", 50 | "id", 51 | "num_comments", 52 | "permalink", 53 | "selftext", 54 | "subreddit", 55 | "subreddit_id", 56 | "subreddit_subscribers", 57 | "subreddit_type", 58 | "url" 59 | ] 60 | } -------------------------------------------------------------------------------- /presets/Reddit-5_Search_submissions_via_pushshift_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "5 Search submissions (via pushshift)", 3 | "category": "Reddit", 4 | "description": "Find submissions: Add a search term as seed node and you'll get the most recent submissions with the word in the title or selftext. \n\n**Parameter Options**\n\n1. Number of results: One request gives you back the 250 most recent submissions matching the query. Increase the maximum pages parameter to get more.\n\n2. Order of results: With the \"sort_type\" parameter you can change the result order between \"score\", \"num_comments\" and \"created_utc\" (the latter is the default). Set the \"sort\" parameter to \"asc\" in order to sort the results ascending instead of the default descending. \n\n3. Filter by date: To filter by date use the parameters \"after\" and \"before\". You can use an epoch timestamp (e.g. \"1669040608\") or a number followed by the characters s,m,h,d, which stand for second, minute, hour and day (e.g. \"30d\") to limit the timeframe. \n\n4. Restrict to a specific subreddit: By using the \"subreddit\" parameter you you can restrict your search to a specific subreddit e.g. \"askscience\".\n\n\nTake a look at the documentation on https://github.com/pushshift/api#search-parameters-for-submissions for further parameter options.", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "https://api.pushshift.io/reddit", 8 | "resource": "/search/submission/", 9 | "params": { 10 | "q": "", 11 | "size": "250" 12 | }, 13 | "extension": "", 14 | "headers": {}, 15 | "verb": "GET", 16 | "format": "json", 17 | "filename": "", 18 | "fileext": "", 19 | "pages": 1, 20 | "paging_type": "key", 21 | "key_paging": "data.249.created_utc", 22 | "paging_stop": null, 23 | "param_paging": "before", 24 | "nodedata": "data", 25 | "objectid": "id", 26 | "scope": "identity read mysubreddits", 27 | "proxy": "", 28 | "auth_type": "Disable", 29 | "auth_uri": "https://www.reddit.com/api/v1/authorize?state=mystate&duration=temporary", 30 | "redirect_uri": "https://localhost", 31 | "token_uri": "https://www.reddit.com/api/v1/access_token", 32 | "auth": "disable", 33 | "auth_tokenname": "", 34 | "auth_prefix": "" 35 | }, 36 | "speed": 200, 37 | "saveheaders": false, 38 | "timeout": 15, 39 | "maxsize": 5, 40 | "columns": [ 41 | "author", 42 | "author_fullname", 43 | "created_utc", 44 | "domain", 45 | "full_link", 46 | "id", 47 | "num_comments", 48 | "permalink", 49 | "selftext", 50 | "subreddit", 51 | "subreddit_id", 52 | "subreddit_subscribers", 53 | "subreddit_type", 54 | "title", 55 | "url" 56 | ] 57 | } -------------------------------------------------------------------------------- /presets/Reddit-6_Search_comments_via_pushshift_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "6 Search comments (via pushshift)", 3 | "category": "Reddit", 4 | "description": "Find comments: Add any search term as seed node and you'll get the most recent comments with the search term in the body of the comment.\n\n**Parameter Options**\n\n1. Number of results: One request gives you back the 250 most recent submissions matching the query. Increase the maximum pages parameter to get more.\n\n2. Order of results: With the \"sort_type\" parameter you can change the result order between \"score\", \"num_comments\" and \"created_utc\" (the latter is the default). Set the \"sort\" parameter to \"asc\" in order to sort the results ascending instead of the default descending. \n\n3. Filter by date: To filter by date use the parameters \"after\" and \"before\". You can use an epoch timestamp (e.g. \"1669040608\") or a number followed by the characters s,m,h,d, which stand for second, minute, hour and day (e.g. \"30d\") to limit the timeframe. \n\n4. Restrict to a specific subreddit: By using the \"subreddit\" parameter you you can restrict your search to a specific subreddit e.g. \"askscience\".\n\nTake a look at the documentation on https://github.com/pushshift/api#search-parameters-for-comments for further parameter options.", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "https://api.pushshift.io/reddit", 8 | "resource": "/search/comment/", 9 | "params": { 10 | "q": "", 11 | "size": "250" 12 | }, 13 | "extension": "", 14 | "headers": {}, 15 | "verb": "GET", 16 | "format": "json", 17 | "filename": "", 18 | "fileext": "", 19 | "pages": 1, 20 | "paging_type": "key", 21 | "key_paging": "data.249.created_utc", 22 | "paging_stop": null, 23 | "param_paging": "before", 24 | "nodedata": "data", 25 | "objectid": "id", 26 | "scope": "identity read mysubreddits", 27 | "proxy": "", 28 | "auth_type": "Disable", 29 | "auth_uri": "https://www.reddit.com/api/v1/authorize?state=mystate&duration=temporary", 30 | "redirect_uri": "https://localhost", 31 | "token_uri": "https://www.reddit.com/api/v1/access_token", 32 | "auth": "disable", 33 | "auth_tokenname": "", 34 | "auth_prefix": "" 35 | }, 36 | "speed": 200, 37 | "saveheaders": false, 38 | "timeout": 15, 39 | "maxsize": 5, 40 | "columns": [ 41 | "author", 42 | "author_fullname", 43 | "body", 44 | "created_utc", 45 | "id", 46 | "link_id", 47 | "parent_id", 48 | "permalink", 49 | "retrieved_utc", 50 | "subreddit", 51 | "subreddit_id", 52 | "subreddit_name_prefixed", 53 | "subreddit_type" 54 | ] 55 | } -------------------------------------------------------------------------------- /presets/Scraping-BBC_Article_content.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BBC: Article content", 3 | "category": "Scraping", 4 | "description": "Add URL to a BBC article as seed node. You can get URLs with the search preset. This preset extracts the text blocks from the page\n\nSee the wiki for information about using CSS and XPath selectors with Facepager: https://github.com/strohne/Facepager/wiki/Webscraping", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "text", 17 | "headers": {}, 18 | "nodedata": "text|xpath://div[@data-component=\"text-block\"]", 19 | "objectid": "text|xpath:string()", 20 | "offset_start": 1, 21 | "offset_step": 1, 22 | "pages": 1, 23 | "paging_type": "count", 24 | "param_paging": null, 25 | "params": {}, 26 | "proxy": "", 27 | "redirect_uri": "", 28 | "resource": "", 29 | "scope": "", 30 | "token_uri": "", 31 | "verb": "GET" 32 | }, 33 | "speed": 200, 34 | "saveheaders": false, 35 | "timeout": 15, 36 | "columns": [] 37 | } -------------------------------------------------------------------------------- /presets/Scraping-BBC_Search_results.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BBC: Search results", 3 | "category": "Scraping", 4 | "description": "Add search terms as seed nodes and fetch data.\n\nThis preset uses the search of https://www.bbc.co.uk/search?q=disinformation+campaign to fetch a list of articles from the BBC news site. \n\nSee the wiki for information about using CSS and XPath selectors with Facepager: https://github.com/strohne/Facepager/wiki/Webscraping", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://www.bbc.co.uk", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "text", 17 | "headers": {}, 18 | "nodedata": "text|css:main ul li", 19 | "objectid": "text|xpath://a/@href", 20 | "offset_start": 1, 21 | "offset_step": 1, 22 | "pages": 1, 23 | "paging_type": "count", 24 | "param_paging": "page", 25 | "params": { 26 | "q": "" 27 | }, 28 | "proxy": "", 29 | "redirect_uri": "", 30 | "resource": "/search", 31 | "scope": "", 32 | "token_uri": "", 33 | "verb": "GET" 34 | }, 35 | "speed": 200, 36 | "saveheaders": false, 37 | "timeout": 15, 38 | "columns": [ 39 | "headline=text|css:a|xpath:string()", 40 | "link=text|xpath://a/@href", 41 | "text|xpath:string(//p[2])", 42 | "text|xpath:string(//span[@type=\"published\"])", 43 | "text|xpath:string(//span[@type=\"category\"])" 44 | ] 45 | } -------------------------------------------------------------------------------- /presets/Scraping-Capture_screenshots.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Capture screenshots", 3 | "category": "Scraping", 4 | "description": "This preset can be used to capture a screenshot, a PDF print, plain text and rendered HTML from single pages. Capturing is an experimental feature (version 4.3.2 only). Maybe it will be enhanced, maybe removed.\n\n1. Adjust the Download folder in the Generic module.\n\n2. Add an URL as seed node, for example https://en.wikipedia.org/wiki/Api_(mountain)\n\n3. Select the seed node and Ctrl-Click (Windows, Linux) or Command-Click (Mac) onto Fetch Data. Hopefully, a browser window will open. \n\n4. When the page is loaded, as seen in the status bar, click Capture. You will get a new child node containing plain text, HTML, path to the screenshot and to the PDF. The files are stored in the choosen download folder.\n\nYou can browse other pages by entering an URL into the adress bar and clicking the Enter key. Every time you click Capture, an new child node and new files will be created.\n", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "", 8 | "resource": "", 9 | "params": {}, 10 | "headers": {}, 11 | "verb": "GET", 12 | "format": "text", 13 | "filename": "", 14 | "fileext": ".html", 15 | "pages": 1, 16 | "paging_type": "key", 17 | "key_paging": null, 18 | "paging_stop": null, 19 | "param_paging": null, 20 | "nodedata": null, 21 | "objectid": null, 22 | "scope": "", 23 | "proxy": "", 24 | "auth_type": "Disable", 25 | "auth_uri": "", 26 | "redirect_uri": "", 27 | "token_uri": "", 28 | "auth": "disable", 29 | "auth_tokenname": "", 30 | "auth_prefix": "" 31 | }, 32 | "speed": 200, 33 | "saveheaders": true, 34 | "columns": [ 35 | "title", 36 | "url.final", 37 | "url.requested", 38 | "screenshot", 39 | "pdf" 40 | ] 41 | } -------------------------------------------------------------------------------- /presets/Scraping-Extract_meta_tags.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Extract meta tags", 3 | "category": "Scraping", 4 | "description": "Add URL as seed node and fetch data. The meta-tags are contained in the HTML body and tell you details about the page, such as the author of the page or the date of the last modification. \n\nThis preset adds a child node for each meta tag.\n\nSome values follow predefined vocabularies, for example, defined in semantic web technology (RDF). You can identify these vocabularies from prefixes used in the name, e.g. the prefix \"og:\" stands for OpenGraph: . See https://ogp.me/\n\nMany social media sites, when sharing links, generate webpage previews from the meta tags. Common schemes are https://ogp.me/ and https://developer.twitter.com/en/docs/twitter-for-websites/cards/overview/abouts-cards\n\nMeta tags usally have a name-property and a content-property. Sometimes a property-property instead of the name-property is used.\n\nSee https://www.w3schools.com/tags/tag_meta.asp\n\n", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "text", 17 | "headers": {}, 18 | "key_paging": null, 19 | "nodedata": "text|xpath://meta", 20 | "objectid": "text|xpath://@content", 21 | "pages": 1, 22 | "paging_stop": null, 23 | "paging_type": "key", 24 | "param_paging": null, 25 | "params": {}, 26 | "proxy": "", 27 | "redirect_uri": "", 28 | "resource": "", 29 | "scope": "", 30 | "token_uri": "", 31 | "verb": "GET" 32 | }, 33 | "speed": 200, 34 | "saveheaders": false, 35 | "timeout": 15, 36 | "columns": [ 37 | "name=text|xpath://@name", 38 | "property=text|xpath://@property", 39 | "content=text|xpath://@content" 40 | ] 41 | } -------------------------------------------------------------------------------- /presets/Scraping-Get_target_of_short_links.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Get target of short links", 3 | "category": "Scraping", 4 | "description": "Add shortlinks as seed nodes. You can add other URLs as well, if you are interested in the final URL after redirections. \n\nNote: The preset uses the HEAD verb, supported by Facepager v4.3.3 or greater.\n", 5 | "module": "Generic", 6 | "options": { 7 | "basepath": "", 8 | "resource": "", 9 | "params": {}, 10 | "headers": { 11 | "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36" 12 | }, 13 | "verb": "HEAD", 14 | "format": "text", 15 | "filename": "", 16 | "fileext": "", 17 | "pages": 1, 18 | "paging_type": "key", 19 | "key_paging": null, 20 | "paging_stop": null, 21 | "param_paging": null, 22 | "nodedata": null, 23 | "objectid": null, 24 | "scope": "", 25 | "proxy": "", 26 | "auth_type": "Disable", 27 | "auth_uri": "", 28 | "redirect_uri": "", 29 | "token_uri": "", 30 | "auth": "disable", 31 | "auth_tokenname": "", 32 | "auth_prefix": "" 33 | }, 34 | "speed": 2000, 35 | "saveheaders": false, 36 | "columns": [ 37 | "finalurl", 38 | "sourcepath", 39 | "content-type" 40 | ] 41 | } -------------------------------------------------------------------------------- /presets/Scraping-Scrape_teaser_texts_from_Zeit_Online.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Scrape teaser texts from Zeit Online", 3 | "category": "Scraping", 4 | "description": "Add search terms as seed nodes and fetch data.\n\nThis preset uses the search of https://www.zeit.de/suche/index?q=corona&p=1 to fetch a list of articles from Zeit Online. \n\nSee the wiki for information about using CSS and XPath selectors with Facepager: https://github.com/strohne/Facepager/wiki/Webscraping", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_type": "Disable", 9 | "auth_uri": "", 10 | "basepath": "https://www.zeit.de/", 11 | "fileext": "", 12 | "filename": "", 13 | "format": "text", 14 | "headers": {}, 15 | "key_paging": null, 16 | "nodedata": "text|css:article.zon-teaser-standard", 17 | "objectid": "text|css:a.zon-teaser-standard__combined-link|xpath:@href", 18 | "offset_start": 1, 19 | "offset_step": 1, 20 | "pages": 3, 21 | "paging_type": "count", 22 | "param_paging": "p", 23 | "params": { 24 | "q": "" 25 | }, 26 | "redirect_uri": "", 27 | "resource": "suche/index", 28 | "scope": "", 29 | "token_uri": "", 30 | "verb": "GET" 31 | }, 32 | "speed": 200, 33 | "saveheaders": null, 34 | "columns": [ 35 | "kicker=text|css:span.zon-teaser-standard__kicker|xpath:string()", 36 | "header=text|css:span.zon-teaser-standard__title|xpath:string()", 37 | "teaser=text|css:p.zon-teaser-standard__text|xpath:string()", 38 | "date=text|css:time.zon-teaser-standard__datetime|xpath:@datetime" 39 | ] 40 | } -------------------------------------------------------------------------------- /presets/Scraping-Search_results_from_the_Financial_Times.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Search results from the Financial Times", 3 | "category": "Scraping", 4 | "description": "Add search terms as seed nodes and fetch data.\n\nThis preset uses the search of https://www.ft.com/search?sort=date&q=corona to fetch a list of articles from the Financial Times. \n\nSee the wiki for information about using CSS and XPath selectors with Facepager: https://github.com/strohne/Facepager/wiki/Webscraping", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "https://www.ft.com", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "text", 17 | "headers": {}, 18 | "nodedata": "text|css:div.search-item", 19 | "objectid": "text|xpath://a[@data-trackable=\"heading-link\"]/@href", 20 | "offset_start": 1, 21 | "offset_step": 1, 22 | "pages": 3, 23 | "paging_type": "count", 24 | "param_paging": "page", 25 | "params": { 26 | "q": "", 27 | "sort": "date" 28 | }, 29 | "proxy": "", 30 | "redirect_uri": "", 31 | "resource": "/search", 32 | "scope": "", 33 | "token_uri": "", 34 | "verb": "GET" 35 | }, 36 | "speed": 200, 37 | "saveheaders": false, 38 | "timeout": 15, 39 | "columns": [ 40 | "meta=text|css:div.o-teaser__meta|xpath:string()", 41 | "heading=text|css:div.o-teaser__heading|xpath:string()", 42 | "standfirst=text|css:p.o-teaser__standfirst|xpath:string()", 43 | "date=text|css:div.o-teaser__timestamp|xpath:string()", 44 | "timestamp=text|css:div.o-teaser__timestamp|xpath:time/@datetime" 45 | ] 46 | } -------------------------------------------------------------------------------- /presets/Scraping-Show_meta_tags_in_columns.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Show meta tags in columns", 3 | "category": "Scraping", 4 | "description": "Add URL as seed node and fetch data. The meta-tags are contained in the HTML body and tell you details about the page, such as the author of the page or the date of the last modification. \n\nThis preset downloads the HTML content and displays common meta tags in columns. \n\nSome values follow predefined vocabularies, for example, defined in semantic web technology (RDF). You can identify these vocabularies from prefixes used in the name, e.g. the prefix \"og:\" stands for OpenGraph: . See https://ogp.me/\n\nMany social media sites, when sharing links, generate webpage previews from the meta tags. Common schemes are https://ogp.me/ and https://developer.twitter.com/en/docs/twitter-for-websites/cards/overview/abouts-cards\n\nMeta tags usally have a name-property and a content-property. Sometimes a property-property instead of the name-property is used.\n\nSee https://www.w3schools.com/tags/tag_meta.asp", 5 | "module": "Generic", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "Disable", 11 | "auth_uri": "", 12 | "basepath": "", 13 | "extension": "", 14 | "fileext": "", 15 | "filename": "", 16 | "format": "text", 17 | "headers": {}, 18 | "key_paging": null, 19 | "nodedata": null, 20 | "objectid": null, 21 | "pages": 1, 22 | "paging_stop": null, 23 | "paging_type": "key", 24 | "param_paging": null, 25 | "params": {}, 26 | "proxy": "", 27 | "redirect_uri": "", 28 | "resource": "", 29 | "scope": "", 30 | "token_uri": "", 31 | "verb": "GET" 32 | }, 33 | "speed": 200, 34 | "saveheaders": false, 35 | "timeout": 15, 36 | "columns": [ 37 | "description=text|xpath://meta[@name='description']/@content", 38 | "og_title=text|xpath://meta[@property='og:title']/@content", 39 | "og_description=text|xpath://meta[@property='og:description']/@content", 40 | "og_image=text|xpath://meta[@property='og:image']/@content", 41 | "og_url=text|xpath://meta[@property='og:url']/@content", 42 | "fb_admins=text|xpath://meta[@property='fb:admins']/@content", 43 | "tw_title=text|xpath://meta[@name='twitter:title']/@content", 44 | "tw_description=text|xpath://meta[@name='twitter:description']/@content", 45 | "tw_creator=text|xpath://meta[@name='twitter:creator']/@content" 46 | ] 47 | } -------------------------------------------------------------------------------- /presets/Twitter-1_Get_tweets_by_hashtag_v1_1_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "1 Get tweets by hashtag (v1.1)", 3 | "category": "Twitter", 4 | "description": "Note. The free pre-registered accesses no longer work. Fetching Twitter data with Facepager is only possible after setting up a paid v2 access (https://developer.twitter.com/en).", 5 | "module": "Twitter", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "OAuth1", 11 | "auth_uri": "", 12 | "basepath": "https://api.twitter.com/1.1", 13 | "extension": ".json", 14 | "key_paging": "next_cursor_str", 15 | "nodedata": "statuses", 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_stop": "next_cursor", 19 | "paging_type": "key", 20 | "param_paging": "cursor", 21 | "params": { 22 | "count": "50", 23 | "q": "", 24 | "result_type": "recent" 25 | }, 26 | "redirect_uri": "", 27 | "resource": "/search/tweets", 28 | "token_uri": "" 29 | }, 30 | "speed": 200, 31 | "saveheaders": false, 32 | "timeout": 15, 33 | "maxsize": 5, 34 | "columns": [ 35 | "id", 36 | "created_at", 37 | "user.screen_name", 38 | "text", 39 | "favorite_count", 40 | "retweet_count", 41 | "entities.hashtags.*.text", 42 | "entities.user_mentions.*.name", 43 | "entities.urls.*.display_url", 44 | "lang" 45 | ] 46 | } -------------------------------------------------------------------------------- /presets/Twitter-2_Get_information_about_users_v1_1_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "2 Get information about users (v1.1)", 3 | "category": "Twitter", 4 | "description": "Note. The free pre-registered accesses no longer work. Fetching Twitter data with Facepager is only possible after setting up a paid v2 access (https://developer.twitter.com/en).", 5 | "options": { 6 | "auth": "disable", 7 | "auth_prefix": "", 8 | "auth_tokenname": "", 9 | "auth_type": "OAuth1", 10 | "auth_uri": "", 11 | "basepath": "https://api.twitter.com/1.1", 12 | "extension": ".json", 13 | "key_paging": "next_cursor_str", 14 | "nodedata": null, 15 | "objectid": "id", 16 | "pages": 1, 17 | "paging_stop": "next_cursor", 18 | "paging_type": "key", 19 | "param_paging": "cursor", 20 | "params": { 21 | "screen_name": "" 22 | }, 23 | "redirect_uri": "", 24 | "resource": "/users/show", 25 | "token_uri": "" 26 | }, 27 | "speed": 200, 28 | "saveheaders": false, 29 | "timeout": 15, 30 | "maxsize": 5, 31 | "columns": [ 32 | "id", 33 | "name", 34 | "screen_name", 35 | "created_at", 36 | "description", 37 | "location", 38 | "lang" 39 | ] 40 | } -------------------------------------------------------------------------------- /presets/Twitter-2_Get_information_about_users_v2_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "2 Get information about users (v2)", 3 | "category": "Twitter", 4 | "description": "Note. The free pre-registered accesses no longer work. Fetching Twitter data with Facepager is only possible after setting up a paid v2 access (https://developer.twitter.com/en). This preset gives you some basic information about a user. Add user names as nodes, without the @ (e.g. \"digicademy\").\n\n**Login Process**\n\n*To use version 2 presets, you have to register an own app, which is easy. An app is nothing more than a specific user account. See below.* \n\nOnce your app is registered (whether the essential version or in the academic research track), you will get an API key and an API secret key. You have to keep these well and should not show them to anyone.\n\nTo use your credentials with Facepager, click on \"Settings\" and change the \"Authentication Type\" to \"OAuth2 Client Credentials\". Enter your personal API key into the \"Consumer Key\" field and your API secret key into the \"Consumer Secret\" field. \n\nClick the \"Login to Twitter\" button to get an access token. Then you can start fetching data.\n\n\n**Get the user information**\n\nThe preset collects the fields id, name, username and description. Take a look at the documentation on https://developer.twitter.com/en/docs/twitter-api/users/lookup/api-reference/to learn handling the parameters.\n\n\n**Differences between v1.1 & v2**\n\nPlease remind that Twitter provides different API versions (https://developer.twitter.com/en/products/twitter-api). Facepager presets are available for API version 1 (v1) and for API version 2 (v2). Each API version has different access levels and limits. To use version 2, in general, you have to register your own app. Registering an app sounds complicated but is easy. In the world of APIs, an app simply is a specific type of user account.\n\nWhen registering an app, you can apply for different access levels, see https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api. \nAcademic researchers can apply for higher tweet caps or full-archive access, see https://developer.twitter.com/en/solutions/academic-research.", 5 | "module": "Twitter", 6 | "options": { 7 | "basepath": "https://api.twitter.com/2", 8 | "resource": "/users/by/username/", 9 | "params": { 10 | "": "", 11 | "user.fields": "id,name,username,description" 12 | }, 13 | "extension": "", 14 | "pages": 1, 15 | "paging_type": "key", 16 | "key_paging": "next_cursor_str", 17 | "paging_stop": "next_cursor", 18 | "param_paging": "cursor", 19 | "nodedata": "data", 20 | "objectid": "id", 21 | "auth_type": "OAuth2 Client Credentials", 22 | "auth_uri": "https://api.twitter.com/oauth2/token/", 23 | "redirect_uri": "", 24 | "token_uri": "", 25 | "auth": "header", 26 | "auth_prefix": "Bearer ", 27 | "auth_tokenname": "Authorization" 28 | }, 29 | "speed": 200, 30 | "saveheaders": false, 31 | "timeout": 15, 32 | "maxsize": 5, 33 | "columns": [ 34 | "id", 35 | "name", 36 | "username", 37 | "description" 38 | ] 39 | } -------------------------------------------------------------------------------- /presets/Twitter-3_Get_tweets_of_users_v1_1_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "3 Get tweets of users (v1.1)", 3 | "category": "Twitter", 4 | "description": "Note. The free pre-registered accesses no longer work. Fetching Twitter data with Facepager is only possible after setting up a paid v2 access (https://developer.twitter.com/en).", 5 | "module": "Twitter", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "OAuth1", 11 | "auth_uri": "", 12 | "basepath": "https://api.twitter.com/1.1", 13 | "extension": ".json", 14 | "key_paging": "next_cursor_str", 15 | "nodedata": null, 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_stop": "next_cursor", 19 | "paging_type": "key", 20 | "param_paging": "cursor", 21 | "params": { 22 | "count": "200", 23 | "include_rts": "1", 24 | "user_id": "" 25 | }, 26 | "redirect_uri": "", 27 | "resource": "/statuses/user_timeline", 28 | "token_uri": "" 29 | }, 30 | "speed": 200, 31 | "saveheaders": false, 32 | "timeout": 15, 33 | "maxsize": 5, 34 | "columns": [ 35 | "created_at", 36 | "text", 37 | "user.screen_name", 38 | "user.id_str", 39 | "retweet_count", 40 | "favorite_count", 41 | "retweeted_status.id", 42 | "quoted_status.id", 43 | "in_reply_to_status_id_str", 44 | "in_reply_to_user_id", 45 | "entities.user_mentions.*.id_str", 46 | "entities.hashtags.*.text", 47 | "entities.urls.*.url", 48 | "entities.symbols" 49 | ] 50 | } -------------------------------------------------------------------------------- /presets/Twitter-4_Get_list_of_followers_v1_1_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "4 Get list of followers (v1.1)", 3 | "category": "Twitter", 4 | "description": "Note. The free pre-registered accesses no longer work. Fetching Twitter data with Facepager is only possible after setting up a paid v2 access (https://developer.twitter.com/en).", 5 | "module": "Twitter", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "OAuth1", 11 | "auth_uri": "", 12 | "basepath": "https://api.twitter.com/1.1", 13 | "extension": ".json", 14 | "key_paging": "next_cursor_str", 15 | "nodedata": "users", 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_stop": "next_cursor", 19 | "paging_type": "key", 20 | "param_paging": "cursor", 21 | "params": { 22 | "count": "100", 23 | "user_id": "" 24 | }, 25 | "redirect_uri": "", 26 | "resource": "/followers/list", 27 | "token_uri": "" 28 | }, 29 | "speed": 200, 30 | "saveheaders": false, 31 | "timeout": 15, 32 | "maxsize": 5, 33 | "columns": [ 34 | "id", 35 | "name", 36 | "screen_name", 37 | "friends_count", 38 | "created_at", 39 | "errors.*.message" 40 | ] 41 | } -------------------------------------------------------------------------------- /presets/Twitter-4_Get_list_of_followers_v2_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "4 Get list of followers (v2)", 3 | "category": "Twitter", 4 | "description": "Note. The free pre-registered accesses no longer work. Fetching Twitter data with Facepager is only possible after setting up a paid v2 access (https://developer.twitter.com/en). This preset gives you the followers of users. Add numeric user IDs as seed nodes (e.g. \"865971338\"). You can get users IDs with the \"2 Get information about user\" preset.\n\n**Login Process**\n\n*To use version 2 presets, you have to register an own app, which is easy. An app is nothing more than a specific user account. See below.* \n\nOnce your app is registered (whether the essential version or in the academic research track), you will get an API key and an API secret key. You have to keep these well and should not show them to anyone.\n\nTo use your credentials with Facepager, click on \"Settings\" and change the \"Authentication Type\" to \"OAuth2 Client Credentials\". Enter your personal API key into the \"Consumer Key\" field and your API secret key into the \"Consumer Secret\" field. \n\nClick the \"Login to Twitter\" button to get an access token. Then you can start fetching data.\n\n**Get the followers**\n\nThis preset collects 100 followers of a user. In case you want to fetch more, change the max_result parameter (maximum per request is 1000) or increase the maximum pages.\n\nThe preset collects the fields id, name, username and description. Take a look at the documentation on https://developer.twitter.com/en/docs/twitter-api/users/follows/api-reference/to learn handling parameters.\n\n**Differences between v1.1 & v2**\n\nPlease remind that Twitter provides different API versions (https://developer.twitter.com/en/products/twitter-api). Facepager presets are available for API version 1 (v1) and for API version 2 (v2). Each API version has different access levels and limits. To use version 2, in general, you have to register your own app. Registering an app sounds complicated but is easy. In the world of APIs, an app simply is a specific type of user account.\n\nWhen registering an app, you can apply for different access levels, see https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api. \nAcademic researchers can apply for higher tweet caps or full-archive access, see https://developer.twitter.com/en/solutions/academic-research.", 5 | "module": "Twitter", 6 | "options": { 7 | "auth": "header", 8 | "auth_prefix": "Bearer ", 9 | "auth_tokenname": "Authorization", 10 | "auth_type": "OAuth2 Client Credentials", 11 | "auth_uri": "https://api.twitter.com/oauth2/token/", 12 | "basepath": "https://api.twitter.com/2", 13 | "extension": "", 14 | "key_paging": "next_cursor_str", 15 | "nodedata": "data", 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_stop": "next_cursor", 19 | "paging_type": "key", 20 | "param_paging": "cursor", 21 | "params": { 22 | "": "", 23 | "max_results": "100", 24 | "user.fields": "id,name,username,description" 25 | }, 26 | "redirect_uri": "", 27 | "resource": "/users//followers", 28 | "token_uri": "" 29 | }, 30 | "speed": 200, 31 | "saveheaders": false, 32 | "timeout": 15, 33 | "maxsize": 5, 34 | "columns": [ 35 | "id", 36 | "name", 37 | "username", 38 | "description" 39 | ] 40 | } -------------------------------------------------------------------------------- /presets/Twitter-5_Get_list_of_followees_v1_1_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "5 Get list of followees (v1.1)", 3 | "category": "Twitter", 4 | "description": "Note. The free pre-registered accesses no longer work. Fetching Twitter data with Facepager is only possible after setting up a paid v2 access (https://developer.twitter.com/en).", 5 | "module": "Twitter", 6 | "options": { 7 | "auth": "disable", 8 | "auth_prefix": "", 9 | "auth_tokenname": "", 10 | "auth_type": "OAuth1", 11 | "auth_uri": "", 12 | "basepath": "https://api.twitter.com/1.1", 13 | "extension": ".json", 14 | "key_paging": "next_cursor_str", 15 | "nodedata": "users", 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_stop": "next_cursor", 19 | "paging_type": "key", 20 | "param_paging": "cursor", 21 | "params": { 22 | "count": "100", 23 | "user_id": "" 24 | }, 25 | "redirect_uri": "", 26 | "resource": "/friends/list", 27 | "token_uri": "" 28 | }, 29 | "speed": 200, 30 | "saveheaders": false, 31 | "timeout": 15, 32 | "maxsize": 5, 33 | "columns": [ 34 | "id", 35 | "name", 36 | "screen_name", 37 | "friends_count", 38 | "created_at", 39 | "errors.*.message" 40 | ] 41 | } -------------------------------------------------------------------------------- /presets/Twitter-5_Get_list_of_followees_v2_.fp4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "5 Get list of followees (v2)", 3 | "category": "Twitter", 4 | "description": "Note. The free pre-registered accesses no longer work. Fetching Twitter data with Facepager is only possible after setting up a paid v2 access (https://developer.twitter.com/en). This preset collects the followees of users. Followees are users an account follows, aka friends or following. Add the numeric user IDs as seed nodes (e.g. \"865971338\"). You can retrieve user IDs with the \"2 Get information about user\" preset.\n\n**Login Process**\n\n*To use version 2 presets, you have to register an own app, which is easy. An app is nothing more than a specific user account. See below.* \n\nOnce your app is registered (whether the essential version or in the academic research track), you will get an API key and an API secret key. You have to keep these well and should not show them to anyone.\n\nTo use your credentials with Facepager, click on \"Settings\" and change the \"Authentication Type\" to \"OAuth2 Client Credentials\". Enter your personal API key into the \"Consumer Key\" field and your API secret key into the \"Consumer Secret\" field. \n\nClick the \"Login to Twitter\" button to get an access token. Then you can start fetching data.\n\n\n**Get the followees**\n\nThis preset collects 100 followees of the user. In case you want to fetch more, change the max_result parameter (maximum per request is 1000) or increase the maximum pages.\n\nThe preset collects the fields id, name, username and description. Take a look at the documentation on https://developer.twitter.com/en/docs/twitter-api/users/follows/api-reference/get-users-id-following to learn handling the parameters.\n\n**Differences between v1.1 & v2**\n\nPlease remind that Twitter provides different API versions (https://developer.twitter.com/en/products/twitter-api). Facepager presets are available for API version 1 (v1) and for API version 2 (v2). Each API version has different access levels and limits. To use version 2, in general, you have to register your own app. Registering an app sounds complicated but is easy. In the world of APIs, an app simply is a specific type of user account.\n\nWhen registering an app, you can apply for different access levels, see https://developer.twitter.com/en/docs/twitter-api/getting-started/about-twitter-api. \nAcademic researchers can apply for higher tweet caps or full-archive access, see https://developer.twitter.com/en/solutions/academic-research.\n", 5 | "module": "Twitter", 6 | "options": { 7 | "auth": "header", 8 | "auth_prefix": "Bearer ", 9 | "auth_tokenname": "Authorization", 10 | "auth_type": "OAuth2 Client Credentials", 11 | "auth_uri": "https://api.twitter.com/oauth2/token/", 12 | "basepath": "https://api.twitter.com/2", 13 | "extension": "", 14 | "key_paging": "next_cursor_str", 15 | "nodedata": "data", 16 | "objectid": "id", 17 | "pages": 1, 18 | "paging_stop": "next_cursor", 19 | "paging_type": "key", 20 | "param_paging": "cursor", 21 | "params": { 22 | "": "", 23 | "max_results": "100", 24 | "user.fields": "id,name,username,description" 25 | }, 26 | "redirect_uri": "", 27 | "resource": "/users//following", 28 | "token_uri": "" 29 | }, 30 | "speed": 200, 31 | "saveheaders": false, 32 | "timeout": 15, 33 | "maxsize": 5, 34 | "columns": [ 35 | "id", 36 | "name", 37 | "username", 38 | "description" 39 | ] 40 | } -------------------------------------------------------------------------------- /presets/YouTube-Channel_information.3_9.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "1 Channel information", 3 | "category": "YouTube", 4 | "description": "Add channel ID as node.\n\nYou can use the search preset to find a channel ID or take the last part of an URL, e.g.the last part of https://www.youtube.com/channel/UCpXaqdaawHUyujeSG1Wnm8Q\n\nIf you don't have the channel ID but the user name, remove the id parameter and add a parameter named \"forUsername\" with the value \"\". The user name can be found in the last part of URLs like https://www.youtube.com/user/MoritzTV\n\nSee https://developers.google.com/youtube/v3/docs/channels/list for further information about channels.", 5 | "module": "YouTube", 6 | "speed": 200, 7 | "options": { 8 | "resource": "/channels", 9 | "basepath": "https://www.googleapis.com/youtube/v3", 10 | "params": { 11 | "part": "snippet,contentDetails,statistics,brandingSettings,topicDetails,status", 12 | "id": "", 13 | "maxResults": "50" 14 | }, 15 | "pages": 1, 16 | "scope": "https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube" 17 | }, 18 | "columns": [ 19 | "id.videoId", 20 | "snippet.title", 21 | "snippet.description" 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /presets/YouTube-Get_comments.3_9.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "3 Get comments", 3 | "category": "YouTube", 4 | "description": "Add video IDs as nodes. You can use the search preset to get video IDs or get one from the last part of a video URL. For example add O2t8EDyBPPs to get comments to the video https://www.youtube.com/watch?v=O2t8EDyBPPs. By the way: the video is very much related to this preset, you should definitely watch it.\n\nFetching comments requires Facepager to get permission to manage your account. For this purpose the scope is changed and you need to login to Google again. Facepager will not modify anything in your account and will not fetch any data you don't request yourself. The status log shows all requests Facepager makes for you.\n\nSee the YouTube Data API reference for further information: https://developers.google.com/youtube/v3/docs/commentThreads/list\n", 5 | "module": "YouTube", 6 | "speed": 200, 7 | "options": { 8 | "params": { 9 | "textFormat": "plainText", 10 | "part": "snippet", 11 | "videoId": "", 12 | "maxResults": "10" 13 | }, 14 | "basepath": "https://www.googleapis.com/youtube/v3", 15 | "resource": "/commentThreads", 16 | "pages": 1, 17 | "scope": "https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube" 18 | }, 19 | "columns": [ 20 | "snippet.topLevelComment.snippet.authorDisplayName", 21 | "snippet.topLevelComment.snippet.textDisplay", 22 | "snippet.topLevelComment.snippet.publishedAt", 23 | "snippet.topLevelComment.snippet.updatedAt", 24 | "snippet.topLevelComment.snippet.likeCount" 25 | ] 26 | } 27 | -------------------------------------------------------------------------------- /presets/YouTube-Get_playlist_items.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "2 Get playlist items", 3 | "category": "YouTube", 4 | "description": "Each channel contains playlists and playlists contain playlist items. In the playlist items you find the videos. Therefore, to get the videos in a channel, you first query the channel information and then fetch the items in the 'uploads' playlist.\n\nTo use this preset, add IDs of playlists as seed nodes. If you already fetched channel information, you can use those nodes as starting nodes and set the parameter playlistId to \"\"\n\nSee https://developers.google.com/youtube/v3/docs/playlists/list and https://developers.google.com/youtube/v3/docs/playlistItems/list for further information.\n", 5 | "module": "YouTube", 6 | "options": { 7 | "basepath": "https://www.googleapis.com/youtube/v3", 8 | "resource": "/playlistItems", 9 | "params": { 10 | "maxResults": "50", 11 | "part": "snippet,id,contentDetails,status", 12 | "playlistId": "" 13 | }, 14 | "pages": 1, 15 | "paging_type": "key", 16 | "key_paging": "nextPageToken", 17 | "paging_stop": null, 18 | "param_paging": "pageToken", 19 | "nodedata": null, 20 | "objectid": null, 21 | "scope": "https://www.googleapis.com/auth/youtube.readonly", 22 | "auth_type": "OAuth2", 23 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 24 | "redirect_uri": "https://localhost", 25 | "token_uri": "https://accounts.google.com/o/oauth2/token", 26 | "auth": "header", 27 | "auth_prefix": "Bearer ", 28 | "auth_tokenname": "Authorization" 29 | }, 30 | "speed": 200, 31 | "headers": false, 32 | "columns": [ 33 | "kind", 34 | "snippet.publishedAt", 35 | "snippet.title", 36 | "snippet.description", 37 | "snippet.thumbnails.default.url", 38 | "snippet.playlistId", 39 | "snippet.position", 40 | "snippet.resourceId.kind", 41 | "snippet.resourceId.videoId", 42 | "snippet.channelId", 43 | "snippet.channelTitle", 44 | "contentDetails.videoId", 45 | "contentDetails.videoPublishedAt", 46 | "status.privacyStatus" 47 | ] 48 | } -------------------------------------------------------------------------------- /presets/YouTube-Get_related_videos.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Get related videos", 3 | "category": "YouTube", 4 | "columns": [ 5 | "id.videoId", 6 | "snippet.title", 7 | "snippet.description", 8 | "snippet.channelId", 9 | "snippet.channelTitle" 10 | ], 11 | "description": "*Note: Due to changes in the API, this preset does no longer work. The relatedToVideoId parameter ceased to exist*\n\nAdd a video IDs as a node. You can use the search preset to get video IDs or get one from the last part of a video URL. For example add O2t8EDyBPPs to get comments to the video https://www.youtube.com/watch?v=O2t8EDyBPPs. By the way: the video is very much related to this preset, you should definitely watch it. \n\nThis preset returns 50 videos, related to the video you entered as a node.\n\nSee https://developers.google.com/youtube/v3/docs/search/list for information about parameters.\n\nOverview about YouTube Data API reference: https://developers.google.com/youtube/v3/docs/", 12 | "headers": false, 13 | "module": "YouTube", 14 | "options": { 15 | "basepath": "https://www.googleapis.com/youtube/v3", 16 | "resource": "/search", 17 | "params": { 18 | "maxResults": "50", 19 | "part": "snippet", 20 | "relatedToVideoId": "", 21 | "type": "video" 22 | }, 23 | "pages": 1, 24 | "paging_type": null, 25 | "key_paging": "nextPageToken", 26 | "param_paging": "pageToken", 27 | "offset_start": 1, 28 | "offset_step": 1, 29 | "objectid": "id.videoId", 30 | "nodedata": "items", 31 | "scope": "https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube", 32 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 33 | "redirect_uri": "https://localhost", 34 | "token_uri": "https://accounts.google.com/o/oauth2/token", 35 | "auth": "param" 36 | }, 37 | "speed": 200 38 | } 39 | -------------------------------------------------------------------------------- /presets/YouTube-Get_replies.3_10.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "4 Get replies", 3 | "category": "YouTube", 4 | "description": "YouTube videos and channels contain two type of comments. Top level comments are attached to a video and can be requested with the commentsThread endpoint. Replies are attached to parent comments and can be fetched with this preset\n\nAdd comment IDs as seed nodes or fetch comments with the respective preset first. Look at the settings: the parentId-parameter makes sure you get the replies instead of the comment itself.\n\nSee https://developers.google.com/youtube/v3/docs/comments/list for further information about replies and https://developers.google.com/youtube/v3/docs/commentThreads/list for information about getting top level comments.\n", 5 | "module": "YouTube", 6 | "options": { 7 | "auth": "header", 8 | "auth_prefix": "Bearer ", 9 | "auth_tokenname": "Authorization", 10 | "auth_type": "OAuth2", 11 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 12 | "basepath": "https://www.googleapis.com/youtube/v3", 13 | "key_paging": "nextPageToken", 14 | "nodedata": null, 15 | "objectid": null, 16 | "pages": 5, 17 | "paging_stop": null, 18 | "paging_type": "key", 19 | "param_paging": "pageToken", 20 | "params": { 21 | "maxResults": "100", 22 | "parentId": "", 23 | "part": "snippet" 24 | }, 25 | "redirect_uri": "https://localhost", 26 | "resource": "/comments", 27 | "scope": "https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube", 28 | "token_uri": "https://accounts.google.com/o/oauth2/token" 29 | }, 30 | "speed": 200, 31 | "headers": false, 32 | "columns": [ 33 | "snippet.topLevelComment.snippet.authorDisplayName", 34 | "snippet.topLevelComment.snippet.textDisplay", 35 | "snippet.topLevelComment.snippet.publishedAt", 36 | "snippet.topLevelComment.snippet.updatedAt", 37 | "snippet.topLevelComment.snippet.likeCount" 38 | ] 39 | } -------------------------------------------------------------------------------- /presets/YouTube-Get_video_statistics.3_9.json: -------------------------------------------------------------------------------- 1 | { 2 | "category": "YouTube", 3 | "description": "Add video IDs as nodes. Alternatively, you can use the results of searching YouTube (see the corresponding preset).\n\nSee https://developers.google.com/youtube/v3/docs/videos/list for further information. Notice the different options for the part-parameter.\n\nOverview about YouTube Data API reference: https://developers.google.com/youtube/v3/docs/", 4 | "module": "YouTube", 5 | "speed": 200, 6 | "options": { 7 | "resource": "/videos", 8 | "basepath": "https://www.googleapis.com/youtube/v3", 9 | "params": { 10 | "part": "snippet,contentDetails,statistics", 11 | "id": "" 12 | }, 13 | "pages": 1, 14 | "scope": "https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube" 15 | }, 16 | "columns": [ 17 | "snippet.title", 18 | "statistics.commentCount", 19 | "statistics.viewCount", 20 | "statistics.favoriteCount", 21 | "statistics.dislikeCount", 22 | "statistics.likeCount" 23 | ], 24 | "name": "Get video statistics" 25 | } 26 | -------------------------------------------------------------------------------- /presets/YouTube-Search_YouTube_videos.3_9.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Search YouTube videos", 3 | "category": "YouTube", 4 | "description": "Add search terms as nodes. Just write the search terms like you would type them on the YouTube website.\n\nThis preset gives you back 50 search results. If you want more change the number in the maxResults-parameter.\n\nSee https://developers.google.com/youtube/v3/docs/search/list for information about parameters.\n\nOverview about YouTube Data API reference: https://developers.google.com/youtube/v3/docs/", 5 | "module": "YouTube", 6 | "speed": 200, 7 | "options": { 8 | "resource": "/search", 9 | "basepath": "https://www.googleapis.com/youtube/v3", 10 | "params": { 11 | "q": "", 12 | "part": "snippet", 13 | "maxResults": "50" 14 | }, 15 | "pages": 1, 16 | "scope": "https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/youtube.readonly https://www.googleapis.com/auth/youtube" 17 | }, 18 | "columns": [ 19 | "id.videoId", 20 | "snippet.title", 21 | "snippet.description" 22 | ] 23 | } 24 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | SQLAlchemy 2 | python-dateutil 3 | requests_oauthlib 4 | requests-toolbelt 5 | rauth 6 | lxml 7 | xmljson 8 | cchardet 9 | pyjsparser 10 | tldextract 11 | cssselect 12 | markdown 13 | PySide2==5.15.* 14 | rdflib -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/src/__init__.py -------------------------------------------------------------------------------- /src/aws/authorize.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | var s3 = new AWS.S3(); 3 | var bucketName = 'facepager-users'; 4 | 5 | // Authorize by module and usertoken parameters 6 | 7 | exports.handler = async (event) => { 8 | 9 | // Default response 10 | let response = { 11 | statusCode: 401, 12 | body: JSON.stringify('Not authorized.'), 13 | }; 14 | 15 | 16 | try { 17 | 18 | // Get bucketKey 19 | let bucketKey = getBucketKey(event); 20 | 21 | // Get item for user 22 | let data = await getBucketItem(bucketKey); 23 | console.log(data) 24 | 25 | // Check blacklist... 26 | if (data.status != 'allowed') 27 | throw "Not authorized!" 28 | 29 | // Add to whitelist... 30 | await createBucketItem(bucketKey,'allowed') 31 | 32 | response = { 33 | statusCode: 200, 34 | body: JSON.stringify('Successfully authorized!'), 35 | }; 36 | } catch(e) { 37 | console.log(e) 38 | } 39 | 40 | return response; 41 | }; 42 | 43 | 44 | // Get bucket key from module and usertoken 45 | function getBucketKey(event){ 46 | let usertoken = ''; 47 | let module = ''; 48 | if (event.queryStringParameters && event.queryStringParameters.usertoken) { 49 | usertoken = event.queryStringParameters.usertoken.replace(/[\W_]+/g,""); 50 | } 51 | if (event.queryStringParameters && event.queryStringParameters.module) { 52 | module = event.queryStringParameters.module.replace(/[\W_]+/g,""); 53 | } 54 | 55 | if ((module == '') || (usertoken == '')) 56 | throw "Missing authorization data." 57 | 58 | return module+'/'+usertoken; 59 | } 60 | 61 | 62 | // Get bucket item 63 | async function getBucketItem(bucketKey) { 64 | try { 65 | let bucketParams = {Bucket: bucketName, Key: bucketKey}; 66 | let item = await s3.getObject(bucketParams).promise(); 67 | return JSON.parse(item.Body.toString('utf-8')) 68 | } catch (e) { 69 | return {}; 70 | } 71 | } 72 | 73 | // Create bucket item 74 | async function createBucketItem(bucketKey,status){ 75 | let bucketContent = {'timestamp':Date.now(),'token':bucketKey,'status':status}; 76 | let bucketParams = {Bucket: bucketName, Key: bucketKey, Body: JSON.stringify(bucketContent)}; 77 | 78 | return s3.putObject(bucketParams).promise(); 79 | } -------------------------------------------------------------------------------- /src/aws/blacklist.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | var s3 = new AWS.S3(); 3 | var bucketName = 'facepager-users'; 4 | 5 | // Block by module and usertoken parameters 6 | 7 | exports.handler = async (event) => { 8 | 9 | 10 | // Default response 11 | let response = { 12 | statusCode: 401, 13 | body: JSON.stringify('Not authorized.'), 14 | }; 15 | 16 | 17 | try { 18 | 19 | // Get bucketKey 20 | let bucketKey = getBucketKey(event); 21 | 22 | 23 | // Add to blacklist... 24 | await createBucketItem(bucketKey,'blacklisted') 25 | 26 | response = { 27 | statusCode: 200, 28 | body: JSON.stringify('Successfully blacklisted!'), 29 | }; 30 | } catch(e) { 31 | console.log(e) 32 | } 33 | 34 | return response; 35 | }; 36 | 37 | 38 | // Get bucket key from module and usertoken 39 | function getBucketKey(event){ 40 | let usertoken = ''; 41 | let module = ''; 42 | if (event.queryStringParameters && event.queryStringParameters.usertoken) { 43 | usertoken = event.queryStringParameters.usertoken.replace(/[\W_]+/g,""); 44 | } 45 | if (event.queryStringParameters && event.queryStringParameters.module) { 46 | module = event.queryStringParameters.module.replace(/[\W_]+/g,""); 47 | } 48 | 49 | if ((module == '') || (usertoken == '')) 50 | throw "Missing authorization data." 51 | 52 | return module+'/'+usertoken; 53 | } 54 | 55 | 56 | // Get bucket item 57 | async function getBucketItem(bucketKey) { 58 | try { 59 | let bucketParams = {Bucket: bucketName, Key: bucketKey}; 60 | let item = await s3.getObject(bucketParams).promise(); 61 | return JSON.parse(item.Body.toString('utf-8')) 62 | } catch (e) { 63 | return {}; 64 | } 65 | } 66 | 67 | // Create bucket item 68 | async function createBucketItem(bucketKey,status){ 69 | let bucketContent = {'timestamp':Date.now(),'token':bucketKey,'status':status}; 70 | let bucketParams = {Bucket: bucketName, Key: bucketKey, Body: JSON.stringify(bucketContent)}; 71 | 72 | return s3.putObject(bucketParams).promise(); 73 | } -------------------------------------------------------------------------------- /src/dialogs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/src/dialogs/__init__.py -------------------------------------------------------------------------------- /src/dialogs/folder.py: -------------------------------------------------------------------------------- 1 | from PySide2.QtCore import * 2 | from PySide2.QtGui import * 3 | from PySide2.QtWidgets import QFileDialog, QCheckBox, QHBoxLayout, QLabel 4 | 5 | class SelectFolderDialog(QFileDialog): 6 | """ 7 | Create a custom Folder Dialog with an option to import files as nodes 8 | """ 9 | 10 | def __init__(self,*args,**kwargs): 11 | super(SelectFolderDialog,self).__init__(*args,**kwargs) 12 | self.setOption(QFileDialog.DontUseNativeDialog) 13 | self.setFileMode(QFileDialog.Directory) 14 | 15 | 16 | #QFileDialog.getExistingDirectory(self, 'Select Download Folder', datadir)) #, QFileDialog.ShowDirsOnly 17 | #self.mainWindow = self.parent() 18 | 19 | self.optionNodes = QCheckBox("Add selected files as nodes",self) 20 | self.optionNodes.clicked.connect(self.optionNodesClick) 21 | #self.optionNodes.setCheckState(Qt.CheckState.Checked) 22 | 23 | layout = self.layout() 24 | row = layout.rowCount() 25 | layout.addWidget(QLabel('Options'),row,0) 26 | 27 | options = QHBoxLayout() 28 | options.addWidget(self.optionNodes) 29 | options.addStretch(1) 30 | layout.addLayout(options,row,1,1,2) 31 | self.setLayout(layout) 32 | 33 | #if self.exec_(): 34 | #if os.path.isfile(self.selectedFiles()[0]): 35 | 36 | def optionNodesClick(self): 37 | if self.optionNodes.isChecked(): 38 | self.setFileMode(QFileDialog.ExistingFiles) 39 | else: 40 | self.setFileMode(QFileDialog.Directory) -------------------------------------------------------------------------------- /src/dialogs/help.py: -------------------------------------------------------------------------------- 1 | from PySide2.QtCore import * 2 | from PySide2.QtGui import * 3 | from PySide2.QtWidgets import QPushButton, QVBoxLayout, QHBoxLayout, QWidget, QMainWindow 4 | 5 | from PySide2.QtWebEngineWidgets import QWebEngineView, QWebEnginePage 6 | 7 | 8 | 9 | import os 10 | import sys 11 | import webbrowser 12 | 13 | class HelpWindow(QMainWindow): 14 | def __init__(self, parent=None): 15 | super(HelpWindow,self).__init__(parent) 16 | 17 | self.setWindowTitle("Facepager 4 - Help") 18 | self.setMinimumWidth(600); 19 | self.setMinimumHeight(600); 20 | central = QWidget() 21 | self.setCentralWidget(central) 22 | vLayout = QVBoxLayout(central) 23 | 24 | self.page = MyQWebEnginePage() 25 | self.browser = QWebEngineView(central) 26 | self.browser.setPage(self.page) 27 | 28 | vLayout.addWidget(self.browser) 29 | hLayout = QHBoxLayout() 30 | vLayout.addLayout(hLayout) 31 | hLayout.addStretch(5) 32 | dismiss = QPushButton(central) 33 | dismiss.setText("Close") 34 | dismiss.clicked.connect(self.hide) 35 | hLayout.addWidget(dismiss) 36 | #browser.setBackgroundRole(QPalette.Window) 37 | 38 | def show(self): 39 | super(HelpWindow,self).show() 40 | self.loadPage() 41 | 42 | 43 | def loadPage(self): 44 | self.browser.load(QUrl("http://strohne.github.io/Facepager/")) 45 | 46 | class MyQWebEnginePage(QWebEnginePage): 47 | def __init__(self, parent=None): 48 | super(MyQWebEnginePage,self).__init__(parent) 49 | 50 | 51 | def acceptNavigationRequest(self, url, type, isMainFrame): 52 | if (type == QWebEnginePage.NavigationTypeLinkClicked): 53 | url = url.toString() 54 | if not url.startswith("http://strohne.github.io/Facepager/"): 55 | webbrowser.open(url) 56 | return False 57 | 58 | return True 59 | -------------------------------------------------------------------------------- /src/settings.py: -------------------------------------------------------------------------------- 1 | settings = { 2 | "version" : "4.6", 3 | "presetListUrl" : "https://api.github.com/repos/strohne/Facepager/contents/presets", 4 | "presetFileUrl": "https://raw.githubusercontent.com/strohne/Facepager/master/", 5 | "userAgent" : 'FACEPAGERBOT/4.6 ([https://github.com/strohne/Facepager](https://github.com/strohne/Facepager)) fp/4.6' 6 | } -------------------------------------------------------------------------------- /src/widgets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/src/widgets/__init__.py -------------------------------------------------------------------------------- /src/widgets/textviewer.py: -------------------------------------------------------------------------------- 1 | from PySide2.QtCore import * 2 | from PySide2.QtGui import * 3 | from PySide2.QtWidgets import QTextBrowser 4 | from markdown import markdown 5 | import re 6 | import html 7 | 8 | class TextViewer(QTextBrowser): 9 | 10 | def __init__(self, *args, **kwargs): 11 | super(TextViewer, self).__init__(*args, **kwargs) 12 | self.setOpenExternalLinks(True) 13 | self.setStyleSheet("border:0px;") 14 | self.setReadOnly(True) 15 | self.document().contentsChanged.connect(self.sizeChanged) 16 | self.document().documentLayout().documentSizeChanged.connect(self.sizeChanged) 17 | 18 | def sizeChanged(self): 19 | docHeight = self.document().size().height() 20 | self.setMaximumHeight(docHeight) 21 | self.setMinimumHeight(docHeight) 22 | 23 | def setText(self,text): 24 | text = '' if text is None else text 25 | #text = self.autoLinkText(text) 26 | 27 | # auto br prevents markdown parsing 28 | # todo: make presets markdown compatible 29 | text = self.autoBrText(text) 30 | 31 | text = markdown(text) 32 | # text = html.escape(text) 33 | 34 | self.setHtml(text) 35 | 36 | def autoBrText(self,html): 37 | return html.replace('\n', '
') 38 | 39 | def autoLinkText(self,html): 40 | """ 41 | @deprecated 42 | 43 | match all the urls 44 | this returns a tuple with two groups 45 | if the url is part of an existing link, the second element 46 | in the tuple will be "> or 47 | if not, the second element will be an empty string 48 | """ 49 | urlre = re.compile("(\(?https?://[-A-Za-z0-9+&@#/%?=~_()|!:,.;]*[-A-Za-z0-9+&@#/%=~_()|])(\">|)?") 50 | urls = urlre.findall(html) 51 | clean_urls = [] 52 | 53 | # remove the duplicate matches 54 | # and replace urls with a link 55 | for url in urls: 56 | # ignore urls that are part of a link already 57 | if url[1]: continue 58 | c_url = url[0] 59 | # ignore parens if they enclose the entire url 60 | if c_url[0] == '(' and c_url[-1] == ')': 61 | c_url = c_url[1:-1] 62 | 63 | if c_url in clean_urls: continue # We've already linked this url 64 | 65 | clean_urls.append(c_url) 66 | # substitute only where the url is not already part of a 67 | # link element. 68 | html = re.sub("(?))" + re.escape(c_url), 69 | "" + c_url + "", 70 | html) 71 | return html -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/strohne/Facepager/4d8d333de67416e075ae78c34acfe5d73f8372a1/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_apiviewer.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from PySide2.QtWidgets import QMainWindow 3 | from dialogs.apiviewer import ApiViewer 4 | 5 | class Test_Utilities(TestCase): 6 | 7 | def setUp(self): 8 | self.mainwindow = QMainWindow() 9 | self.apiviewer = ApiViewer(self.mainwindow) 10 | 11 | def tearDown(self): 12 | del self.apiviewer 13 | del self.mainwindow 14 | 15 | def test_get_api_doc(self): 16 | pass 17 | 18 | # out = self.apiviewer.getApiDoc("Facebook") 19 | # self.assertEqual('Facebook',out['x-facepager-module']) 20 | # self.assertEqual('https://graph.facebook.com/v3.3', out['servers'][0]['url']) 21 | -------------------------------------------------------------------------------- /tests/test_utilities.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | from utilities import getDictValue, extractValue 3 | 4 | class Test_Utilities(TestCase): 5 | 6 | def setUp(self): 7 | self.fixture = { 8 | 'posts': { 9 | 'comments': [ 10 | {'text': 'smartidea1','tags':['red']}, 11 | {'text': 'smartidea2','tags':['blue','green']}, 12 | {'text': 'smartidea3', 'wild.*.card': 'party'} 13 | ] 14 | } 15 | } 16 | 17 | def tearDown(self): 18 | self.fixture = None 19 | 20 | def test_get_dict_value(self): 21 | out = getDictValue(self.fixture,'posts.comments.0.text') 22 | self.assertEqual('smartidea1',out) 23 | 24 | out = getDictValue(self.fixture,'posts.comments.*.tags',dump=False) 25 | self.assertEqual(['red','blue','green',''],out) 26 | 27 | out = getDictValue(self.fixture,'posts.comments.*.wild\.*\.card',dump=False) 28 | self.assertEqual(['','','party'],out) 29 | 30 | def test_extract_value(self): 31 | out = extractValue(self.fixture, 'posts.comments.*.wild\.*\.card', dump=False) 32 | self.assertEqual((None,['','','party']),out) --------------------------------------------------------------------------------