├── .github └── workflows │ └── push.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── build.boot ├── changelog.md ├── docker-compose.yml ├── src └── keboola │ ├── docker │ ├── config.clj │ └── runtime.clj │ ├── facebook │ ├── api │ │ ├── exponential_backoff.clj │ │ ├── parser.clj │ │ └── request.clj │ └── extractor │ │ ├── core.clj │ │ ├── output.clj │ │ ├── query.clj │ │ ├── query_parser.clj │ │ └── sync_actions.clj │ ├── http │ ├── client.clj │ └── recording.clj │ └── utils │ └── json_to_csv.clj └── test └── keboola ├── docker ├── config_test.clj └── runtime_test.clj ├── facebook ├── api │ └── request_test.clj └── extractor │ ├── choose_token_test.clj │ ├── core_test.clj │ ├── output_test.clj │ ├── query_parser_test.clj │ └── query_test.clj ├── http └── client_test.clj ├── snapshots ├── ads │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── ads.manifest │ │ │ ├── ads │ │ │ └── 1507033518092 │ │ │ ├── adsets.manifest │ │ │ ├── adsets │ │ │ └── 1507033521606 │ │ │ ├── campaigns.manifest │ │ │ └── campaigns │ │ │ └── 1507033519047 │ └── test_ads.clj ├── adsinsights │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── ads_insights.manifest │ │ │ └── ads_insights │ │ │ └── 1507033655209 │ └── test_adsinsights.clj ├── asyncinisghtscampaigns │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── query_insights.manifest │ │ │ └── query_insights │ │ │ └── 1628614276830 │ └── test_asyncinisghtscampaigns.clj ├── campaignsinsights │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── campaigns_insights.manifest │ │ │ ├── campaigns_insights │ │ │ └── 1507031538125 │ │ │ ├── campaigns_insights_reaction_insights.manifest │ │ │ ├── campaigns_insights_reaction_insights │ │ │ └── 1507031560441 │ │ │ ├── campaigns_insights_type_insights.manifest │ │ │ └── campaigns_insights_type_insights │ │ │ └── 1507031550083 │ └── test_campaignsinsights.clj ├── core.clj ├── feed │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── feed.manifest │ │ │ ├── feed │ │ │ └── 1516621526977 │ │ │ ├── feed_comments.manifest │ │ │ └── feed_comments │ │ │ └── 1516621526976 │ └── test_feed.clj ├── feedsummary │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── feed_likes_summary.manifest │ │ │ └── feed_likes_summary │ │ │ └── 1516621525270 │ └── test_feedsummary.clj ├── outdirs_check.clj ├── pageinsights │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── summarytest_posts.manifest │ │ │ ├── summarytest_posts │ │ │ └── 1490947862466 │ │ │ ├── summarytest_summary.manifest │ │ │ └── summarytest_summary │ │ │ └── 1490947862466 │ └── test_pageinsights.clj ├── postsinsights │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── posts_insights.manifest │ │ │ └── posts_insights │ │ │ └── 1492272750239 │ └── test_postsinsights.clj ├── runbyid │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── ads_insights.manifest │ │ │ └── ads_insights │ │ │ └── 1570782087866 │ └── test_runbyid.clj ├── serializelists │ ├── apicalls.clj │ ├── config.json │ ├── out │ │ └── tables │ │ │ ├── accounts │ │ │ ├── adsets.manifest │ │ │ └── adsets │ │ │ └── 1638290127158 │ └── test_serializelists.clj └── template.mustache ├── test_utils └── core.clj └── utils └── json_to_csv_test.clj /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | name: GitHub Actions 2 | on: [ push ] 3 | concurrency: ci-${{ github.ref }} # to avoid tag collisions in the ECR 4 | env: 5 | # Name of the image in the ECR 6 | APP_IMAGE: keboola/ex-facebook-graph-api 7 | 8 | # Developer portal login 9 | KBC_DEVELOPERPORTAL_VENDOR: "keboola" 10 | KBC_DEVELOPERPORTAL_APP_FB: "keboola.ex-facebook" 11 | KBC_DEVELOPERPORTAL_APP_FB_ADS: "keboola.ex-facebook-ads" 12 | KBC_DEVELOPERPORTAL_APP_INSTAGRAM: "keboola.ex-instagram" 13 | KBC_DEVELOPERPORTAL_USERNAME: "keboola+facebook_extractor_github_actions" 14 | KBC_DEVELOPERPORTAL_PASSWORD: ${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }} 15 | 16 | # DockerHub login 17 | DOCKERHUB_USER: "keboolabot" 18 | DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} 19 | 20 | # Test KBC project 21 | KBC_STORAGE_TOKEN: ${{ secrets.KBC_STORAGE_TOKEN }} 22 | KBC_TEST_PROJECT_URL: "https://connection.keboola.com/admin/projects/395" 23 | KBC_TEST_PROJECT_CONFIGS_FB: "348767954" # space separated list 24 | KBC_TEST_PROJECT_CONFIGS_FB_ADS: "395964032" # space separated list 25 | jobs: 26 | build: 27 | runs-on: ubuntu-latest 28 | outputs: 29 | app_image_tag: ${{ steps.tag.outputs.app_image_tag }} 30 | is_semantic_tag: ${{ steps.tag.outputs.is_semantic_tag }} 31 | steps: 32 | - 33 | name: Check out the repo 34 | uses: actions/checkout@v2 35 | - 36 | name: Print Docker version 37 | run: docker -v 38 | - 39 | name: Docker login 40 | if: env.DOCKERHUB_TOKEN 41 | run: docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN" 42 | - 43 | name: Build image 44 | run: docker build -t $APP_IMAGE . 45 | - 46 | name: Set image tag 47 | id: tag 48 | run: | 49 | TAG="${GITHUB_REF##*/}" 50 | IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false) 51 | echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'" 52 | echo "::set-output name=app_image_tag::$TAG" 53 | echo "::set-output name=is_semantic_tag::$IS_SEMANTIC_TAG" 54 | - 55 | name: Push Facebook ads extractor image to ECR 56 | uses: keboola/action-push-to-ecr@master 57 | with: 58 | vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} 59 | app_id: ${{ env.KBC_DEVELOPERPORTAL_APP_FB }} 60 | username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} 61 | password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} 62 | tag: ${{ steps.tag.outputs.app_image_tag }} 63 | push_latest: ${{ steps.tag.outputs.is_semantic_tag }} 64 | source_image: ${{ env.APP_IMAGE}} 65 | 66 | - name: Push Facebook ads extractor image to ECR 67 | uses: keboola/action-push-to-ecr@master 68 | with: 69 | vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} 70 | app_id: ${{ env.KBC_DEVELOPERPORTAL_APP_FB_ADS }} 71 | username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} 72 | password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} 73 | tag: ${{ steps.tag.outputs.app_image_tag }} 74 | push_latest: ${{ steps.tag.outputs.is_semantic_tag }} 75 | source_image: ${{ env.APP_IMAGE}} 76 | 77 | - name: Push Instagram extractor image to ECR 78 | uses: keboola/action-push-to-ecr@master 79 | with: 80 | vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} 81 | app_id: ${{ env.KBC_DEVELOPERPORTAL_APP_INSTAGRAM }} 82 | username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} 83 | password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} 84 | tag: ${{ steps.tag.outputs.app_image_tag }} 85 | push_latest: ${{ steps.tag.outputs.is_semantic_tag }} 86 | source_image: ${{ env.APP_IMAGE}} 87 | 88 | tests: 89 | needs: build 90 | runs-on: ubuntu-latest 91 | steps: 92 | - 93 | name: Check out the repo 94 | uses: actions/checkout@v2 95 | - 96 | name: Pull image from ECR 97 | uses: keboola/action-pull-from-ecr@master 98 | with: 99 | vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} 100 | app_id: ${{ env.KBC_DEVELOPERPORTAL_APP_FB }} 101 | username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} 102 | password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} 103 | tag: ${{ needs.build.outputs.app_image_tag }} 104 | target_image: ${{ env.APP_IMAGE}} 105 | tag_as_latest: true 106 | - 107 | name: Run tests 108 | run: docker run ${{env.APP_IMAGE}} boot test 109 | 110 | tests-in-kbc: 111 | needs: build 112 | runs-on: ubuntu-latest 113 | steps: 114 | - 115 | name: Run KBC test jobs (FB Extractor) 116 | if: env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS_FB 117 | uses: keboola/action-run-configs-parallel@master 118 | with: 119 | token: ${{ env.KBC_STORAGE_TOKEN }} 120 | componentId: ${{ env.KBC_DEVELOPERPORTAL_APP_FB }} 121 | tag: ${{ needs.build.outputs.app_image_tag }} 122 | configs: ${{ env.KBC_TEST_PROJECT_CONFIGS_FB }} 123 | 124 | - 125 | name: Run KBC test jobs (FB Ads Extractor) 126 | if: env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS_FB_ADS 127 | uses: keboola/action-run-configs-parallel@master 128 | with: 129 | token: ${{ env.KBC_STORAGE_TOKEN }} 130 | componentId: ${{ env.KBC_DEVELOPERPORTAL_APP_FB_ADS }} 131 | tag: ${{ needs.build.outputs.app_image_tag }} 132 | configs: ${{ env.KBC_TEST_PROJECT_CONFIGS_FB_ADS }} 133 | 134 | deploy: 135 | needs: 136 | - build 137 | - tests 138 | - tests-in-kbc 139 | runs-on: ubuntu-latest 140 | if: startsWith(github.ref, 'refs/tags/') && needs.build.outputs.is_semantic_tag == 'true' 141 | steps: 142 | - 143 | name: Set tag in the Deloper Portal (Facebook Extractor) 144 | uses: keboola/action-set-tag-developer-portal@master 145 | with: 146 | vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} 147 | app_id: ${{ env.KBC_DEVELOPERPORTAL_APP_FB }} 148 | username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} 149 | password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} 150 | tag: ${{ needs.build.outputs.app_image_tag }} 151 | 152 | - 153 | name: Set tag in the Deloper Portal (Facebook Ads Extractor) 154 | uses: keboola/action-set-tag-developer-portal@master 155 | with: 156 | vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} 157 | app_id: ${{ env.KBC_DEVELOPERPORTAL_APP_FB_ADS }} 158 | username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} 159 | password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} 160 | tag: ${{ needs.build.outputs.app_image_tag }} 161 | 162 | - 163 | name: Set tag in the Deloper Portal (Instagram Extractor) 164 | uses: keboola/action-set-tag-developer-portal@master 165 | with: 166 | vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }} 167 | app_id: ${{ env.KBC_DEVELOPERPORTAL_APP_INSTAGRAM }} 168 | username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }} 169 | password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }} 170 | tag: ${{ needs.build.outputs.app_image_tag }} 171 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /targetinsights/ 2 | /target/ 3 | /tmp/* 4 | /.nrepl-history 5 | /.nrepl-port 6 | /src/keboola/facebook/api/sandbox.clj 7 | /.fbtokens-env 8 | *tern-port 9 | .idea 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM clojure:boot-2.8.2-alpine 2 | MAINTAINER 3 | 4 | # ENV BOOT_JVM_OPTIONS=-Xmx256m 5 | ENV BOOT_CLOJURE_VERSION=1.11.1 6 | 7 | ADD . /code 8 | WORKDIR /code 9 | RUN boot build 10 | RUN chmod a+r target/ex-fb-graph-api-1.0.jar 11 | CMD ["java", "-jar", "-Xmx1g", "-Xrs", "target/ex-fb-graph-api-1.0.jar", "-d", "/data/"] 12 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Keboola :(){:|:&};: s.r.o. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # 2 | # Makefile commands to manage and simplify development 3 | 4 | DATADIR = ${PWD}/tmp/ 5 | COMPONENTID = keboola.ex-facebook 6 | FILTER =".*" 7 | -include .fbtokens-env 8 | export 9 | export KBC_COMPONENTID=$(COMPONENTID) 10 | 11 | #starts clojure repl in a docker container 12 | repl: 13 | docker-compose run --rm --service-ports app start-docker-repl 14 | 15 | # starts bash in a docker container 16 | bash: 17 | docker-compose run --rm --entrypoint /bin/bash app 18 | 19 | # builds java app jar file - result is targe/project.jar file 20 | build-jar: 21 | boot build 22 | 23 | # build docker image from dockerfile 24 | build-image: 25 | docker build -t keboola/ex-facebook-graph-api . 26 | 27 | # runs extractor docker container - docker-runner command 28 | run-docker: 29 | docker run --rm -i -t -v $(DATADIR):/data keboola/ex-facebook-graph-api 30 | 31 | # runs extractor a compiled jar file as java app. Built via boot build 32 | # command or make build-jar command. 33 | run-jar: 34 | java -Xmx1g -jar target/ex-fb-graph-api-1.0.jar -d $(DATADIR) 35 | 36 | # runs extractor directly from boot ie runs as clojure program 37 | run-boot: 38 | boot run-extractor --args "-d $(DATADIR)" 39 | regenerate-snapshots: 40 | boot regenerate-snapshots "-f$(FILTER)" 41 | 42 | docker-test: 43 | docker-compose run app test 44 | 45 | # runs jar with visual vm profiler. Needs visualVM to be running with Start up profiler plugin 46 | run-jar-agent: 47 | java -Xmx256m -agentpath:/Applications/VisualVM.app/Contents/Resources/visualvm/profiler/lib/deployed/jdk16/mac/libprofilerinterface.jnilib=/Applications/VisualVM.app/Contents/Resources/visualvm/profiler/lib,5140 -jar target/ex-fb-graph-api-1.0.jar -d tmp/ 48 | rmi: 49 | -docker rmi -f $$(docker images -q -f "dangling=true") 50 | rm: 51 | -docker rm $$(docker ps -q -f 'status=exited') 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.com/keboola/ex-facebook-graph-api.svg?branch=master)](https://travis-ci.com/keboola/ex-facebook-graph-api) 2 | 3 | [![Docker Repository on Quay](https://quay.io/repository/keboola/ex-facebook-graph-api/status "Docker Repository on Quay")](https://quay.io/repository/keboola/ex-facebook-graph-api) 4 | 5 | ## Sample Nested Queries 6 | ##### the whole fb feed - all page posts, its likes, comments, likes of the comments, subcomments, likes of the subcomments 7 | ``` 8 | { 9 | "id": 1, 10 | "name": "scrape", 11 | "type": "nested-query", 12 | "query": { 13 | "path": "feed", 14 | "fields": "caption,message,created_time,type,description,likes{link,name,username},comments{message,created_time,from,likes{link,name,username},comments{message,created_time,from,likes{link,name,username}}}", 15 | "ids": "" 16 | } 17 | ``` 18 | ##### extract page [metrics](https://developers.facebook.com/docs/graph-api/reference/insights) `page_impressions`, `page_fans` and `page_engaged_users` for last 5 days 19 | 20 | ``` 21 | { 22 | "id": 2, 23 | "name": "page_insights", 24 | "type": "nested-query", 25 | "query": { 26 | "path": "", 27 | "fields": "insights.since(5 days ago).metric(page_impressions,page_fans,page_engaged_users)", 28 | "ids": "" 29 | } 30 | } 31 | ``` 32 | According to the last facebook graph api all insights metrics that are needed to extract must be explicitely listed in the query, i.e., there is no general get-all-metrics data query type of call. 33 | ##### extract posts [metrics](https://developers.facebook.com/docs/graph-api/reference/insights) `page_posts_impressions` and `post_impressions` for all posts: 34 | Note day we are using `since(now)` specification of insights time to get the most recent values, otherwise it may paginate over small periods of time and consume lots of request to facebook api. 35 | ``` 36 | { 37 | "id": 3, 38 | "name": "posts_insights", 39 | "type": "nested-query", 40 | "query": { 41 | "path": "feed", 42 | "fields": "insights.since(now).metric(page_posts_impressions,post_impressions)", 43 | "ids": "" 44 | } 45 | } 46 | ``` 47 | 48 | You can try the examples above by calling Facebook Graph api directly in a http client(e.g. Postman) as follows: 49 | `GET https://graph.facebook.com//?fields=&ids=&access_token=` 50 | You can get access token in [Graph Api Explorer](https://developers.facebook.com/tools/explorer). 51 | # Configuration 52 | ## Facebook Graph API 53 | This extractor extracts data from facebook graph api: 54 | https://developers.facebook.com/docs/graph-api/ 55 | . You can try it live here: https://developers.facebook.com/tools/explorer 56 | 57 | 58 | ### Nested Query 59 | 60 | Lets say that in Facebook Graph Api every endpoint represents a node in a graph. Example of a node could be /me - ie user info, me/posts - ie posts of the current user. To get data from a particullar endpoint one can make typical REST api call GET `me/posts` or make a **nested** api call that basically allows to extract the whole subtree of a node. for example GET `me?fields=posts,comments,likes` extracts all posts comments and likes of an id(in this case it is me - current user). 61 | For more info see Making Nested Queries in https://developers.facebook.com/docs/graph-api/using-graph-api#reading 62 | #### Configuring nested query 63 | In configuration under parameters there is an array of `queries`(see sample configuration). Each query besides obvious properties such as `id`, `name`, `type`(currently only nested-query type), `disabled` also contains object `query` with the following properties: 64 | - `path` : enpoint url so the absolute url will be like graph.facebook.com/version/path. Typically it is endpoint **feed**. Can be an empty string if we want to start extracting from the "root" node that is the page itself. 65 | - `fields`: fields parameter of the graph api nested-query 66 | - `ids`: comma separated list of ids(typically page-ids) that will be prepended with path. It is also a parameter of graph api. If empty string than all ids from `accounts` object will be used. Can also be completely removed from the query. 67 | - `limit` - size of one page(response). Default is 25, maximum 100. Useful when fb api returns error that the request is "too big" - in such case use smaller limit. This parameter also affects the total number of request made to fb api. 68 | - `since` - relates to the *created_time* of **path** parameter i.e., if path is "posts" then it takes all posts with *created_time* since the specified date in **since** parameter. If path is empty then it does not have any effect. Can be specified relatively, e.g. 10 days ago. 69 | - `until` - same as since above but specifies date until data with *created_time* date. 70 | 71 | 72 | The most important parameter is `fields` - tells what is going to extract. so here are few hints: 73 | - you can specify additional params of a node with dot e.g to specify since and limit params of posts: `posts.limit(100).since(2016-12-24){message,likes,comments{comments}}` 74 | - you can also specify date range using `since` or/and `until` that accepts unix timestamp values(in seconds) or date in format **yyy-mm-dd** or relative values: e.g. all posts posted in last 10 days `posts.since(10 days ago){message,likes,comments}` 75 | - relative date specification is parsed by this cool PHP function [strtotime](http://php.net/manual/en/function.strtotime.php) 76 | - if an object(e.g comments) does not have nesting specified it will extract some of its columns but once the nesting is specified e.g commens{likes} then one has to explicitely specify all its column in the nesting e.g. `comments{from,message,created_time,likes}` 77 | - for each row id is extracted autmatically and no need to be specified in the query 78 | - to extract all posts with its comments, subcomments,likes and sublikes the query would look like this: 79 | ``` 80 | { 81 | "fields": "posts{message,story,created_time,likes,comments{from,message,created_time,comments,likes}}", 82 | "path": "", 83 | "ids": "" 84 | 85 | } 86 | ``` 87 | 88 | ### Async Insights Query 89 | 90 | Sometimes when extracting ads insights via nested query returns "Please reduce the amount of data you're asking for". In such case it is better to use `async-insights-query`, that extracts ads insights asynchronously and should deal with asking bigger amount of data. The query specification is similar nested query, however it only contains 2 91 | - `parameters` - URL query string specifiying ads insights parameters as described in https://developers.facebook.com/docs/marketing-api/reference/adgroup/insights/. The parameters are separated by `&`, e.g. `fields=ad_id,actions&level=ad&action_breakdowns=action_type&date_preset=last_month&time_increment=1` 92 | - `ids` - comma separated list of ids(typically page-ids) that will be prepended with path. It is also a parameter of graph api. If empty string than all ids from `accounts` object will be used. Can also be completely removed from the query. 93 | 94 | #### Sample async insights query 95 | ``` 96 | { 97 | "id": 3, 98 | "name": "ads_async_insights", 99 | "type": "async-insights-query", 100 | "query": { 101 | "parameters": "fields=ad_id,actions&level=ad&action_breakdowns=action_type&date_preset=last_month&time_increment=1", 102 | "ids": "" 103 | } 104 | } 105 | ``` 106 | 107 | ### Sample configuration: 108 | Note that you can specify facebook api version via `api-version` parameter. Default is **v5.0**. 109 | 110 | ``` 111 | { 112 | "storage": {}, 113 | "parameters": { 114 | "accounts": { 115 | "": { 116 | "id": "", 117 | "name": "my fancy page", 118 | "category": "entertainment" 119 | }, 120 | "": { 121 | "id": "", 122 | "name": "keboola", 123 | "category": "software" 124 | } 125 | }, 126 | "api-version": "v5.0", 127 | "queries": [ 128 | { 129 | "id": 1, 130 | "name": "qname", 131 | "type": "nested-query", 132 | "query": { 133 | "path": "feed", 134 | "fields": "message,story,likes,comments{from}", 135 | "ids": "," 136 | } 137 | }, 138 | { 139 | "id": 3, 140 | "name": "ads_async_insights", 141 | "type": "async-insights-query", 142 | "query": { 143 | "parameters": "fields=ad_id,actions&level=ad&action_breakdowns=action_type&date_preset=last_month&time_increment=1", 144 | "ids": "" 145 | } 146 | } 147 | ] 148 | }, 149 | "authorization": { 150 | "oauth_api": { 151 | "id": "{OAUTH_API_ID}" 152 | } 153 | } 154 | } 155 | ``` 156 | 157 | ## Result tables description 158 | For each query extractor generates a number of tables prefixed with query name. Each table represents one type of node so typically tables would be `queryname_post`, `queryname_likes`, `queryname_comments` `queryname_insights`. Same nested structure type will be in the same table. So for example comments and subcomments will be in the same table `comments`. Every table has different columns but the following columns will always be the same: 159 | - **id**: unique id of the row, some tables may not have it, e.g. ads insights 160 | - **parent_id**: the parent node id, e.g. comments parent-id refers to a post row id 161 | - **ex_account_id**: top root source under the extraction is beign done, it could be a page id or ads account id and it is specified in config under `ids` property. 162 | - **fb_graph_node**: describes the row "vertical position" of the resulting tree. e.g for comments it will be `page_feed_comments`, for subcomments(i.e. comments of comments) it will be `page_feed_comments_comments` etc 163 | - insights data objects will be flatten into columns `key1`, `key2` and `value` along with columns metric name, title, description etc 164 | 165 | 166 | # Authorization 167 | Register component api to oauth-bundle-v2 by calling POST to https://syrup.keboola.com/oauth-v2/manage with manage-token, storage-api token in the header and body: 168 | 169 | ``` 170 | { 171 | "component_id": "keboola.ex-facebook-insights-v2", 172 | "friendly_name": "Facebook Insights", 173 | "app_key": "xxx", 174 | "app_secret": "xxx", 175 | "oauth_version": "facebook", 176 | "permissions": "manage_pages,public_profile,read_insights,pages_show_list", 177 | "graph_api_version": "v2.8" 178 | 179 | } 180 | ``` 181 | 182 | more info about authorization registration here: https://github.com/keboola/oauth-v2-bundle 183 | 184 | 185 | # Development 186 | The app is written in Clojure(1.8), evaluated and build via [Boot-clj](https://github.com/boot-clj/boot#install) which requires [Java Development Kit](http://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html). 187 | To try the app locally check target commands in the [Makefile](Makefile) 188 | For example to build and run the app locally type from the repo root: 189 | 190 | `make build-jar run-jar` 191 | 192 | ## License 193 | 194 | MIT licensed, see [LICENSE](./LICENSE) file. 195 | -------------------------------------------------------------------------------- /build.boot: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env boot 2 | (set-env! 3 | :source-paths #{"src" "test"} 4 | :dependencies '[ 5 | [org.clojure/clojure "1.11.1"] 6 | [cheshire "5.11.0"] 7 | [clj-http "3.12.3"] 8 | [de.ubercode.clostache/clostache "1.4.0"] 9 | [org.clojure/tools.cli "0.4.1"] 10 | [semantic-csv "0.2.0"] 11 | [org.clojure/data.csv "1.0.0"] 12 | [org.clojure/data.json "2.4.0"] 13 | [org.clojure/test.check "1.1.0" :scope "test"] 14 | [org.clojure/core.async "1.5.648"] 15 | [adzerk/boot-test "1.2.0" :scope "test"] 16 | [clj-http-fake "1.0.3"] 17 | [slingshot "0.12.2"] 18 | [clj-time "0.15.2"] 19 | ]) 20 | 21 | 22 | (require '[adzerk.boot-test :refer :all]) 23 | (require '[keboola.facebook.extractor.core]) 24 | 25 | (deftask run-extractor 26 | "run extractor" 27 | [x args VAL str "arguments string for main- function"] 28 | (if-not args 29 | (do (boot.util/fail "arguments string x is requried. ") 30 | (*usage*))) 31 | (do 32 | (require '[keboola.facebook.extractor.core]) 33 | ((resolve 'keboola.facebook.extractor.core/-main) args))) 34 | 35 | (deftask generate-test 36 | "given data dir with config.json, this task runs extraktor, 37 | record api calls, create snapshot tests with recrded api calls and compare result dirs" 38 | [d data VAL str "name of directory in test/keboola/snapshots containing config.json" 39 | s skip-token bool "skip token anonymization in config.json"] 40 | (if-not data 41 | (do (boot.util/fail "arguments string d is requried. ") 42 | (*usage*))) 43 | (do(require '[keboola.snapshots.core]) 44 | ((resolve 'keboola.snapshots.core/generate-test) data (not skip-token)))) 45 | 46 | (deftask regenerate-snapshots [f dirfilter VAL str "regexp to filter dirs to process"] 47 | (require '[keboola.snapshots.core]) 48 | ((resolve 'keboola.snapshots.core/regenerate-all-snapshot-dirs) dirfilter)) 49 | 50 | (deftask build 51 | "Builds an uberjar extractor that can be run with java -jar" 52 | [] 53 | (comp 54 | (aot :all true) 55 | (pom :project 'ex-fb-graph-api 56 | :version "1.0") 57 | (uber) 58 | (jar :main 'keboola.facebook.extractor.core) 59 | (target :dir #{"target"}))) 60 | 61 | (deftask start-docker-repl 62 | "run repl server on 1111 port" 63 | [] 64 | (require 'boot.repl) 65 | (swap! boot.repl/*default-dependencies* 66 | concat '[[cider/cider-nrepl "0.15.1"]]) 67 | (swap! boot.repl/*default-middleware* 68 | conj 'cider.nrepl/cider-middleware) 69 | (repl :bind "0.0.0.0" :port 1111) 70 | ) 71 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | ### 0.0.6 2 | - list accounts 3 | - save account-id 4 | - various fixes 5 | ### 3.7.0 6 | - debug token sync action 7 | ### 3.8.0 8 | - log debug token info on every run action 9 | ### 3.11.0 10 | - igaccounts - get instagram linked accounts sync action 11 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "2" 2 | services: 3 | app: 4 | build: . 5 | entrypoint: "boot" 6 | image: keboola/ex-facebook-graph-api 7 | volumes: 8 | - ./:/code 9 | -------------------------------------------------------------------------------- /src/keboola/docker/config.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.docker.config 2 | (:require [cheshire.core :refer [parse-string]] 3 | [clojure.string :refer [trim]])) 4 | 5 | 6 | (def default-dir "/data/") 7 | 8 | (defn check-path [path] 9 | (if (clojure.string/ends-with? path "/") 10 | (trim path) 11 | (str (trim path) "/"))) 12 | 13 | (defn- load-config-once 14 | ([] (load-config-once default-dir)) 15 | ([datadir] 16 | (let [dirpath (check-path datadir) 17 | file-content (slurp (trim (str dirpath "config.json")))] 18 | (parse-string file-content true)))) 19 | 20 | (def load-config load-config-once) 21 | 22 | (defn mkdirp [path] 23 | (let [dir (java.io.File. path)] 24 | (if-not (.exists dir) 25 | (.mkdirs dir)) 26 | path)) 27 | 28 | (defn out-dir-path 29 | ([] 30 | (out-dir-path "./")) 31 | ([datadir] 32 | (let [result (mkdirp (str (check-path (trim datadir)) "out/tables/"))] 33 | result))) 34 | 35 | 36 | (defn config [& datadir] 37 | (apply load-config datadir)) 38 | 39 | 40 | (defn parameters [& datadir] 41 | (:parameters (apply load-config datadir))) 42 | 43 | 44 | (defn app-access-token [& datadir] 45 | (let 46 | [auth-info (get-in 47 | (apply load-config datadir) 48 | [:authorization :oauth_api :credentials]) 49 | app-secret (:#appSecret auth-info) 50 | app-id (:appKey auth-info)] 51 | (str app-id "|" app-secret))) 52 | 53 | (defn user-credentials [& datadir] 54 | (let 55 | [data (get-in 56 | (apply load-config datadir) 57 | [:authorization :oauth_api :credentials :#data])] 58 | (parse-string data true))) 59 | 60 | (defn get-fb-token [user-credentials] 61 | (:access_token user-credentials (:token user-credentials))) 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/keboola/docker/runtime.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.docker.runtime 2 | (:require [cheshire.core :refer [generate-string]])) 3 | 4 | 5 | (defn exit [status] 6 | (System/exit status)) 7 | 8 | 9 | (defn log-error [& error-msg] 10 | (binding [*out* *err*] 11 | (apply println error-msg))) 12 | 13 | 14 | (defn user-error [error-msg] 15 | (log-error error-msg) 16 | (exit 1)) 17 | 18 | 19 | (defn app-error [error-msg] 20 | (log-error error-msg) 21 | (exit 2)) 22 | 23 | 24 | (defn log-strings [& strings] 25 | (apply println (map 26 | #(if (map? %) 27 | ; pretty print maps 28 | (generate-string % {:pretty true}) 29 | ; else just print 30 | %) 31 | strings))) 32 | 33 | (defn log [what] 34 | (println what)) 35 | 36 | (defn log-error-and-exit [what] 37 | (log-error what) 38 | (exit 0)) 39 | 40 | (defn get-component-id [] 41 | (System/getenv "KBC_COMPONENTID")) 42 | 43 | (defn keboola-ex-facebook-component? [] 44 | (= "keboola.ex-facebook" (get-component-id))) 45 | 46 | (defn save-manifest [csvfile-path body] 47 | (let [manifest (select-keys body [:destination :columns :incremental :primary_key :delimiter :enclosure]) 48 | manifest-path (str csvfile-path ".manifest")] 49 | (spit manifest-path (generate-string manifest)))) 50 | -------------------------------------------------------------------------------- /src/keboola/facebook/api/exponential_backoff.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.api.exponential-backoff 2 | (:require [keboola.docker.runtime :as runtime])) 3 | 4 | (def time-slot-ms 1000) 5 | (def truncate 6) ; sleep for 64 seconds at most (2^6) 6 | (def MAX_WAIT_TIME (* 1000 60 60 24)) ; poll for 24 hours at most 7 | 8 | (defn with-exp-backoff [action!] 9 | (loop [c 0 10 | waited 0] 11 | (let [slot (* time-slot-ms (dec (Math/pow 2 c)))] 12 | (when (> waited MAX_WAIT_TIME) 13 | (runtime/user-error (str "Polling timeout exceeded:" waited))) 14 | (Thread/sleep slot) 15 | (when (not (action!)) 16 | (recur (if (>= c truncate) c (inc c)) (+ waited slot)))))) 17 | 18 | (defn- try-n-times [f n] 19 | (if (zero? n) 20 | (f) 21 | (try 22 | (f) 23 | (catch Throwable _ 24 | (Thread/sleep (* 1000 10)) 25 | (try-n-times f (dec n)))))) 26 | 27 | (defn try-3-times [f] 28 | (try-n-times f 3)) 29 | -------------------------------------------------------------------------------- /src/keboola/facebook/api/parser.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.api.parser 2 | (:require 3 | [keboola.docker.runtime :refer [app-error]] 4 | [clj-time.core :as t] 5 | [clojure.data.json :as json] 6 | [clj-time.format :refer [formatter unparse]] 7 | [clojure.string :as string])) 8 | 9 | (defn relative-days-timestamp [days] 10 | (unparse (formatter "YYYY-MM-dd") (t/plus (t/now) (t/days (Integer/parseInt days))))) 11 | 12 | (defn preparse-fields [fields-str] 13 | (string/replace fields-str #"%%days:-?\d+%%" 14 | #(relative-days-timestamp (re-find #"-?\d+" %)))) 15 | 16 | (defn nested-object? 17 | "Returns true if objet is map and contains :data keyword" 18 | [object] 19 | (and (map? object) (contains? object :data))) 20 | 21 | (defn extract-summary [object-name object row params] 22 | (if (contains? object :summary) 23 | {:name "summary" 24 | :data {:data [(:summary object)]} 25 | :parent-id (or (:id row) (:parent-id params)) 26 | :fb-graph-node (str (:fb-graph-node params) "_" object-name)})) 27 | 28 | (defn get-nested-objects 29 | "Traverse body-data array and take out nested-object like structures. 30 | Return array of objects with keys :name :data :parent-id :fb-graph-node " 31 | [body-data params] 32 | (reduce 33 | (fn [memo, row] 34 | (if-let [objects (reduce-kv (fn [m k v] 35 | (if (nested-object? v) 36 | (conj m 37 | {:name (name k) 38 | :data v 39 | :parent-id (or (:id row) (:parent-id params)) 40 | :fb-graph-node (str (:fb-graph-node params) "_" (name k))} 41 | 42 | (extract-summary (name k) v row params)) m)) 43 | [] row)] 44 | (concat memo (keep identity objects)) 45 | memo)) 46 | [] body-data)) 47 | 48 | (defn flatten-value-object 49 | "flattens object values and prepends and prepend key1" 50 | [key1 object-value] 51 | (cond 52 | (map? object-value) (map (fn [[key2 val]] {:key1 key1 :key2 (name key2) :value val}) 53 | object-value) 54 | :else (list {:key1 key1 :key2 "" :value object-value}))) 55 | 56 | (defn flatten-array-value [item end_time] 57 | (map 58 | #(assoc % :end_time end_time) 59 | (cond 60 | (map? item) (mapcat (fn [[key1 val]] (flatten-value-object (name key1) val)) item) 61 | :else (list {:key1 "" :key2 "" :value item})))) 62 | 63 | (def ads-action-stats-types #{:actions :properties :conversion_values 64 | :action_values :canvas_component_avg_pct_view 65 | :cost_per_10_sec_video_view :cost_per_action_type :cost_per_unique_action_type 66 | :unique_actions :video_10_sec_watched_actions :video_15_sec_watched_actions 67 | :video_30_sec_watched_actions :video_avg_pct_watched_actions 68 | :video_avg_percent_watched_actions :video_avg_sec_watched_actions 69 | :video_avg_time_watched_actions :video_complete_watched_actions 70 | :video_p100_watched_actions :video_p25_watched_actions 71 | :video_p50_watched_actions :video_p75_watched_actions :cost_per_conversion :cost_per_outbound_click 72 | :video_p95_watched_actions :website_ctr :website_purchase_roas :purchase_roas :outbound_clicks :conversions :video_play_actions :video_thruplay_watched_actions}) 73 | 74 | (def serialized-lists-types #{:issues_info :frequency_control_specs}) 75 | 76 | (defn flatten-array 77 | "flattens array of object with same structure prefixing its keys with array-name 78 | returns list of key-value pairs" 79 | [array array-name] 80 | (cond (= array-name :values) 81 | (mapcat #(flatten-array-value (:value %) (:end_time %)) array) 82 | (some? (ads-action-stats-types array-name)) 83 | (map #(assoc % :ads_action_name (name array-name)) array) 84 | (some? (serialized-lists-types array-name)) 85 | (list (assoc {} array-name (json/write-str array))) 86 | (and (= array-name :media) (empty? array)) '() 87 | :else (app-error (str "unsuported array:" array-name array)))) 88 | 89 | (defn filter-scalars [row] 90 | (into {} (filter (fn [[k v]] 91 | (and (-> v map? not) (-> v sequential? not))) 92 | row))) 93 | 94 | (defn filter-flatten-objects 95 | [row] 96 | (let [simple-objects (filter (fn [[k v]] (and (-> v map?) (-> v nested-object? not))) row)] 97 | (apply hash-map 98 | (mapcat (fn [[object-name object]] 99 | (mapcat (fn [[k v]] 100 | (list (keyword (str (name object-name) "_" (name k))) v)) 101 | object)) 102 | simple-objects)))) 103 | -------------------------------------------------------------------------------- /src/keboola/facebook/extractor/core.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.extractor.core 2 | (:gen-class) 3 | (:require [clojure.string :as string] 4 | [clojure.tools.cli :refer [parse-opts]] 5 | [keboola.docker.config :as docker-config] 6 | [keboola.docker.runtime 7 | :as 8 | docker-runtime 9 | :refer 10 | [app-error log log-error-and-exit log-strings user-error]] 11 | [keboola.facebook.extractor.query :as query] 12 | [keboola.facebook.extractor.sync-actions :as sync-actions] 13 | [keboola.http.client :refer [fb-requests-count]] 14 | [keboola.http.recording :refer [turn-log-responses-on]] 15 | [keboola.utils.json-to-csv :as csv] 16 | [slingshot.slingshot :refer [throw+ try+]])) 17 | 18 | (def cli-options [["-d" "--dataDir path" "Path to data directory e.g. /data"]]) 19 | 20 | (defn usage [options-summary] 21 | (->> ["Keboola Facebook Graph Api Extractor" 22 | "Usage: program-name options" 23 | "Options:" 24 | options-summary] 25 | (string/join \newline))) 26 | 27 | (defn treat [fn-to-treat] 28 | (try+ 29 | (fn-to-treat) 30 | (catch #(and (some? (:status %)) (number? (:status %)) (<= 400 (:status %) 550)) e 31 | (let [msg (:body e)] 32 | (if (.contains msg "User request limit reached") 33 | (log-error-and-exit (str "REQUEST LIMIT REACHED. Up to now extracted data will be uploaded to storage." msg)) 34 | (user-error (str "Facebook api error:" msg))))) 35 | (catch Throwable e 36 | (app-error (str "unexpected error:" (with-out-str (clojure.stacktrace/print-stack-trace e))))) 37 | (catch Object e 38 | (app-error (str "unexpected error:" (str e)))))) 39 | 40 | (defn make-accounts-csv [parameters out-dir] 41 | (let [filepath (str out-dir "accounts") 42 | accounts (:accounts parameters) 43 | header (vec (set (apply concat (map #(keys (dissoc (second %) :access_token)) accounts)))) ;[:id :name :category] 44 | data (into [] (map (fn [[k v]] (select-keys v header)) accounts))] 45 | (log "writing accounts table") 46 | (csv/write filepath header data))) 47 | 48 | (defn run [credentials parameters out-dir app-access-token] 49 | (let [version (:api-version parameters) 50 | all-ids (apply str (interpose "," (map name (keys (:accounts parameters)))))] 51 | (make-accounts-csv parameters out-dir) 52 | (sync-actions/log-debug-token app-access-token credentials "Access token info:") 53 | (dorun (map (fn [query] 54 | (if (:disabled query) 55 | (log-strings "Skipping query" (:name query)) 56 | ; else run query: 57 | (query/run-query (assoc query :api-version version) all-ids credentials out-dir))) 58 | (:queries parameters)))) 59 | (log-strings "Finished, total count of requests to facebook api:" @fb-requests-count)) 60 | 61 | (defn prepare-and-run [datadir] 62 | (let [parameters (docker-config/parameters datadir) 63 | config (docker-config/config datadir) 64 | action (:action config) 65 | app-access-token (docker-config/app-access-token datadir) 66 | out-dir-path (docker-config/out-dir-path datadir) 67 | credentials (docker-config/user-credentials datadir)] 68 | (when (:debug-mode parameters) 69 | (log "Running in debug mode") 70 | (sync-actions/disable-log-token) 71 | (turn-log-responses-on)) 72 | (cond 73 | (empty? credentials) (docker-runtime/user-error "Missing facebook credentials") 74 | (empty? (docker-config/get-fb-token credentials)) (docker-runtime/user-error "Missing facebook token")) 75 | (case action 76 | "debugtoken" (sync-actions/log-debug-token app-access-token credentials nil) 77 | "accounts" (sync-actions/accounts credentials config) 78 | "adaccounts" (sync-actions/adaccounts credentials config) 79 | "igaccounts" (sync-actions/igaccounts credentials config) 80 | (treat #(run credentials parameters out-dir-path app-access-token))))) 81 | 82 | (defn -main [& args] 83 | (let [{:keys [options summary errors] :as parsed-args} (parse-opts args cli-options)] 84 | (cond 85 | (not-empty errors) (docker-runtime/user-error (string/join \newline errors)) 86 | (empty? (:dataDir options)) (docker-runtime/user-error (usage summary))) 87 | (prepare-and-run (:dataDir options)))) 88 | -------------------------------------------------------------------------------- /src/keboola/facebook/extractor/output.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.extractor.output 2 | (:require [keboola.utils.json-to-csv :as csv] 3 | [keboola.docker.runtime :as runtime] 4 | [keboola.docker.config :refer [mkdirp]] 5 | [clojure.string :refer [split]] 6 | [clj-time.coerce :refer [to-long]] 7 | [clj-time.core :refer [now]] 8 | [slingshot.slingshot :refer [try+ throw+]] 9 | [clojure.core.async :as async :refer [onto-chan >! !! row :keboola :table-name)) 62 | 63 | (defn add-id-coloumns [row] 64 | (assoc row 65 | :ex-account-id (-> row :keboola :ex-account-id) 66 | :parent-id (-> row :keboola :parent-id) 67 | :fb-graph-node (-> row :keboola :fb-graph-node))) 68 | 69 | (def columns-map (atom {})) 70 | (defn reset-columns-map [] (reset! columns-map {})) 71 | 72 | (defn write-manifest [manifest-path columns is-write? table-name context async-insights? incremental?] 73 | (if is-write? 74 | (let [manifest (if incremental? 75 | {:incremental true :primary_key (get-primary-key columns table-name context async-insights?) :columns columns} 76 | {:incremental false :columns columns})] 77 | (if (not (contains? @columns-map manifest-path)) 78 | (do 79 | (runtime/save-manifest manifest-path manifest) 80 | (swap! columns-map assoc manifest-path columns)))))) 81 | 82 | (defn prepare-header [rows manifest-path] 83 | (let [columns (set (mapcat #(-> % (dissoc :keboola) keys) rows)) 84 | all-columns (conj columns :parent-id :fb-graph-node :ex-account-id) 85 | new-columns (sort-columns all-columns) 86 | has-data-columns (not-empty (disj (set new-columns) :id :ex-account-id :fb-graph-node :parent-id)) 87 | saved-columns (@columns-map manifest-path)] 88 | (if has-data-columns ;else return nil -> no data to write 89 | (if saved-columns ;else returns new-columns 90 | (if (= saved-columns new-columns) ;else throws error 91 | ; return saved columns from previous query 92 | saved-columns 93 | ;else throw error on columns mismatch 94 | (throw+ (str "columns mismatch, original write columns: " saved-columns " ,current write columns: " new-columns))) 95 | ; else return newly computed columns 96 | new-columns)))) 97 | ; by default return nil saying we have no data to save 98 | 99 | 100 | (defn flush-buffer [csv-file manifest-path table-name memo async-insights? incremental?] 101 | (let [first-write? (:first-write? memo) 102 | header (if first-write? 103 | (prepare-header (:buffer memo) manifest-path) 104 | (:header memo)) 105 | context (-> memo :buffer first :keboola)] 106 | (if header 107 | (do 108 | (write-manifest manifest-path header first-write? table-name context async-insights? incremental?) 109 | (csv/write-to-file csv-file header (:buffer memo) false) 110 | (if (= (mod (:cnt memo) chan-buffer-size) 0) 111 | (runtime/log-strings "Written" (:cnt memo) "rows to" table-name)) 112 | ;return header 113 | header) 114 | ; else has only ids columns return nil header 115 | (do 116 | (if first-write? (runtime/log-strings "Skipping table" table-name "containing only id and parent-id columns")) 117 | nil)))) 118 | 119 | (defn process-row [row csv-file manifest-path table-name memo async-insights? incremental?] 120 | (if (= (count (:buffer memo)) chan-buffer-size) 121 | (let [header (flush-buffer csv-file manifest-path table-name memo async-insights? incremental?)] 122 | {:cnt (inc (:cnt memo)) :header header :buffer (list row) :first-write? false}) 123 | ;else part 124 | {:cnt (inc (:cnt memo)) 125 | :header (:header memo) 126 | :buffer (conj (:buffer memo) row) 127 | :first-write? (:first-write? memo)})) 128 | 129 | (defn create-write-thread [table-name input-ch out-dir qname-prefix async-insights? incremental?] 130 | (async/thread 131 | (try+ 132 | (let [kbc-table-name (if (= (last (split qname-prefix #"_")) table-name) 133 | qname-prefix 134 | (str qname-prefix "_" table-name)) 135 | sliced-dir-path (str out-dir kbc-table-name) 136 | sliced-file-name (str (to-long (now))) 137 | sliced-file-path (str sliced-dir-path "/" sliced-file-name)] 138 | (mkdirp sliced-dir-path) 139 | (with-open [out-file (io/writer sliced-file-path)] 140 | (loop [memo {:cnt 0 :buffer '() :header {} :first-write? true}] 141 | (let [row ( don't call recur, 145 | (if (some? (flush-buffer out-file sliced-dir-path table-name memo async-insights? incremental?)) 146 | (runtime/log-strings "Total written" (:cnt memo) "rows to table" table-name)))))) 147 | ;; thread terminates 148 | 149 | (delete-file-if-empty sliced-file-path) 150 | (delete-dir-if-empty sliced-dir-path)) 151 | ; return true as succesfull finish of thread 152 | {:return true} 153 | (catch Object e 154 | {:return false :error (str "failed write " table-name " with error: " e)})))) 155 | 156 | (defn create-tables-map [tables-names value-fn] 157 | (into {} (map #(vector % (value-fn %)) tables-names))) 158 | 159 | (defn close-channels [table-map thread-chans-vec error-strategy] 160 | (doseq [[_ c] table-map] 161 | (close! c)) 162 | (while (when-let [return-value (async/> (s/split params-str #"&") 15 | (map #(s/split % #"=" 2)) 16 | (map (fn [[k v]] [k v])) 17 | (into {})))) 18 | 19 | ;; Parses the time_ranges parameter into a vector of maps. 20 | (defn parse-time-ranges [time-ranges-str] 21 | (if (empty? time-ranges-str) 22 | [] 23 | (let [json-str (s/replace time-ranges-str #"'" "\"")] 24 | (json/read-str json-str :key-fn keyword)))) 25 | 26 | ;; Generates a sequence of dates from start-date to end-date exclusive 27 | (defn date-range [start-date end-date] 28 | (let [formatter (DateTimeFormatter/ofPattern "yyyy-MM-dd") 29 | start (LocalDate/parse start-date formatter) 30 | end (LocalDate/parse end-date formatter)] 31 | (cond 32 | ;; If start date is before end date, return dates up to but not including end date 33 | (.isBefore start end) 34 | (take-while (fn [date] (.isBefore date end)) 35 | (iterate #(.plusDays % 1) start)) 36 | ;; If start date equals end date, return a sequence containing that date 37 | (.isEqual start end) 38 | [start] 39 | ;; If start date is after end date, return an empty sequence 40 | :else 41 | []))) 42 | 43 | ;; Generates a list of time ranges for each day within the given date range. 44 | (defn generate-date-ranges [start-date end-date] 45 | 46 | (let [dates (date-range start-date end-date)] 47 | (map (fn [date] 48 | {:since (.toString date) 49 | :until (.toString date)}) 50 | dates))) 51 | 52 | ;; Expands all time ranges into individual day ranges. 53 | (defn expand-time-ranges [time-ranges] 54 | (mapcat (fn [time-range] 55 | (generate-date-ranges (:since time-range) (:until time-range))) 56 | time-ranges)) 57 | 58 | ;; Generates a new parameters string with the updated time_range. 59 | (defn generate-parameters-for-dates [parameters-map time-range] 60 | (let [time-ranges-json (json/write-str [time-range] :value-fn (fn [k v] (if (string? v) v (str v)))) 61 | ;; Replace double quotes with single quotes 62 | time-ranges-json-single-quotes (s/replace time-ranges-json #"\"" "'") 63 | params-with-new-time-range (-> parameters-map 64 | (dissoc "date_preset") 65 | (assoc "time_ranges" time-ranges-json-single-quotes))] 66 | (->> params-with-new-time-range 67 | (map (fn [[k v]] (str k "=" v))) 68 | (s/join "&")))) 69 | 70 | ;; Converts date_preset values to the number of days. 71 | (defn date-preset-to-days [date-preset] 72 | (case date-preset 73 | "last_3d" 3 74 | "last_7d" 7 75 | "last_30d" 30 76 | nil)) 77 | 78 | ;;"Returns the date string for 'days' days ago from the given reference date. 79 | ;; If reference-date is not provided, uses LocalDate/now." 80 | (defn get-past-date 81 | ([days] 82 | (get-past-date days (LocalDate/now))) 83 | ([days reference-date] 84 | (let [formatter (DateTimeFormatter/ofPattern "yyyy-MM-dd")] 85 | (.format (.minusDays reference-date days) formatter)))) -------------------------------------------------------------------------------- /src/keboola/facebook/extractor/sync_actions.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.extractor.sync-actions 2 | (:require [keboola.facebook.api.request :as request] 3 | [cheshire.core :refer [generate-string]] 4 | [slingshot.slingshot :refer [try+ throw+]] 5 | [keboola.docker.config :as docker-config] 6 | [keboola.docker.runtime :refer [log]])) 7 | 8 | (def log-token? (atom true)) 9 | (defn disable-log-token [] (reset! log-token? false)) 10 | 11 | (defn accounts [credentials config] 12 | (try+ 13 | (let [token (docker-config/get-fb-token credentials) 14 | version (-> config :parameters :api-version) 15 | accounts (mapv #(dissoc % :access_token) (request/get-accounts token :version version))] 16 | (log (generate-string accounts))) 17 | (catch Object e 18 | (log (generate-string {:code (:status e 500) :error (:body e)}))))) 19 | 20 | (defn adaccounts [credentials config] 21 | (try+ 22 | (let [token (docker-config/get-fb-token credentials) 23 | version (-> config :parameters :api-version) 24 | accounts (request/get-adaccounts token :version version)] 25 | (log (generate-string accounts))) 26 | (catch Object e 27 | (log (generate-string {:code (:status e 500) :error (:body e)}))))) 28 | 29 | (defn igaccounts [credentials config] 30 | (try+ 31 | (let [token (docker-config/get-fb-token credentials) 32 | version (-> config :parameters :api-version) 33 | accounts (request/get-igaccounts token :version version) 34 | ig-accounts (filter #(contains? % :instagram_business_account) accounts) 35 | result (map #(assoc (select-keys % [:name :category]) :id (-> % :instagram_business_account :id) :fb_page_id (:id %)) ig-accounts)] 36 | (log (generate-string result))) 37 | (catch Object e 38 | (log (generate-string {:code (:status e 500) :error (:body e)}))))) 39 | 40 | (defn log-debug-token [app-token credentials prepend-message] 41 | (try+ 42 | (if @log-token? 43 | (let [input-token (docker-config/get-fb-token credentials) 44 | response-data (:data (request/debug-token app-token input-token)) 45 | result (dissoc response-data :app_id)] 46 | (log (str prepend-message (generate-string result))))) 47 | (catch Object e 48 | (log (generate-string {:message "Failed to log token info" :error (:body e)}))))) 49 | -------------------------------------------------------------------------------- /src/keboola/http/client.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.http.client 2 | (:require [clj-http.client :as http] 3 | [clojure.string :as str] 4 | [keboola.http.recording :refer [record-request]] 5 | [keboola.docker.runtime :as runtime])) 6 | 7 | (def MAX_TRY_COUNT 4) 8 | 9 | (def fb-requests-count (atom 0)) 10 | 11 | ; retry handler for IOExceptions 12 | (defn retry-handler [ex try-count http-context] 13 | (if (> MAX_TRY_COUNT try-count) 14 | (do 15 | (Thread/sleep (* 1000 (Math/pow try-count 2))) 16 | (runtime/log-strings "retrying request[" (str try-count) "]") 17 | true) 18 | ; else return false 19 | false)) 20 | 21 | (defn check-fb-requests [url] 22 | (if (str/starts-with? url "https://graph.facebook.com") 23 | (do 24 | (swap! fb-requests-count inc) 25 | (if (= 0 (mod @fb-requests-count 500)) (runtime/log-strings "Made" @fb-requests-count "requests to facebook api so far."))))) 26 | 27 | (defn- make-request [method url & rest] 28 | (check-fb-requests url) 29 | (method url (assoc (apply hash-map rest) :retry-handler retry-handler))) 30 | 31 | (defn GET [url & rest] 32 | (let [response (apply make-request http/get url rest)] 33 | (record-request response :get url rest))) 34 | ; (println "response" response) 35 | 36 | 37 | (defn POST [url & rest] 38 | (let [response (apply make-request http/post url rest)] 39 | (record-request response :post url rest))) 40 | -------------------------------------------------------------------------------- /src/keboola/http/recording.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.http.recording 2 | (:require [clojure.walk :refer [postwalk postwalk-demo]] 3 | [cheshire.core :refer [generate-string]])) 4 | 5 | (def recording (atom '())) 6 | (def do-recording? (atom false)) 7 | (def do-log-responses? (atom false)) 8 | 9 | 10 | (defn reset-recording [] (reset! recording '())) 11 | 12 | (defn turn-recording-on [] (reset! do-recording? true)) 13 | (defn turn-recording-off [] (reset! do-recording? false)) 14 | 15 | (defn turn-log-responses-on [] (reset! do-log-responses? true)) 16 | 17 | (def VALID-CHARS 18 | (map char (concat (range 48 58) ; 0-9 19 | (range 66 91) ; A-Z 20 | (range 97 123)))) ; a-z 21 | 22 | (defn random-char [] 23 | (nth VALID-CHARS (rand (count VALID-CHARS)))) 24 | 25 | (defn random-str [length] 26 | (apply str (take length (repeatedly random-char)))) 27 | 28 | (def keywords-to-anonymize #{:name :story :caption :message :description :title :account_name :campaign_name :ad_name :impressions}) 29 | (defn- anonymize-item [item] 30 | (if (and 31 | (vector? item) 32 | (= (count item) 2) 33 | (keywords-to-anonymize (first item))) 34 | ;return anonymized string value for key value pair where key is 35 | ;from keywords-to-anonymize 36 | [(first item) (random-str 5)] 37 | ; else return item untouched 38 | item)) 39 | 40 | (defn anonymize-map [m] 41 | (postwalk anonymize-item m)) 42 | 43 | 44 | (defn replace-token-by-regexp [item] 45 | (cond 46 | (string? item) 47 | (clojure.string/replace item #"access_token=[^&]*" "access_token=TOKEN") 48 | 49 | (map? item) 50 | (if (contains? item :access_token) 51 | (assoc item :access_token "TOKEN") 52 | item) 53 | 54 | :else item)) 55 | 56 | (defn record-request [response method url request-rest] 57 | (when @do-log-responses? 58 | (let [result-map {:method method 59 | :url url 60 | :response {:status (:status response) :body (:body response)}} 61 | result (postwalk replace-token-by-regexp result-map)] 62 | (println (generate-string result {:pretty true}))) 63 | (Thread/sleep 350)) 64 | (if @do-recording? 65 | (let [request-base {:method method :address url} 66 | request (merge request-base (apply hash-map request-rest)) 67 | anonymized-response (update-in response [:body] anonymize-map) 68 | body-string-response (update-in anonymized-response [:body] generate-string)] 69 | (swap! recording conj {:response body-string-response 70 | :request request}) 71 | anonymized-response) 72 | response)) 73 | 74 | (defn replace-token [item token] 75 | (if (string? item) 76 | (clojure.string/replace item token "XXTOKENXX") 77 | item)) 78 | 79 | (defn pprint [what] 80 | (with-out-str (clojure.pprint/pprint what))) 81 | 82 | (defn prepare-recording [token] 83 | (apply str (mapcat (fn [r] 84 | (let [request (postwalk #(replace-token % token) (:request r)) 85 | response (postwalk #(replace-token % token) (:response r)) 86 | shaved-response (select-keys response [:status :body])] 87 | [(pprint request) 88 | (str "(fn [req]" (pprint shaved-response) ")")])) 89 | @recording))) 90 | 91 | (defn save-current-recording [path namespace-name token-to-replace] 92 | (let [ns-str (str "(ns " namespace-name ")\n") 93 | recording-str (str "\n{\n" (prepare-recording token-to-replace) "\n}")] 94 | (with-open [w (clojure.java.io/writer path)] 95 | (.write w (str ns-str (str "(def recorded " recording-str ")")))))) 96 | -------------------------------------------------------------------------------- /src/keboola/utils/json_to_csv.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.utils.json-to-csv 2 | (:require [semantic-csv.core :as csv] 3 | [clojure.data.csv :as cd-csv] 4 | [clojure.set :refer [rename-keys]] 5 | [clojure.spec.alpha :as s 6 | ] 7 | [clojure.string])) 8 | 9 | ; (mapcat #(conj '(:aa) %) (filter #(.contains (name %) "-") (keys {:a-id 2 :b 2}))) => (:a-id :aa) 10 | 11 | (s/fdef replace-dash 12 | :args (s/cat :kw keyword?) 13 | :fn #(not (clojure.string/includes? (:ret %) "-")) 14 | :ret keyword?) 15 | (defn replace-dash [kw] 16 | (keyword (clojure.string/replace (name kw) #"-" "_"))) 17 | 18 | (s/fdef prepare-kw-map 19 | :args (s/cat :m (s/map-of keyword? (s/or :string string? :int int?))) 20 | :fn (fn [val] (every? #(contains? (:ret val) %) 21 | (-> val :args :m keys))) 22 | :ret (s/map-of keyword? keyword?)) 23 | (defn prepare-kw-map [m] 24 | (apply hash-map (mapcat #(conj '() (replace-dash %) %) (keys m)))) 25 | 26 | (s/fdef underscorize 27 | :args (s/cat :coll (s/coll-of 28 | (s/map-of keyword? (s/or :string string? :int int?) :max-count 20) 29 | :max-count 20)) 30 | :fn (fn [val] 31 | (every? (fn [m] (every? #(or 32 | (not (clojure.string/includes? (str %) "-")) 33 | (clojure.string/includes? (str %) "_")) 34 | (keys m))) 35 | (:ret val))) 36 | :ret (s/coll-of (s/map-of keyword? (s/or :string string? :int int?)))) 37 | (defn underscorize [coll] 38 | (map #(rename-keys % (prepare-kw-map %)) coll)) 39 | 40 | (defn write [path header body] 41 | (csv/spit-csv path {:header (map replace-dash header)} (underscorize body))) 42 | 43 | 44 | (defn write-to-file [file header rows prepend-header] 45 | (let [data 46 | (->> (underscorize rows) 47 | (csv/vectorize {:header (map replace-dash header) :prepend-header prepend-header}))] 48 | (cd-csv/write-csv file data :quote? (fn [_] true)))) 49 | -------------------------------------------------------------------------------- /test/keboola/docker/config_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.docker.config-test 2 | (:require [keboola.docker.config :as sut] 3 | [clojure.test :as t :refer :all] 4 | [clojure.string :refer [trim ends-with? starts-with?]] 5 | [keboola.test-utils.core :as test-utils])) 6 | 7 | 8 | (def ^:dynamic *tmpdir* "") 9 | 10 | (defn setup-tmpdir [f] 11 | (binding [*tmpdir* (test-utils/mk-tmp-dir! "exfbtest")] 12 | (f) 13 | (test-utils/recursive-delete *tmpdir*))) 14 | 15 | (defn trimmed? [str-val] 16 | (= (trim str-val) str-val)) 17 | 18 | (deftest test-check-path 19 | (are [test-value] (and (ends-with? test-value "/") (-> test-value (ends-with? "//") not ) (trimmed? test-value)) 20 | (sut/check-path "asd") 21 | (sut/check-path "asd/") 22 | (sut/check-path " aa/") 23 | (sut/check-path " aa"))) 24 | 25 | (deftest test-out-dir-path 26 | (let [path (sut/out-dir-path (test-utils/mk-path *tmpdir* "."))] 27 | (is (and 28 | (trimmed? path) 29 | (ends-with? path "out/tables/") 30 | (.isDirectory (java.io.File. path)) 31 | (.exists (java.io.File. path)))))) 32 | 33 | (deftest test-parameters 34 | (is (thrown? java.io.FileNotFoundException (sut/parameters)))) 35 | 36 | (deftest test-get-fb-token 37 | (is (empty? (sut/get-fb-token {}))) 38 | (is (= "access_token" (sut/get-fb-token {:access_token "access_token" :token "token"}))) 39 | (is (= "access_token" (sut/get-fb-token {:access_token "access_token"}))) 40 | (is (= "token" (sut/get-fb-token {:token "token"})))) 41 | 42 | (use-fixtures :once setup-tmpdir) 43 | -------------------------------------------------------------------------------- /test/keboola/docker/runtime_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.docker.runtime-test 2 | (:require [keboola.docker.runtime :as sut] 3 | [clojure.test :as t :refer [deftest]] 4 | [keboola.test-utils.core :refer [prints-error? prints-msg?]])) 5 | 6 | 7 | (deftest test-log-error 8 | (prints-error? sut/log-error "asd" "fff") 9 | (prints-error? sut/log-error "asd") 10 | (prints-error? sut/log-error "")) 11 | 12 | 13 | 14 | (deftest test-log-strings 15 | (prints-msg? sut/log-strings "asd" "asd") 16 | (prints-msg? sut/log-strings "asf") 17 | (prints-msg? sut/log-strings "") 18 | (prints-msg? sut/log-strings)) 19 | 20 | -------------------------------------------------------------------------------- /test/keboola/facebook/extractor/choose_token_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.extractor.choose-token-test 2 | (:require [clojure.test :refer :all] 3 | [keboola.facebook.extractor.query :refer [choose-token]]) 4 | (:use clj-http.fake)) 5 | 6 | (def accounts-response {:request-time 261, :repeatable? false, :protocol-version {:name "HTTP", :major 1, :minor 1}, :streaming? true, :chunked? false, :reason-phrase "OK", :headers {"Content-Type" "text/javascript; charset=UTF-8", "Access-Control-Allow-Origin" "*", "X-FB-Debug" "fXQt79Gaetuj2SncCSgnNZCuKuyJMtzr2dHtTs1lGouuGx3Jpm777iP4+pYPIbRkbb0jGfIMQftbtMN3e/mBBw==", "facebook-api-version" "v3.1", "Strict-Transport-Security" "max-age=15552000; preload", "Connection" "close", "Pragma" "no-cache", "Expires" "Sat, 01 Jan 2000 00:00:00 GMT", "x-fb-rev" "4368033", "ETag" "\"2ee48088b20f04459ee2c5ef75f40d8138ef6ecf\"", "x-fb-trace-id" "AxSI8JAYFqD", "Date" "Mon, 01 Oct 2018 09:34:22 GMT", "Vary" "Accept-Encoding", "Cache-Control" "private, no-cache, no-store, must-revalidate", "x-app-usage" "{\"call_count\":0,\"total_cputime\":0,\"total_time\":0}"}, :orig-content-encoding nil, :status 200, :body "{\"data\":[{\"access_token\":\"pagetoken1\",\"category\":\"Community\",\"category_list\":[{\"id\":\"2612\",\"name\":\"blabla\"}],\"name\":\"page1\",\"id\":\"1017786331693295\",\"tasks\":[\"ANALYZE\",\"ADVERTISE\",\"MODERATE\",\"CREATE_CONTENT\",\"MANAGE\"]},{\"access_token\":\"pagetoken2\",\"category\":\"Community\",\"category_list\":[{\"id\":\"2612\",\"name\":\"4h0CG\"}],\"name\":\"0xk3z\",\"id\":\"1960059394258217\",\"tasks\":[\"ANALYZE\",\"ADVERTISE\",\"MODERATE\",\"CREATE_CONTENT\",\"MANAGE\"]},{\"access_token\":\"pagetoken3\",\"category\":\"Business Consultant\",\"category_list\":[{\"id\":\"179672275401610\",\"name\":\"n2DCf\"},{\"id\":\"2211\",\"name\":\"boKMN\"}],\"name\":\"PInBh\",\"id\":\"177057932317550\",\"tasks\":[\"ANALYZE\"]}],\"paging\":{\"cursors\":{\"before\":\"MTAxNzc4NjMzMTY5MzI5NQZDZD\",\"after\":\"MTc3MDU3OTMyMzE3NTUw\"}}}"}) 7 | 8 | 9 | (def usertoken "XXXTOKEN") 10 | 11 | (def api-mock 12 | { 13 | "https://graph.facebook.com/v3.1/me/accounts?access_token=XXXTOKEN" 14 | (fn [req] 15 | accounts-response)}) 16 | 17 | 18 | (deftest test-choose-token 19 | (with-global-fake-routes-in-isolation 20 | api-mock 21 | (is (= usertoken (choose-token "foo" usertoken "v3.1"))) 22 | (is (= "pagetoken1" (choose-token "1017786331693295" usertoken "v3.1"))))) 23 | 24 | (deftest test-choose-token-fallback 25 | (is (= "othertoken" (choose-token "1017786331693295" "othertoken" "v3.1")))) 26 | -------------------------------------------------------------------------------- /test/keboola/facebook/extractor/core_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.extractor.core-test 2 | (:require [keboola.facebook.extractor.core :as sut] 3 | [clojure.test :refer :all] 4 | [slingshot.slingshot :refer [throw+]])) 5 | (deftest test-treat-exception 6 | (with-redefs [keboola.docker.runtime/app-error (fn [_] "app error")] 7 | (is (= "app error" (sut/treat #(throw "aaa")))) 8 | (is (= "app error" (sut/treat #(throw+ "aaa")))) 9 | (is (= "app error" (sut/treat #(throw+ {:a "aaa"})))))) 10 | -------------------------------------------------------------------------------- /test/keboola/facebook/extractor/output_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.extractor.output-test 2 | (:require [clojure.test :refer :all] 3 | [keboola.facebook.extractor.output :as sut])) 4 | 5 | (defn test-pk [value expected] 6 | (let [complete-expected (conj expected "parent_id")] 7 | (is (= (count value) (count complete-expected))) 8 | (is (= (set value) (set complete-expected))))) 9 | 10 | (deftest test-primary-key 11 | (test-pk (sut/get-primary-key ["id"] "" {} false) []) 12 | (test-pk (sut/get-primary-key [:id] "" {} false) ["id"]) 13 | (test-pk (sut/get-primary-key ["foo"] "" {} false) []) 14 | (test-pk (sut/get-primary-key [:foo] "" {} false) []) 15 | (test-pk (sut/get-primary-key [:id :key] "" {} false) ["id"]) 16 | (test-pk (sut/get-primary-key [:id :key1 :foo :key2] "insights" {} false) ["id" "key1" "key2"]) 17 | (test-pk (sut/get-primary-key [:id :placement :device_platform :foo :key2 :age :ad_id :adset_id] "insights" {} false) ["id" "placement" "key2" "age" "device_platform"]) 18 | (test-pk (sut/get-primary-key [:id :placement :device_platform :foo :key2 :age :ad_id :adset_id :date_start :date_stop] "insights" {} true) ["id" "placement" "key2" "age" "device_platform" "date_stop" "date_start" "ad_id" "adset_id"]) 19 | (test-pk (sut/get-primary-key [:id :placement :foo :key2 :age :region :country :gender] "asdasfoo" {} false) ["id" "key2"]) 20 | (test-pk (sut/get-primary-key [:id :key1 :foo :key2] "" {} false) ["id" "key1" "key2"]) 21 | (test-pk (sut/get-primary-key [:id :key1 :foo :key2] "" {:path ""} false) ["id" "key1" "key2"]) 22 | (test-pk (sut/get-primary-key [:id :key1 :foo :key2 :ad_id] "" {:path ""} false) ["id" "key1" "key2" "ad_id"]) 23 | (test-pk (sut/get-primary-key [:id :key1 :foo :key2 :ad_id :adset_id] "" {:path ""} false) ["id" "key1" "key2" "ad_id"]) 24 | (test-pk (sut/get-primary-key [:id :key1 :foo :key2 :adset_id] "" {:path ""} false) ["id" "key1" "key2" "adset_id"]) 25 | (test-pk (sut/get-primary-key [:id :key1 :foo :key2 :ad_id] "" {:path "ads"} false) ["id" "key1" "key2"]) 26 | (test-pk (sut/get-primary-key [:id :key1 :foo :key2 :reviewer_id] "ratings" {} false) ["id" "key1" "key2" "reviewer_id"])) 27 | -------------------------------------------------------------------------------- /test/keboola/facebook/extractor/query_parser_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.extractor.query-parser-test 2 | (:require [clojure.test :refer :all] 3 | [keboola.facebook.extractor.query-parser :as sut]) 4 | (:import [java.time LocalDate] 5 | [java.time.format DateTimeParseException] 6 | [java.time.format DateTimeFormatter])) 7 | 8 | 9 | (deftest test-parse-parameters 10 | (testing "parse-parameters function" 11 | ;; Test with multiple parameters 12 | (is (= (sut/parse-parameters "param1=value1¶m2=value2") 13 | {"param1" "value1", "param2" "value2"})) 14 | (is (= (sut/parse-parameters "time_ranges=value1&date_preset=last_3d") 15 | {"time_ranges" "value1", "date_preset" "last_3d"})) 16 | ;; ;; Test with a single parameter 17 | (is (= (sut/parse-parameters "param1=value1") 18 | {"param1" "value1"})) 19 | ;; ;; Test with empty string 20 | (is (= {} 21 | (sut/parse-parameters ""))) 22 | ;; Test with parameter without value 23 | (is (= (sut/parse-parameters "param1=¶m2=value2") 24 | {"param1" "", "param2" "value2"})) 25 | ;; Test with duplicate parameter names 26 | (is (= (sut/parse-parameters "param1=value1¶m1=value2") 27 | {"param1" "value2"})) ;; The last value should overwrite 28 | ;; Test with value containing equal sign 29 | (is (= (sut/parse-parameters "param1=value1=value2") 30 | {"param1" "value1=value2"})) 31 | ;; Test with parameter without equal sign 32 | (is (= (sut/parse-parameters "param1") 33 | {"param1" nil})) 34 | ;; Test with parameters having empty keys or values 35 | (is (= (sut/parse-parameters "=value1¶m2=") 36 | {"" "value1", "param2" ""})))) 37 | 38 | 39 | (deftest test-parse-time-ranges 40 | (testing "parse-time-ranges function" 41 | ;; Test with single time range 42 | (is (= [{:since "2024-08-10", :until "2024-08-13"}] 43 | (sut/parse-time-ranges "[{'since':'2024-08-10','until':'2024-08-13'}]"))) 44 | ;; Test with multiple time ranges 45 | (is (= [{:since "2024-08-10", :until "2024-08-10"} 46 | {:since "2024-08-11", :until "2024-08-11"} 47 | {:since "2024-08-12", :until "2024-08-12"} 48 | {:since "2024-08-13", :until "2024-08-13"}] 49 | (sut/parse-time-ranges "[{'since':'2024-08-10','until':'2024-08-10'},{'since':'2024-08-11','until':'2024-08-11'},{'since':'2024-08-12','until':'2024-08-12'},{'since':'2024-08-13','until':'2024-08-13'}]"))) 50 | ;; Test with empty string 51 | (is (= [] 52 | (sut/parse-time-ranges ""))) 53 | ;; Test with invalid JSON (should throw an exception) 54 | (is (thrown? Exception 55 | (sut/parse-time-ranges "[{'since':'2024-08-10','until':'2024-08-13'"))) 56 | ;; Test with nil input (should return nil) 57 | (is (= [] 58 | (sut/parse-time-ranges nil))))) 59 | 60 | 61 | 62 | (deftest test-date-range 63 | (testing "date-range function" 64 | ;; Test with start and end dates being the same 65 | (is (= [(LocalDate/parse "2024-08-10")] 66 | (sut/date-range "2024-08-10" "2024-08-10"))) 67 | ;; Test with start date before end date 68 | (is (= [(LocalDate/parse "2024-08-10") 69 | (LocalDate/parse "2024-08-11") 70 | (LocalDate/parse "2024-08-12")] 71 | (sut/date-range "2024-08-10" "2024-08-13"))) 72 | ;; Test with start date after end date (should return an empty sequence) 73 | (is (= [] 74 | (sut/date-range "2024-08-13" "2024-08-10"))) 75 | ;; Test with invalid date format (should throw an exception) 76 | (is (thrown? DateTimeParseException 77 | (sut/date-range "2024/08/10" "2024/08-13"))) 78 | ;; Test with nil inputs (should throw an exception) 79 | (is (thrown? NullPointerException 80 | (sut/date-range nil "2024-08-13"))) 81 | (is (thrown? NullPointerException 82 | (sut/date-range "2024-08-10" nil))) 83 | ;; Test with large date range 84 | (is (= (map #(LocalDate/parse %) 85 | ["2024-01-01" "2024-01-02" "2024-01-03" "2024-01-04"]) 86 | (sut/date-range "2024-01-01" "2024-01-05"))))) 87 | 88 | (deftest test-expand-time-ranges 89 | (testing "expand-time-ranges function" 90 | ;; Test with a single time range 91 | (let [input [{:since "2024-08-10" :until "2024-08-13"}] 92 | expected [{:since "2024-08-10" :until "2024-08-10"} 93 | {:since "2024-08-11" :until "2024-08-11"} 94 | {:since "2024-08-12" :until "2024-08-12"}]] 95 | (is (= expected (sut/expand-time-ranges input)))) 96 | ;; Test with multiple time ranges 97 | (let [input [{:since "2024-08-10" :until "2024-08-11"} 98 | {:since "2024-08-13" :until "2024-08-14"}] 99 | expected [{:since "2024-08-10" :until "2024-08-10"} 100 | {:since "2024-08-13" :until "2024-08-13"}]] 101 | (is (= expected (sut/expand-time-ranges input)))) 102 | ;; Test with start date equal to end date 103 | (let [input [{:since "2024-08-10" :until "2024-08-10"}] 104 | expected [{:since "2024-08-10" :until "2024-08-10"}]] 105 | (is (= expected (sut/expand-time-ranges input)))) 106 | ;; Test with start date after end date (should return empty list) 107 | (let [input [{:since "2024-08-13" :until "2024-08-10"}] 108 | expected []] 109 | (is (= expected (sut/expand-time-ranges input)))) 110 | ;; Test with empty input 111 | (let [input [] 112 | expected []] 113 | (is (= expected (sut/expand-time-ranges input)))))) 114 | 115 | 116 | (deftest test-generate-parameters-for-dates 117 | (testing "generate-parameters-for-dates function with single quotes" 118 | ;; Test with basic parameters and time-range 119 | (let [parameters-map {"time_increment" "1" 120 | "breakdowns" "product_id" 121 | "date_preset" "last_7d"} 122 | time-range {:since "2024-08-10" :until "2024-08-10"} 123 | expected "time_increment=1&breakdowns=product_id&time_ranges=[{'since':'2024-08-10','until':'2024-08-10'}]"] 124 | (is (= expected (sut/generate-parameters-for-dates parameters-map time-range)))) 125 | 126 | ;; Test with parameters containing existing time_ranges (should replace time_ranges) 127 | (let [parameters-map {"time_increment" "1" 128 | "breakdowns" "product_id" 129 | "time_ranges" "[{'since':'2024-08-10','until':'2024-08-13'}]"} 130 | time-range {:since "2024-08-10" :until "2024-08-13"} 131 | expected "time_increment=1&breakdowns=product_id&time_ranges=[{'since':'2024-08-10','until':'2024-08-13'}]"] 132 | (is (= expected (sut/generate-parameters-for-dates parameters-map time-range)))) 133 | 134 | ;; Test with parameters without date_preset 135 | (let [parameters-map {"time_increment" "1" 136 | "breakdowns" "product_id"} 137 | time-range {:since "2024-08-10" :until "2024-08-10"} 138 | expected "time_increment=1&breakdowns=product_id&time_ranges=[{'since':'2024-08-10','until':'2024-08-10'}]"] 139 | (is (= expected (sut/generate-parameters-for-dates parameters-map time-range)))) 140 | 141 | ;; Test with parameters containing date_preset and other parameters 142 | (let [parameters-map {"time_increment" "1" 143 | "breakdowns" "product_id" 144 | "date_preset" "last_3d" 145 | "level" "ad"} 146 | time-range {:since "2024-08-10" :until "2024-08-12"} 147 | expected "time_increment=1&breakdowns=product_id&level=ad&time_ranges=[{'since':'2024-08-10','until':'2024-08-12'}]"] 148 | (is (= expected (sut/generate-parameters-for-dates parameters-map time-range)))) 149 | 150 | ;; Test with empty parameters-map 151 | (let [parameters-map {} 152 | time-range {:since "2024-08-10" :until "2024-08-10"} 153 | expected "time_ranges=[{'since':'2024-08-10','until':'2024-08-10'}]"] 154 | (is (= expected (sut/generate-parameters-for-dates parameters-map time-range)))))) 155 | 156 | 157 | (deftest test-date-preset-to-days 158 | (testing "date-preset-to-days function" 159 | ;; Test with known date_preset values 160 | (is (= 3 (sut/date-preset-to-days "last_3d"))) 161 | (is (= 7 (sut/date-preset-to-days "last_7d"))) 162 | (is (= 30 (sut/date-preset-to-days "last_30d"))) 163 | ;; Test with unknown date_preset value 164 | (is (= nil (sut/date-preset-to-days "last_5d"))) 165 | ;; Test with nil input 166 | (is (= nil (sut/date-preset-to-days nil))) 167 | ;; Test with empty string 168 | (is (= nil (sut/date-preset-to-days ""))))) 169 | 170 | 171 | (deftest test-get-past-date 172 | (testing "get-past-date function with reference-date parameter" 173 | (let [formatter (DateTimeFormatter/ofPattern "yyyy-MM-dd") 174 | reference-date (LocalDate/of 2024 10 22)] ; Fixed reference date: 2024-10-22 175 | ;; Test with 0 days ago (should return the reference date) 176 | (is (= "2024-10-22" (sut/get-past-date 0 reference-date))) 177 | ;; Test with 1 day ago 178 | (is (= "2024-10-21" (sut/get-past-date 1 reference-date))) 179 | ;; Test with 7 days ago 180 | (is (= "2024-10-15" (sut/get-past-date 7 reference-date))) 181 | ;; Test with 3 days ago 182 | (is (= "2024-10-19" (sut/get-past-date 3 reference-date))) 183 | ;; Test with negative days (future date) 184 | (is (= "2024-10-25" (sut/get-past-date -3 reference-date))) 185 | ;; Test with large number of days 186 | (is (= "2023-10-22" (sut/get-past-date 366 reference-date)))) 187 | 188 | (testing "get-past-date function without reference-date (uses LocalDate/now)" 189 | ;; Since LocalDate/now is used, this test is date-sensitive. 190 | ;; We'll test that the function returns the same date as calculated here. 191 | (let [days 5 192 | formatter (DateTimeFormatter/ofPattern "yyyy-MM-dd") 193 | expected-date (.format (.minusDays (LocalDate/now) days) formatter)] 194 | (is (= expected-date (sut/get-past-date days))))))) -------------------------------------------------------------------------------- /test/keboola/facebook/extractor/query_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.facebook.extractor.query-test 2 | (:require [keboola.facebook.extractor.query :as sut] 3 | [keboola.facebook.extractor.query-parser :as query-parser] 4 | [keboola.test-utils.core :as test-utils] 5 | [clojure.java.io :as io] 6 | [keboola.facebook.api.request-test :refer [media-posted-before-error-response]] 7 | [clojure.test :refer :all]) 8 | (:use clj-http.fake) 9 | (:import [java.time LocalDate] 10 | [java.time.format DateTimeFormatter])) 11 | 12 | (def ^:dynamic *tmpdir* "") 13 | 14 | (defn setup-tmpdir [f] 15 | (binding [*tmpdir* (test-utils/mk-tmp-dir! "exfbtestquery")] 16 | (f) 17 | (test-utils/recursive-delete *tmpdir*))) 18 | 19 | (deftest test-query-contains-insights? 20 | (is (not (sut/query-contains-insights? {}))) 21 | (is (not (sut/query-contains-insights? {:fields "asasd"}))) 22 | (is (not (sut/query-contains-insights? {:fields "asasd" :path "asdfff"}))) 23 | (is (not (sut/query-contains-insights? {:fields "" :path nil}))) 24 | (is (not (sut/query-contains-insights? {:fields nil :path nil}))) 25 | (is (not (sut/query-contains-insights? {:fields nil :path ""}))) 26 | (is (sut/query-contains-insights? {:fields nil :path "insights"})) 27 | (is (sut/query-contains-insights? {:fields "asdasdasd,insights" :path "insights"})) 28 | (is (sut/query-contains-insights? {:fields "asdasdasd,insights" :path "feed"})) 29 | (is (sut/query-contains-insights? {:fields "asdasdasd,insights" :path nil})) 30 | (is (sut/query-contains-insights? {:fields nil :path "insights"}))) 31 | 32 | (deftest test-query-path-feed? 33 | (is (not (sut/query-path-feed? {}))) 34 | (is (not (sut/query-path-feed? {:path "ratings" :fields "feed"}))) 35 | (is (sut/query-path-feed? {:path "feed" :fields "insights"})) 36 | (is (sut/query-path-feed? {:path "me/feed" :fields "insights"}))) 37 | 38 | (deftest test-query-path-posts? 39 | (is (not (sut/query-path-posts? {}))) 40 | (is (not (sut/query-path-posts? {:path "ratings" :fields "feed"}))) 41 | (is (not (sut/query-path-posts? {:path "feed" :fields "insights"}))) 42 | (is (sut/query-path-posts? {:path "posts" :fields "insights"})) 43 | (is (sut/query-path-posts? {:path "me/posts" :fields "insights"})) 44 | (is (sut/query-path-posts? {:path "me/published_posts" :fields "insights"})) 45 | (is (sut/query-path-posts? {:path "published_posts" :fields "insights"}))) 46 | 47 | (deftest test-query-need-userinfo? 48 | (is (not (sut/query-need-userinfo? {}))) 49 | (is (not (sut/query-need-userinfo? {:path "ratings" :fields "feed"}))) 50 | (is (sut/query-need-userinfo? {:path "likes" :fields "insights"})) 51 | (is (sut/query-need-userinfo? {:path "" :fields "likes"})) 52 | (is (sut/query-need-userinfo? {:path "me/feed" :fields "from"}))) 53 | 54 | (defn empty-dir? [path] 55 | (let [file (io/file path)] 56 | (assert (.exists file)) 57 | (assert (.isDirectory file)) 58 | (-> file .list empty?))) 59 | 60 | (deftest test-media-posted-before-error-response-query 61 | (let [query {:name "mediatest" :version "v2.11" :query {:path "media" :ids "123" :fields "fields" :since "now" :until "now"}} 62 | token "token" 63 | out-dir *tmpdir*] 64 | (println *tmpdir*) 65 | (with-global-fake-routes-in-isolation 66 | {"https://graph.facebook.com/v2.11/media?path=media&ids=123&fields=fields&since=now&until=now&access_token=token" 67 | (fn [req] 68 | media-posted-before-error-response)} 69 | (sut/run-nested-query token out-dir query) 70 | (is (empty-dir? *tmpdir*))))) 71 | 72 | (use-fixtures :once setup-tmpdir) 73 | 74 | 75 | 76 | (deftest test-make-run-insights-query-with-time-range 77 | (testing "date_preset=last_3d" 78 | ;; Placeholder values for unimportant parameters 79 | (let [token "token" 80 | id "id" 81 | version "version" 82 | query {} 83 | ;; Parameters string as per your example 84 | parameters-str "time_increment=1&breakdowns=product_id&date_preset=last_3d" 85 | ;; Atom to capture calls to the mock function 86 | async-insights-request-calls (atom []) 87 | ;; Mock function to capture calls and parameters 88 | mock-async-insights-request-fn (fn [token id new-parameters version query] 89 | (swap! async-insights-request-calls conj 90 | {:token token 91 | :id id 92 | :parameters new-parameters 93 | :version version 94 | :query query}) 95 | ;; Return a mock result 96 | [{:data "mock-result"}])] 97 | 98 | ;; Call the function under test 99 | (let [results (sut/make-run-inishgts-query-with-time-range 100 | mock-async-insights-request-fn 101 | token 102 | id 103 | parameters-str 104 | version 105 | query) 106 | ;; Capture the calls made to the mock function 107 | calls @async-insights-request-calls 108 | num-calls (count calls) 109 | ;; Compute expected dates based on current date 110 | date-preset "last_3d" 111 | days (query-parser/date-preset-to-days date-preset) ;; Should be 3 for "last_3d" 112 | formatter (DateTimeFormatter/ofPattern "yyyy-MM-dd") 113 | today (LocalDate/now) 114 | start-date (query-parser/get-past-date days) 115 | ;; Generate dates including today 116 | dates (map #(.format % formatter) 117 | (query-parser/date-range start-date (.format today formatter))) 118 | expected-time-ranges (map (fn [date] 119 | {:since date 120 | :until date}) 121 | dates)] 122 | 123 | ;; Verify the number of calls made matches expected number of dates 124 | (is (= num-calls (count expected-time-ranges)) 125 | (str "Expected " (count expected-time-ranges) " calls to async-insights-request-fn, but got " num-calls)) 126 | 127 | ;; Parse the parameters once for use in generating expected parameters 128 | (let [parameters-map (query-parser/parse-parameters parameters-str)] 129 | 130 | ;; Verify each call was made with the correct parameters 131 | (doseq [[call expected-time-range] (map vector calls expected-time-ranges)] 132 | (let [expected-parameters (query-parser/generate-parameters-for-dates parameters-map expected-time-range)] 133 | (is (= (:parameters call) expected-parameters) 134 | (str "Expected parameters: " expected-parameters ", but got: " (:parameters call)))))) 135 | 136 | ;; Optionally, verify that results contain the expected data 137 | (is (= results (apply concat (repeat num-calls [{:data "mock-result"}]))) 138 | "Expected results to contain the mock data repeated for each call")))) 139 | (testing "time_ranges=[{'since':'2024-08-10','until':'2024-08-12'}]" 140 | ;; Placeholder values for unimportant parameters 141 | (let [token "token" 142 | id "id" 143 | version "version" 144 | query {} 145 | ;; Parameters string as per your example 146 | parameters-str "time_increment=1&breakdowns=product_id&time_ranges=[{'since':'2024-08-10','until':'2024-08-12'}]" 147 | ;; Atom to capture calls to the mock function 148 | async-insights-request-calls (atom []) 149 | ;; Mock function to capture calls and parameters 150 | mock-async-insights-request-fn (fn [token id new-parameters version query] 151 | (swap! async-insights-request-calls conj 152 | {:token token 153 | :id id 154 | :parameters new-parameters 155 | :version version 156 | :query query}) 157 | ;; Return a mock result 158 | [{:data "mock-result"}])] 159 | 160 | ;; Call the function under test 161 | (let [results (sut/make-run-inishgts-query-with-time-range 162 | mock-async-insights-request-fn 163 | token 164 | id 165 | parameters-str 166 | version 167 | query) 168 | ;; Capture the calls made to the mock function 169 | calls @async-insights-request-calls 170 | num-calls (count calls) 171 | ;; Compute expected dates based on current date 172 | time-ranges-str "[{'since':'2024-08-10','until':'2024-08-12'}]" 173 | time-ranges (query-parser/parse-time-ranges time-ranges-str) 174 | expected-time-ranges (query-parser/expand-time-ranges time-ranges)] 175 | 176 | ;; Verify the number of calls made matches expected number of dates 177 | (is (= num-calls (count expected-time-ranges)) 178 | (str "Expected " (count expected-time-ranges) " calls to async-insights-request-fn, but got " num-calls)) 179 | 180 | ;; Parse the parameters once for use in generating expected parameters 181 | (let [parameters-map (query-parser/parse-parameters parameters-str)] 182 | 183 | ;; Verify each call was made with the correct parameters 184 | (doseq [[call expected-time-range] (map vector calls expected-time-ranges)] 185 | (let [expected-parameters (query-parser/generate-parameters-for-dates parameters-map expected-time-range)] 186 | (is (= (:parameters call) expected-parameters) 187 | (str "Expected parameters: " expected-parameters ", but got: " (:parameters call)))))) 188 | 189 | ;; Optionally, verify that results contain the expected data 190 | (is (= results (apply concat (repeat num-calls [{:data "mock-result"}]))) 191 | "Expected results to contain the mock data repeated for each call"))))) -------------------------------------------------------------------------------- /test/keboola/http/client_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.http.client-test 2 | (:require [keboola.http.client :as sut] 3 | [clojure.test :refer :all])) 4 | 5 | (deftest test-retry-handler 6 | (is (sut/retry-handler nil 1 nil)) 7 | (is (not (sut/retry-handler nil 4 nil)))) 8 | -------------------------------------------------------------------------------- /test/keboola/snapshots/ads/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "act_10152412627713995" : { 6 | "account_id" : "10152412627713995", 7 | "business_name" : "", 8 | "currency" : "EUR", 9 | "id" : "act_10152412627713995", 10 | "name" : "blabla" 11 | } 12 | }, 13 | "api-version" : "v2.9", 14 | "queries" : [ { 15 | "name" : "ads", 16 | "type" : "nested-query", 17 | "disabled" : false, 18 | "query" : { 19 | "path" : "ads", 20 | "fields" : "id,name,adset_id", 21 | "ids" : null 22 | } 23 | }, { 24 | "name" : "campaigns", 25 | "type" : "nested-query", 26 | "disabled" : false, 27 | "query" : { 28 | "path" : "campaigns", 29 | "fields" : "id,name,account_id", 30 | "ids" : null 31 | } 32 | }, { 33 | "name" : "adsets", 34 | "type" : "nested-query", 35 | "disabled" : false, 36 | "query" : { 37 | "path" : "adsets", 38 | "fields" : "id,name,account_id", 39 | "ids" : null 40 | } 41 | } ] 42 | }, 43 | "authorization" : { 44 | "oauth_api" : { 45 | "id" : "keboola.ex-facebook-ads", 46 | "credentials" : { 47 | "id" : "main", 48 | "authorizedFor" : "Myself", 49 | "creator" : { 50 | "id" : "1234", 51 | "description" : "me@keboola.com" 52 | }, 53 | "created" : "2016-01-31 00:13:30", 54 | "oauthVersion" : "facebook", 55 | "appKey" : "xxx", 56 | "#data" : "{\"token\":\"XXTOKENXX\"}", 57 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 58 | } 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/ads/out/tables/accounts: -------------------------------------------------------------------------------- 1 | account_id,name,business_name,currency,id 2 | 10152412627713995,blabla,,EUR,act_10152412627713995 3 | -------------------------------------------------------------------------------- /test/keboola/snapshots/ads/out/tables/ads.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["id","ex-account-id","fb-graph-node","parent-id","name","adset_id"],"incremental":true,"primary_key":["parent_id","id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/ads/out/tables/adsets.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["id","ex-account-id","fb-graph-node","parent-id","name","account_id"],"incremental":true,"primary_key":["parent_id","id","account_id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/ads/out/tables/campaigns.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["id","ex-account-id","fb-graph-node","parent-id","name","account_id"],"incremental":true,"primary_key":["parent_id","id","account_id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/ads/out/tables/campaigns/1507033519047: -------------------------------------------------------------------------------- 1 | "6018062854308","act_10152412627713995","page_campaigns","act_10152412627713995","8Gj7i","10152412627713995" 2 | "6018300843708","act_10152412627713995","page_campaigns","act_10152412627713995","hvWzD","10152412627713995" 3 | "6018300843908","act_10152412627713995","page_campaigns","act_10152412627713995","XwYPZ","10152412627713995" 4 | "6019173393708","act_10152412627713995","page_campaigns","act_10152412627713995","oGkEu","10152412627713995" 5 | "6019363983508","act_10152412627713995","page_campaigns","act_10152412627713995","QELGH","10152412627713995" 6 | "6019982601108","act_10152412627713995","page_campaigns","act_10152412627713995","covB4","10152412627713995" 7 | "6019982771708","act_10152412627713995","page_campaigns","act_10152412627713995","RqiZ8","10152412627713995" 8 | "6020012167508","act_10152412627713995","page_campaigns","act_10152412627713995","TuLKl","10152412627713995" 9 | "6020764641508","act_10152412627713995","page_campaigns","act_10152412627713995","Xw0YP","10152412627713995" 10 | "6021430990308","act_10152412627713995","page_campaigns","act_10152412627713995","pay1H","10152412627713995" 11 | "6021618797908","act_10152412627713995","page_campaigns","act_10152412627713995","eiMuF","10152412627713995" 12 | "6021704237308","act_10152412627713995","page_campaigns","act_10152412627713995","rw0TB","10152412627713995" 13 | "6022227532708","act_10152412627713995","page_campaigns","act_10152412627713995","nC8pw","10152412627713995" 14 | "6023110050908","act_10152412627713995","page_campaigns","act_10152412627713995","T0CUW","10152412627713995" 15 | "6023987405508","act_10152412627713995","page_campaigns","act_10152412627713995","DvoMQ","10152412627713995" 16 | "6023987405708","act_10152412627713995","page_campaigns","act_10152412627713995","fDNUQ","10152412627713995" 17 | "6025342736908","act_10152412627713995","page_campaigns","act_10152412627713995","5mHlc","10152412627713995" 18 | "6025523608508","act_10152412627713995","page_campaigns","act_10152412627713995","blxHI","10152412627713995" 19 | "6026287405308","act_10152412627713995","page_campaigns","act_10152412627713995","WCR5z","10152412627713995" 20 | "6026287405508","act_10152412627713995","page_campaigns","act_10152412627713995","Wh6jH","10152412627713995" 21 | "6026992252308","act_10152412627713995","page_campaigns","act_10152412627713995","McnqL","10152412627713995" 22 | "6026992779108","act_10152412627713995","page_campaigns","act_10152412627713995","OoLHu","10152412627713995" 23 | "6028106057908","act_10152412627713995","page_campaigns","act_10152412627713995","XbNoo","10152412627713995" 24 | "6028106058108","act_10152412627713995","page_campaigns","act_10152412627713995","9ISyt","10152412627713995" 25 | "6029050661508","act_10152412627713995","page_campaigns","act_10152412627713995","YKB5o","10152412627713995" 26 | "6029050661908","act_10152412627713995","page_campaigns","act_10152412627713995","LpEbN","10152412627713995" 27 | "6030899843508","act_10152412627713995","page_campaigns","act_10152412627713995","9CmVC","10152412627713995" 28 | "6030899843708","act_10152412627713995","page_campaigns","act_10152412627713995","jU043","10152412627713995" 29 | "6031426188908","act_10152412627713995","page_campaigns","act_10152412627713995","uOYg4","10152412627713995" 30 | "6031426189108","act_10152412627713995","page_campaigns","act_10152412627713995","QhtfU","10152412627713995" 31 | "6032597915908","act_10152412627713995","page_campaigns","act_10152412627713995","nczg6","10152412627713995" 32 | "6032597916108","act_10152412627713995","page_campaigns","act_10152412627713995","PKp9t","10152412627713995" 33 | "6032597916308","act_10152412627713995","page_campaigns","act_10152412627713995","3ynst","10152412627713995" 34 | "6032597916508","act_10152412627713995","page_campaigns","act_10152412627713995","ntQVj","10152412627713995" 35 | "6032883801708","act_10152412627713995","page_campaigns","act_10152412627713995","PV01o","10152412627713995" 36 | "6034087406108","act_10152412627713995","page_campaigns","act_10152412627713995","vT2Kf","10152412627713995" 37 | "6034087406308","act_10152412627713995","page_campaigns","act_10152412627713995","cXdGL","10152412627713995" 38 | "6034087406508","act_10152412627713995","page_campaigns","act_10152412627713995","auoGJ","10152412627713995" 39 | "6034087406708","act_10152412627713995","page_campaigns","act_10152412627713995","3wNMU","10152412627713995" 40 | "6034803642708","act_10152412627713995","page_campaigns","act_10152412627713995","83URP","10152412627713995" 41 | "6036310188308","act_10152412627713995","page_campaigns","act_10152412627713995","o5rDt","10152412627713995" 42 | "6037764899108","act_10152412627713995","page_campaigns","act_10152412627713995","21ctp","10152412627713995" 43 | "6037795175108","act_10152412627713995","page_campaigns","act_10152412627713995","pMeRv","10152412627713995" 44 | "6039331667708","act_10152412627713995","page_campaigns","act_10152412627713995","v6PMg","10152412627713995" 45 | "6040739715908","act_10152412627713995","page_campaigns","act_10152412627713995","pLL8f","10152412627713995" 46 | "6040739716108","act_10152412627713995","page_campaigns","act_10152412627713995","t2d96","10152412627713995" 47 | "6040740277908","act_10152412627713995","page_campaigns","act_10152412627713995","xikYt","10152412627713995" 48 | "6040740278108","act_10152412627713995","page_campaigns","act_10152412627713995","VsXdg","10152412627713995" 49 | "6042191850708","act_10152412627713995","page_campaigns","act_10152412627713995","cyHKO","10152412627713995" 50 | "6042213729708","act_10152412627713995","page_campaigns","act_10152412627713995","poKK7","10152412627713995" 51 | "6042213729908","act_10152412627713995","page_campaigns","act_10152412627713995","In4VK","10152412627713995" 52 | "6042601724508","act_10152412627713995","page_campaigns","act_10152412627713995","yHEZ7","10152412627713995" 53 | "6042601724708","act_10152412627713995","page_campaigns","act_10152412627713995","S5kYi","10152412627713995" 54 | "6044315728708","act_10152412627713995","page_campaigns","act_10152412627713995","jOwZ2","10152412627713995" 55 | "6044315728908","act_10152412627713995","page_campaigns","act_10152412627713995","YsQpP","10152412627713995" 56 | "6044315729108","act_10152412627713995","page_campaigns","act_10152412627713995","JWsNS","10152412627713995" 57 | "6044315729308","act_10152412627713995","page_campaigns","act_10152412627713995","ZrlnR","10152412627713995" 58 | "6044315729508","act_10152412627713995","page_campaigns","act_10152412627713995","oex08","10152412627713995" 59 | "6046816614308","act_10152412627713995","page_campaigns","act_10152412627713995","CKsXC","10152412627713995" 60 | "6046816614508","act_10152412627713995","page_campaigns","act_10152412627713995","OvvPv","10152412627713995" 61 | "6049130283508","act_10152412627713995","page_campaigns","act_10152412627713995","wRoPk","10152412627713995" 62 | "6049130283708","act_10152412627713995","page_campaigns","act_10152412627713995","6EUEL","10152412627713995" 63 | "6052149267708","act_10152412627713995","page_campaigns","act_10152412627713995","zpZri","10152412627713995" 64 | -------------------------------------------------------------------------------- /test/keboola/snapshots/ads/test_ads.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.ads.test-ads 2 | (:require [keboola.snapshots.ads.apicalls :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest ads-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "ads"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "test/keboola/snapshots/ads") 17 | (test-utils/copy-config-tmp "test/keboola/snapshots/ads" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "test/keboola/snapshots/ads" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/snapshots/adsinsights/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "act_10152562141153995" : { 6 | "account_id" : "10152562141153995", 7 | "business_name" : "", 8 | "currency" : "EUR", 9 | "id" : "act_10152562141153995", 10 | "name" : "blabla" 11 | } 12 | }, 13 | "api-version" : "v2.9", 14 | "queries" : [ { 15 | "name" : "ads_insights", 16 | "type" : "nested-query", 17 | "disabled" : false, 18 | "query" : { 19 | "path" : "ads", 20 | "fields" : "insights.action_breakdowns(action_type).date_preset(last_3d).time_increment(1){ad_id,impressions,reach,clicks,spend,video_30_sec_watched_actions,video_p95_watched_actions,actions}", 21 | "ids" : "act_10152437784203995" 22 | } 23 | } ] 24 | }, 25 | "authorization" : { 26 | "oauth_api" : { 27 | "id" : "keboola.ex-facebook-ads", 28 | "credentials" : { 29 | "id" : "main", 30 | "authorizedFor" : "Myself", 31 | "creator" : { 32 | "id" : "1234", 33 | "description" : "me@keboola.com" 34 | }, 35 | "created" : "2016-01-31 00:13:30", 36 | "oauthVersion" : "facebook", 37 | "appKey" : "xxx", 38 | "#data" : "{\"token\":\"XXTOKENXX\"}", 39 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 40 | } 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/adsinsights/out/tables/accounts: -------------------------------------------------------------------------------- 1 | account_id,name,business_name,currency,id 2 | 10152562141153995,blabla,,EUR,act_10152562141153995 3 | -------------------------------------------------------------------------------- /test/keboola/snapshots/adsinsights/out/tables/ads_insights.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["ex-account-id","fb-graph-node","parent-id","ads_action_name","action_type","value","ad_id","clicks","date_start","date_stop","impressions","reach","spend"],"incremental":true,"primary_key":["parent_id","date_start","date_stop","ads_action_name","action_type"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/adsinsights/test_adsinsights.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.adsinsights.test-adsinsights 2 | (:require [keboola.snapshots.adsinsights.apicalls :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest adsinsights-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "adsinsights"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "test/keboola/snapshots/adsinsights") 17 | (test-utils/copy-config-tmp "test/keboola/snapshots/adsinsights" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "test/keboola/snapshots/adsinsights" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/snapshots/asyncinisghtscampaigns/apicalls.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.asyncinisghtscampaigns.apicalls) 2 | (def recorded 3 | { 4 | {:method :get, 5 | :address 6 | "https://graph.facebook.com/v10.0/192025512975195/insights/?access_token=XXTOKENXX", 7 | :as :json} 8 | (fn [req]{:status 200, 9 | :body 10 | "{\"data\":[{\"account_id\":\"522606278080331\",\"actions\":[{\"action_reaction\":\"wow\",\"action_type\":\"post_reaction\",\"value\":\"1\"},{\"action_reaction\":\"like\",\"action_type\":\"post_reaction\",\"value\":\"7\"},{\"action_type\":\"comment\",\"value\":\"3\"},{\"action_type\":\"onsite_conversion.post_save\",\"value\":\"1\"},{\"action_type\":\"link_click\",\"value\":\"106\"},{\"action_type\":\"post\",\"value\":\"1\"},{\"action_type\":\"post_reaction\",\"value\":\"8\"},{\"action_type\":\"video_view\",\"value\":\"1711\"},{\"action_type\":\"landing_page_view\",\"value\":\"68\"},{\"action_type\":\"offsite_conversion.fb_pixel_add_payment_info\",\"value\":\"5\"},{\"action_type\":\"offsite_conversion.fb_pixel_add_to_cart\",\"value\":\"16\"},{\"action_type\":\"offsite_conversion.fb_pixel_initiate_checkout\",\"value\":\"5\"},{\"action_type\":\"offsite_conversion.fb_pixel_purchase\",\"value\":\"1\"},{\"action_type\":\"offsite_conversion.fb_pixel_search\",\"value\":\"3\"},{\"action_type\":\"offsite_conversion.fb_pixel_view_content\",\"value\":\"100\"},{\"action_type\":\"post_engagement\",\"value\":\"1830\"},{\"action_type\":\"page_engagement\",\"value\":\"1830\"},{\"action_type\":\"add_payment_info\",\"value\":\"5\"},{\"action_type\":\"omni_add_to_cart\",\"value\":\"16\"},{\"action_type\":\"omni_initiated_checkout\",\"value\":\"5\"},{\"action_type\":\"omni_purchase\",\"value\":\"1\"},{\"action_type\":\"omni_search\",\"value\":\"3\"},{\"action_type\":\"omni_view_content\",\"value\":\"100\"},{\"action_type\":\"add_to_cart\",\"value\":\"16\"},{\"action_type\":\"initiate_checkout\",\"value\":\"5\"},{\"action_type\":\"purchase\",\"value\":\"1\"},{\"action_type\":\"search\",\"value\":\"3\"},{\"action_type\":\"view_content\",\"value\":\"100\"}],\"date_start\":\"2021-08-07\",\"date_stop\":\"2021-08-07\"},{\"account_id\":\"522606278080331\",\"actions\":[{\"action_reaction\":\"like\",\"action_type\":\"post_reaction\",\"value\":\"7\"},{\"action_type\":\"onsite_conversion.post_save\",\"value\":\"3\"},{\"action_type\":\"link_click\",\"value\":\"130\"},{\"action_type\":\"post\",\"value\":\"1\"},{\"action_type\":\"post_reaction\",\"value\":\"7\"},{\"action_type\":\"video_view\",\"value\":\"2364\"},{\"action_type\":\"landing_page_view\",\"value\":\"92\"},{\"action_type\":\"offsite_conversion.fb_pixel_add_payment_info\",\"value\":\"2\"},{\"action_type\":\"offsite_conversion.fb_pixel_add_to_cart\",\"value\":\"7\"},{\"action_type\":\"offsite_conversion.fb_pixel_initiate_checkout\",\"value\":\"3\"},{\"action_type\":\"offsite_conversion.fb_pixel_purchase\",\"value\":\"1\"},{\"action_type\":\"offsite_conversion.fb_pixel_search\",\"value\":\"1\"},{\"action_type\":\"offsite_conversion.fb_pixel_view_content\",\"value\":\"131\"},{\"action_type\":\"post_engagement\",\"value\":\"2505\"},{\"action_type\":\"page_engagement\",\"value\":\"2505\"},{\"action_type\":\"add_payment_info\",\"value\":\"2\"},{\"action_type\":\"omni_add_to_cart\",\"value\":\"7\"},{\"action_type\":\"omni_initiated_checkout\",\"value\":\"3\"},{\"action_type\":\"omni_purchase\",\"value\":\"1\"},{\"action_type\":\"omni_search\",\"value\":\"1\"},{\"action_type\":\"omni_view_content\",\"value\":\"131\"},{\"action_type\":\"add_to_cart\",\"value\":\"7\"},{\"action_type\":\"initiate_checkout\",\"value\":\"3\"},{\"action_type\":\"purchase\",\"value\":\"1\"},{\"action_type\":\"search\",\"value\":\"1\"},{\"action_type\":\"view_content\",\"value\":\"131\"}],\"date_start\":\"2021-08-08\",\"date_stop\":\"2021-08-08\"},{\"account_id\":\"522606278080331\",\"actions\":[{\"action_reaction\":\"wow\",\"action_type\":\"post_reaction\",\"value\":\"1\"},{\"action_reaction\":\"like\",\"action_type\":\"post_reaction\",\"value\":\"55\"},{\"action_reaction\":\"love\",\"action_type\":\"post_reaction\",\"value\":\"2\"},{\"action_type\":\"comment\",\"value\":\"6\"},{\"action_type\":\"onsite_conversion.post_save\",\"value\":\"13\"},{\"action_type\":\"link_click\",\"value\":\"558\"},{\"action_type\":\"post\",\"value\":\"9\"},{\"action_type\":\"post_reaction\",\"value\":\"58\"},{\"action_type\":\"video_view\",\"value\":\"6771\"},{\"action_type\":\"landing_page_view\",\"value\":\"422\"},{\"action_type\":\"offsite_conversion.fb_pixel_add_payment_info\",\"value\":\"21\"},{\"action_type\":\"offsite_conversion.fb_pixel_add_to_cart\",\"value\":\"62\"},{\"action_type\":\"offsite_conversion.fb_pixel_complete_registration\",\"value\":\"3\"},{\"action_type\":\"offsite_conversion.fb_pixel_initiate_checkout\",\"value\":\"34\"},{\"action_type\":\"offsite_conversion.fb_pixel_purchase\",\"value\":\"11\"},{\"action_type\":\"offsite_conversion.fb_pixel_search\",\"value\":\"8\"},{\"action_type\":\"offsite_conversion.fb_pixel_view_content\",\"value\":\"542\"},{\"action_type\":\"post_engagement\",\"value\":\"7415\"},{\"action_type\":\"page_engagement\",\"value\":\"7415\"},{\"action_type\":\"add_payment_info\",\"value\":\"21\"},{\"action_type\":\"omni_add_to_cart\",\"value\":\"62\"},{\"action_type\":\"omni_complete_registration\",\"value\":\"3\"},{\"action_type\":\"omni_initiated_checkout\",\"value\":\"34\"},{\"action_type\":\"omni_purchase\",\"value\":\"11\"},{\"action_type\":\"omni_search\",\"value\":\"8\"},{\"action_type\":\"omni_view_content\",\"value\":\"542\"},{\"action_type\":\"add_to_cart\",\"value\":\"62\"},{\"action_type\":\"complete_registration\",\"value\":\"3\"},{\"action_type\":\"initiate_checkout\",\"value\":\"34\"},{\"action_type\":\"purchase\",\"value\":\"11\"},{\"action_type\":\"search\",\"value\":\"8\"},{\"action_type\":\"view_content\",\"value\":\"542\"}],\"date_start\":\"2021-08-09\",\"date_stop\":\"2021-08-09\"}],\"paging\":{\"cursors\":{\"before\":\"MAZDZD\",\"after\":\"MgZDZD\"}}}"} 11 | ){:method :get, 12 | :address 13 | "https://graph.facebook.com/v10.0/192025512975195?access_token=XXTOKENXX", 14 | :as :json} 15 | (fn [req]{:status 200, 16 | :body 17 | "{\"id\":\"192025512975195\",\"account_id\":\"522606278080331\",\"time_ref\":1628614274,\"time_completed\":1628614275,\"async_status\":\"Job Completed\",\"async_percent_completion\":100,\"date_start\":\"2021-08-07\",\"date_stop\":\"2021-08-09\"}"} 18 | ){:method :post, 19 | :address 20 | "https://graph.facebook.com/v10.0/act_522606278080331/insights?fields=account_id,campaign_id,actions&action_breakdowns=action_reaction&date_preset=last_3d&time_increment=1&level=account&access_token=XXTOKENXX", 21 | :as :json} 22 | (fn [req]{:status 200, :body "{\"report_run_id\":\"192025512975195\"}"} 23 | ) 24 | }) 25 | -------------------------------------------------------------------------------- /test/keboola/snapshots/asyncinisghtscampaigns/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "act_522606278080331" : { 6 | "account_id" : "522606278080331", 7 | "business_name" : "My bussiness", 8 | "currency" : "CZK", 9 | "id" : "act_522606278080331", 10 | "name" : "My bussines" 11 | } 12 | }, 13 | "api-version" : "v10.0", 14 | "queries" : [ { 15 | "name" : "query", 16 | "type" : "async-insights-query", 17 | "disabled" : false, 18 | "query" : { 19 | "parameters" : "fields=account_id,campaign_id,actions&action_breakdowns=action_reaction&date_preset=last_3d&time_increment=1&level=account", 20 | "ids" : "" 21 | } 22 | } ] 23 | }, 24 | "authorization" : { 25 | "oauth_api" : { 26 | "id" : "keboola.ex-facebook-ads", 27 | "credentials" : { 28 | "id" : "main", 29 | "authorizedFor" : "Myself", 30 | "creator" : { 31 | "id" : "1234", 32 | "description" : "me@keboola.com" 33 | }, 34 | "created" : "2016-01-31 00:13:30", 35 | "oauthVersion" : "facebook", 36 | "appKey" : "xxx", 37 | "#data" : "{\"token\":\"XXTOKENXX\"}", 38 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 39 | } 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/asyncinisghtscampaigns/out/tables/accounts: -------------------------------------------------------------------------------- 1 | account_id,name,business_name,currency,id 2 | 522606278080331,My bussines,My bussiness,CZK,act_522606278080331 3 | -------------------------------------------------------------------------------- /test/keboola/snapshots/asyncinisghtscampaigns/out/tables/query_insights.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["ex-account-id","fb-graph-node","parent-id","ads_action_name","action_type","action_reaction","value","account_id","date_start","date_stop"],"incremental":true,"primary_key":["parent_id","account_id","date_start","date_stop","ads_action_name","action_type","action_reaction"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/asyncinisghtscampaigns/test_asyncinisghtscampaigns.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.asyncinisghtscampaigns.test-asyncinisghtscampaigns 2 | (:require [keboola.snapshots.asyncinisghtscampaigns.apicalls :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest asyncinisghtscampaigns-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "asyncinisghtscampaigns"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "test/keboola/snapshots/asyncinisghtscampaigns") 17 | (test-utils/copy-config-tmp "test/keboola/snapshots/asyncinisghtscampaigns" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "test/keboola/snapshots/asyncinisghtscampaigns" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/snapshots/campaignsinsights/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "act_10152562141153995" : { 6 | "account_id" : "10152562141153995", 7 | "business_name" : "", 8 | "currency" : "EUR", 9 | "id" : "act_10152562141153995", 10 | "name" : "blabla" 11 | } 12 | }, 13 | "api-version" : "v2.9", 14 | "queries" : [ { 15 | "name" : "campaigns", 16 | "type" : "nested-query", 17 | "disabled" : false, 18 | "query" : { 19 | "path" : "campaigns", 20 | "fields" : "insights.action_breakdowns(action_reaction).date_preset(last_28d){account_id,account_name,campaign_id,campaign_name,impressions,clicks,spend,reach} ", 21 | "ids" : "act_10152562141153995,act_10152437784203995" 22 | } 23 | }, { 24 | "name" : "campaigns_insights_type", 25 | "type" : "nested-query", 26 | "disabled" : false, 27 | "query" : { 28 | "path" : "campaigns", 29 | "fields" : "insights.time_range({'since':'2017-04-01','until':'2017-04-06'}).action_breakdowns(action_type).time_increment(1){account_id,account_name,campaign_id,campaign_name,actions}", 30 | "ids" : "act_10152562141153995,act_10152437784203995" 31 | } 32 | }, { 33 | "name" : "campaigns_insights_reaction", 34 | "type" : "nested-query", 35 | "disabled" : false, 36 | "query" : { 37 | "path" : "campaigns", 38 | "fields" : "insights.action_breakdowns(action_reaction).date_preset(last_3d).time_increment(1){account_id,account_name,campaign_id,campaign_name,actions}", 39 | "ids" : "act_10152562141153995,act_10152437784203995" 40 | } 41 | } ] 42 | }, 43 | "authorization" : { 44 | "oauth_api" : { 45 | "id" : "keboola.ex-facebook-ads", 46 | "credentials" : { 47 | "id" : "main", 48 | "authorizedFor" : "Myself", 49 | "creator" : { 50 | "id" : "1234", 51 | "description" : "me@keboola.com" 52 | }, 53 | "created" : "2016-01-31 00:13:30", 54 | "oauthVersion" : "facebook", 55 | "appKey" : "xxx", 56 | "#data" : "{\"token\":\"XXTOKENXX\"}", 57 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 58 | } 59 | } 60 | } 61 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/campaignsinsights/out/tables/accounts: -------------------------------------------------------------------------------- 1 | account_id,name,business_name,currency,id 2 | 10152562141153995,blabla,,EUR,act_10152562141153995 3 | -------------------------------------------------------------------------------- /test/keboola/snapshots/campaignsinsights/out/tables/campaigns_insights.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["ex-account-id","fb-graph-node","parent-id","account_id","account_name","campaign_id","campaign_name","clicks","date_start","date_stop","impressions","reach","spend"],"incremental":true,"primary_key":["parent_id","account_id","campaign_id","date_start","date_stop"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/campaignsinsights/out/tables/campaigns_insights/1507031538125: -------------------------------------------------------------------------------- 1 | "act_10152437784203995","page_campaigns_insights","6074689556670","10152437784203995","L7wDe","6074689556670","iJpYG","1318","2017-09-05","2017-10-02","33258","21275","166" 2 | "act_10152437784203995","page_campaigns_insights","6074690268470","10152437784203995","tbJHb","6074690268470","a38Vu","638","2017-09-05","2017-10-02","108569","49464","392.02" 3 | "act_10152437784203995","page_campaigns_insights","6077413250870","10152437784203995","ypee3","6077413250870","6iuz6","2281","2017-09-05","2017-10-02","130444","63973","683.53" 4 | "act_10152437784203995","page_campaigns_insights","6077417491670","10152437784203995","4IYqn","6077417491670","2CEbM","905","2017-09-05","2017-10-02","315985","116782","496.38" 5 | "act_10152437784203995","page_campaigns_insights","6077980757070","10152437784203995","W7Yd9","6077980757070","6IBcd","549","2017-09-05","2017-10-02","11893","11743","30.9" 6 | -------------------------------------------------------------------------------- /test/keboola/snapshots/campaignsinsights/out/tables/campaigns_insights_reaction_insights.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["ex-account-id","fb-graph-node","parent-id","ads_action_name","action_type","action_reaction","value","account_id","account_name","campaign_id","campaign_name","date_start","date_stop"],"incremental":true,"primary_key":["parent_id","account_id","campaign_id","date_start","date_stop","ads_action_name","action_type","action_reaction"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/campaignsinsights/out/tables/campaigns_insights_reaction_insights/1507031560441: -------------------------------------------------------------------------------- 1 | "act_10152437784203995","page_campaigns_insights","6074690268470","actions","post_engagement","","7497","10152437784203995","QmZ4l","6074690268470","FYqnH","2017-10-02","2017-10-02" 2 | "act_10152437784203995","page_campaigns_insights","6074690268470","actions","page_engagement","","7497","10152437784203995","QmZ4l","6074690268470","FYqnH","2017-10-02","2017-10-02" 3 | "act_10152437784203995","page_campaigns_insights","6074690268470","actions","post_reaction","like","3","10152437784203995","QmZ4l","6074690268470","FYqnH","2017-10-02","2017-10-02" 4 | "act_10152437784203995","page_campaigns_insights","6074690268470","actions","video_view","","7469","10152437784203995","QmZ4l","6074690268470","FYqnH","2017-10-02","2017-10-02" 5 | "act_10152437784203995","page_campaigns_insights","6074690268470","actions","post","","2","10152437784203995","QmZ4l","6074690268470","FYqnH","2017-10-02","2017-10-02" 6 | "act_10152437784203995","page_campaigns_insights","6074690268470","actions","link_click","","22","10152437784203995","QmZ4l","6074690268470","FYqnH","2017-10-02","2017-10-02" 7 | "act_10152437784203995","page_campaigns_insights","6074690268470","actions","comment","","1","10152437784203995","QmZ4l","6074690268470","FYqnH","2017-10-02","2017-10-02" 8 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","post_engagement","","2456","10152437784203995","JU1Ti","6077413250870","BqdkT","2017-10-02","2017-10-02" 9 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","page_engagement","","2458","10152437784203995","JU1Ti","6077413250870","BqdkT","2017-10-02","2017-10-02" 10 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","post_reaction","like","24","10152437784203995","JU1Ti","6077413250870","BqdkT","2017-10-02","2017-10-02" 11 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","video_view","","2297","10152437784203995","JU1Ti","6077413250870","BqdkT","2017-10-02","2017-10-02" 12 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","post","","1","10152437784203995","JU1Ti","6077413250870","BqdkT","2017-10-02","2017-10-02" 13 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","link_click","","134","10152437784203995","JU1Ti","6077413250870","BqdkT","2017-10-02","2017-10-02" 14 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","like","","2","10152437784203995","JU1Ti","6077413250870","BqdkT","2017-10-02","2017-10-02" 15 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","post_engagement","","2742","10152437784203995","R9B0i","6077413250870","ISlja","2017-10-01","2017-10-01" 16 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","page_engagement","","2747","10152437784203995","R9B0i","6077413250870","ISlja","2017-10-01","2017-10-01" 17 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","post_reaction","love","1","10152437784203995","R9B0i","6077413250870","ISlja","2017-10-01","2017-10-01" 18 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","post_reaction","like","45","10152437784203995","R9B0i","6077413250870","ISlja","2017-10-01","2017-10-01" 19 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","video_view","","2547","10152437784203995","R9B0i","6077413250870","ISlja","2017-10-01","2017-10-01" 20 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","link_click","","149","10152437784203995","R9B0i","6077413250870","ISlja","2017-10-01","2017-10-01" 21 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","like","","5","10152437784203995","R9B0i","6077413250870","ISlja","2017-10-01","2017-10-01" 22 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","post_engagement","","2176","10152437784203995","qFY9s","6077413250870","PtYyF","2017-09-30","2017-09-30" 23 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","page_engagement","","2177","10152437784203995","qFY9s","6077413250870","PtYyF","2017-09-30","2017-09-30" 24 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","post_reaction","like","36","10152437784203995","qFY9s","6077413250870","PtYyF","2017-09-30","2017-09-30" 25 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","video_view","","2022","10152437784203995","qFY9s","6077413250870","PtYyF","2017-09-30","2017-09-30" 26 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","link_click","","118","10152437784203995","qFY9s","6077413250870","PtYyF","2017-09-30","2017-09-30" 27 | "act_10152437784203995","page_campaigns_insights","6077413250870","actions","like","","1","10152437784203995","qFY9s","6077413250870","PtYyF","2017-09-30","2017-09-30" 28 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","post_engagement","","3448","10152437784203995","RGEZg","6077417491670","b1LRp","2017-10-02","2017-10-02" 29 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","page_engagement","","3448","10152437784203995","RGEZg","6077417491670","b1LRp","2017-10-02","2017-10-02" 30 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","video_view","","3352","10152437784203995","RGEZg","6077417491670","b1LRp","2017-10-02","2017-10-02" 31 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","link_click","","96","10152437784203995","RGEZg","6077417491670","b1LRp","2017-10-02","2017-10-02" 32 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","post_engagement","","3404","10152437784203995","JSUXq","6077417491670","o1Yj3","2017-10-01","2017-10-01" 33 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","page_engagement","","3404","10152437784203995","JSUXq","6077417491670","o1Yj3","2017-10-01","2017-10-01" 34 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","video_view","","3325","10152437784203995","JSUXq","6077417491670","o1Yj3","2017-10-01","2017-10-01" 35 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","link_click","","79","10152437784203995","JSUXq","6077417491670","o1Yj3","2017-10-01","2017-10-01" 36 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","post_engagement","","3051","10152437784203995","XhW5S","6077417491670","BSnWN","2017-09-30","2017-09-30" 37 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","page_engagement","","3051","10152437784203995","XhW5S","6077417491670","BSnWN","2017-09-30","2017-09-30" 38 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","video_view","","2983","10152437784203995","XhW5S","6077417491670","BSnWN","2017-09-30","2017-09-30" 39 | "act_10152437784203995","page_campaigns_insights","6077417491670","actions","link_click","","68","10152437784203995","XhW5S","6077417491670","BSnWN","2017-09-30","2017-09-30" 40 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","post_engagement","","291","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 41 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","page_engagement","","300","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 42 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","post_reaction","angry","1","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 43 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","post_reaction","love","6","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 44 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","post_reaction","like","51","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 45 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","post","","6","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 46 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","link_click","","215","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 47 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","like","","9","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 48 | "act_10152437784203995","page_campaigns_insights","6077980757070","actions","comment","","12","10152437784203995","SWpLR","6077980757070","4RDvi","2017-10-02","2017-10-02" 49 | -------------------------------------------------------------------------------- /test/keboola/snapshots/campaignsinsights/out/tables/campaigns_insights_type_insights.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["ex-account-id","fb-graph-node","parent-id","ads_action_name","action_type","value","account_id","account_name","campaign_id","campaign_name","date_start","date_stop"],"incremental":true,"primary_key":["parent_id","account_id","campaign_id","date_start","date_stop","ads_action_name","action_type"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/campaignsinsights/test_campaignsinsights.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.campaignsinsights.test-campaignsinsights 2 | (:require [keboola.snapshots.campaignsinsights.apicalls :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest campaignsinsights-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "campaignsinsights"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "test/keboola/snapshots/campaignsinsights") 17 | (test-utils/copy-config-tmp "test/keboola/snapshots/campaignsinsights" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "test/keboola/snapshots/campaignsinsights" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/snapshots/core.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.core 2 | (:require [cheshire.core :refer [generate-stream]] 3 | [clj-http.fake :refer [with-global-fake-routes-in-isolation]] 4 | [clojure.test :as t :refer :all] 5 | clostache.parser 6 | [keboola.docker.config :refer [load-config user-credentials get-fb-token]] 7 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 10 | [keboola.http.client :refer [GET]] 11 | [keboola.http.recording 12 | :refer 13 | [recording 14 | reset-recording 15 | save-current-recording 16 | turn-recording-off 17 | turn-recording-on]] 18 | [keboola.test-utils.core :as test-utils]) 19 | (:import java.io.File)) 20 | 21 | (defn create-test-file [dir-path ns-name recording-ns test-name] 22 | (let [test-file-path (str dir-path "/" "test" "_" test-name ".clj") 23 | test-ns (str ns-name ".test-" test-name) 24 | template {:ns-name test-ns 25 | :apicalls-ns recording-ns 26 | :test-name test-name 27 | :dir-path dir-path} 28 | test-content-template (slurp "test/keboola/snapshots/template.mustache") 29 | test-file-content (clostache.parser/render test-content-template template)] 30 | (with-open [w (clojure.java.io/writer test-file-path)] 31 | (.write w test-file-content)))) 32 | 33 | (defn clean-test-directory [dirpath] 34 | (println "cleaning test directory " dirpath) 35 | (let [files (.listFiles (File. dirpath)) 36 | filtered-files (filter #(not= (.getName %) "config.json") files)] 37 | (doseq [f filtered-files] 38 | (if (.isDirectory f) 39 | (test-utils/delete-recursively f) 40 | (clojure.java.io/delete-file f))))) 41 | 42 | (defn save-token-to-config! [dirpath token] 43 | (let [config (load-config dirpath) 44 | oauth-data-path [:authorization :oauth_api :credentials :#data] 45 | anonymized-oauth-data (str "{\"token\":\"" token "\"}") 46 | anonymized-config (assoc-in config oauth-data-path anonymized-oauth-data) 47 | path (str dirpath "/config.json")] 48 | (println "saving token" (subs token 0 2) " into " dirpath) 49 | (generate-stream anonymized-config (clojure.java.io/writer path) {:pretty true}))) 50 | 51 | (defn anonymize-config-token [dirpath] 52 | (save-token-to-config! dirpath "XXTOKENXX")) 53 | 54 | (defn get-token-from-env [component-id] 55 | (let [env-name 56 | (get { 57 | "keboola.ex-facebook" "FB_TOKEN" 58 | "keboola.ex-facebook-ads" "FB_ADS_TOKEN" 59 | } component-id)] 60 | (if env-name (System/getenv env-name)))) 61 | 62 | (defn save-config-token-from-env [dirpath] 63 | (let [config (load-config dirpath) 64 | component-id (get-in config [:authorization :oauth_api :id]) 65 | token (get-token-from-env component-id)] 66 | (if token 67 | (do (println "Read token from env for " component-id) 68 | (save-token-to-config! dirpath token)) 69 | (println "No token read for" component-id)))) 70 | 71 | (defn generate-test 72 | ([dirname] (generate-test dirname true)) 73 | ([dirname anonymize-token?] 74 | (let [clj-compliant-name (clojure.string/replace dirname #"_" "-") 75 | ns-name (str "keboola.snapshots." clj-compliant-name) 76 | dir-path (str "test/keboola/snapshots/" dirname) 77 | recording-path (str "test/keboola/snapshots/" dirname "/apicalls.clj") 78 | recording-ns (str ns-name ".apicalls")] 79 | (disable-log-token) 80 | (turn-recording-on) 81 | (clean-test-directory dir-path) 82 | (reset-recording) 83 | (reset-columns-map) 84 | (save-config-token-from-env dir-path) 85 | (prepare-and-run dir-path) 86 | (println "saving apicalls in " dirname) 87 | (save-current-recording recording-path recording-ns (get-fb-token (user-credentials dir-path))) 88 | (println "creating test file " clj-compliant-name) 89 | (create-test-file dir-path ns-name recording-ns clj-compliant-name) 90 | (turn-recording-off) 91 | (if anonymize-token? (anonymize-config-token dir-path))))) 92 | 93 | (defn regenerate-all-snapshot-dirs 94 | ([] (regenerate-all-snapshot-dirs nil)) 95 | ([dirfilter] 96 | (let [snapshot-dirs (filter #(.isDirectory %) (.listFiles (File. "test/keboola/snapshots"))) 97 | dir-names (map #(.getName %) snapshot-dirs) 98 | filter-regexp (re-pattern (or dirfilter "")) 99 | dirs-filtered (filter #(re-find filter-regexp %) dir-names)] 100 | 101 | (println "found snapshot dirs(" dirfilter "):" dirs-filtered) 102 | (doseq [dname dirs-filtered] 103 | (generate-test dname))))) 104 | -------------------------------------------------------------------------------- /test/keboola/snapshots/feed/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "177057932317550" : { 6 | "id" : "177057932317550", 7 | "name" : "keboola", 8 | "category" : "software" 9 | } 10 | }, 11 | "api-version" : "v2.11", 12 | "queries" : [ { 13 | "name" : "feed", 14 | "type" : "nested-query", 15 | "disabled" : false, 16 | "query" : { 17 | "path" : "feed", 18 | "fields" : "caption,message,created_time,type,description,likes{name,username},comments{message,created_time,from,likes{name,username}}", 19 | "ids" : "177057932317550", 20 | "since" : "3 years ago", 21 | "until" : "now" 22 | } 23 | } ] 24 | }, 25 | "authorization" : { 26 | "oauth_api" : { 27 | "id" : "keboola.ex-facebook", 28 | "credentials" : { 29 | "id" : "main", 30 | "authorizedFor" : "Myself", 31 | "creator" : { 32 | "id" : "1234", 33 | "description" : "me@keboola.com" 34 | }, 35 | "created" : "2016-01-31 00:13:30", 36 | "oauthVersion" : "facebook", 37 | "appKey" : "xxx", 38 | "#data" : "{\"token\":\"XXTOKENXX\"}", 39 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 40 | } 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/feed/out/tables/accounts: -------------------------------------------------------------------------------- 1 | category,name,id 2 | software,keboola,177057932317550 3 | -------------------------------------------------------------------------------- /test/keboola/snapshots/feed/out/tables/feed.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["id","ex-account-id","fb-graph-node","parent-id","caption","created_time","description","message","type"],"incremental":true,"primary_key":["parent_id","id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/feed/out/tables/feed/1516621526977: -------------------------------------------------------------------------------- 1 | "177057932317550_863179447038725","177057932317550","page_feed","177057932317550","MoUVi","2015-01-22T16:06:00+0000","QUkJ7","rN5UJ","link" 2 | "177057932317550_867662119923791","177057932317550","page_feed","177057932317550","TOODK","2015-02-01T22:50:49+0000","tlKaK","O7mBO","link" 3 | "177057932317550_867918849898118","177057932317550","page_feed","177057932317550","obTSR","2015-02-02T08:52:14+0000","JWWW4","0kzPw","link" 4 | "177057932317550_868084153214921","177057932317550","page_feed","177057932317550","eE4uR","2015-02-02T17:30:06+0000","","iEx5Z","link" 5 | "177057932317550_868201209869882","177057932317550","page_feed","177057932317550","MK4jq","2015-02-03T01:03:20+0000","abWhz","GHYIM","link" 6 | "177057932317550_868721253151211","177057932317550","page_feed","177057932317550","coxVD","2015-02-04T08:19:07+0000","0XtVU","t1TNL","link" 7 | "177057932317550_868726169817386","177057932317550","page_feed","177057932317550","P5xhN","2015-02-04T08:47:55+0000","TlTHw","6CPSm","link" 8 | "177057932317550_871299946226675","177057932317550","page_feed","177057932317550","8cQiu","2015-02-10T09:06:05+0000","eFikT","IaYBi","link" 9 | "177057932317550_871542359535767","177057932317550","page_feed","177057932317550","sPl83","2015-02-10T23:04:55+0000","VaLf1","Wbfgj","link" 10 | "177057932317550_871770149512988","177057932317550","page_feed","177057932317550","Jcr3G","2015-02-11T12:46:47+0000","6ziKG","m3hJh","link" 11 | "177057932317550_876365615720108","177057932317550","page_feed","177057932317550","20yg1","2015-02-19T21:42:46+0000","06Ejk","54qbI","link" 12 | "177057932317550_876391385717531","177057932317550","page_feed","177057932317550","aYyDG","2015-02-19T22:57:01+0000","oNCUx","PzGFC","link" 13 | "177057932317550_876759985680671","177057932317550","page_feed","177057932317550","jW53B","2015-02-20T19:01:57+0000","ki8t1","P4GzX","link" 14 | "177057932317550_878717075484962","177057932317550","page_feed","177057932317550","olONV","2015-02-24T10:49:45+0000","IH8FT","T8f6s","link" 15 | "177057932317550_878834132139923","177057932317550","page_feed","177057932317550","rUc3R","2015-02-24T16:55:24+0000","Nyb3l","63vKu","link" 16 | "177057932317550_880197812003555","177057932317550","page_feed","177057932317550","sGhkz","2015-02-27T18:53:14+0000","5qNaz","THiDT","link" 17 | "177057932317550_882565748433428","177057932317550","page_feed","177057932317550","sHB38","2015-03-05T14:15:36+0000","FDzCS","g0kbV","link" 18 | "177057932317550_890214517668551","177057932317550","page_feed","177057932317550","","2015-03-19T02:27:18+0000","","XJXV3","photo" 19 | "177057932317550_894591280564208","177057932317550","page_feed","177057932317550","m43yp","2015-03-25T18:39:57+0000","OjJy5","7B6pQ","link" 20 | "177057932317550_894908463865823","177057932317550","page_feed","177057932317550","G5OF8","2015-03-26T09:38:34+0000","RzIc3","i60mK","link" 21 | "177057932317550_895159487174054","177057932317550","page_feed","177057932317550","w0ysY","2015-03-26T19:20:42+0000","VoUzP","D3XJ2","link" 22 | "177057932317550_895172080506128","177057932317550","page_feed","177057932317550","Q3LcJ","2015-03-26T19:54:02+0000","3lzUN","FJiGM","link" 23 | "177057932317550_895638570459479","177057932317550","page_feed","177057932317550","6norZ","2015-03-27T13:59:19+0000","YMgXa","0OBCV","link" 24 | "177057932317550_895669693789700","177057932317550","page_feed","177057932317550","P1SRB","2015-03-27T15:31:12+0000","IncFv","I84Tz","link" 25 | "177057932317550_897828396907163","177057932317550","page_feed","177057932317550","YLFUs","2015-03-31T08:56:02+0000","xk53D","gp8wX","link" 26 | "177057932317550_898249010198435","177057932317550","page_feed","177057932317550","wd4eW","2015-04-01T01:22:18+0000","pB4mm","Ih4zf","link" 27 | "177057932317550_898322620191074","177057932317550","page_feed","177057932317550","x0Leb","2015-04-01T04:52:15+0000","M5qHU","eeh6Q","link" 28 | "177057932317550_898705676819435","177057932317550","page_feed","177057932317550","hDQL5","2015-04-01T23:26:00+0000","ETRtQ","dsSlg","link" 29 | "177057932317550_899104843446185","177057932317550","page_feed","177057932317550","4qrUY","2015-04-02T17:53:54+0000","Oylr4","LLLXJ","link" 30 | "177057932317550_899507860072550","177057932317550","page_feed","177057932317550","kL4jo","2015-04-03T10:05:14+0000","ydoVL","KGLmo","link" 31 | "177057932317550_902157386474264","177057932317550","page_feed","177057932317550","1qDpf","2015-04-08T06:32:14+0000","DO4HV","I7vhf","link" 32 | "177057932317550_903010183055651","177057932317550","page_feed","177057932317550","NuLrs","2015-04-10T09:20:52+0000","ZaYaF","SPqMU","link" 33 | "177057932317550_905758402780829","177057932317550","page_feed","177057932317550","q1hfJ","2015-04-15T17:48:02+0000","HDxyF","Dev4K","link" 34 | "177057932317550_908277042528965","177057932317550","page_feed","177057932317550","MywPj","2015-04-20T10:55:26+0000","tQiat","gbll9","link" 35 | "177057932317550_910073965682606","177057932317550","page_feed","177057932317550","geNPo","2015-04-23T08:10:30+0000","mptUF","CO576","link" 36 | "177057932317550_913371222019547","177057932317550","page_feed","177057932317550","WLOKX","2015-04-28T09:40:20+0000","xNMkD","8e1tH","link" 37 | "177057932317550_913890861967583","177057932317550","page_feed","177057932317550","VpRap","2015-04-29T05:09:50+0000","VKkL1","kU1CX","link" 38 | "177057932317550_921759614514041","177057932317550","page_feed","177057932317550","","2015-05-16T06:54:46+0000","","","photo" 39 | "177057932317550_921776967845639","177057932317550","page_feed","177057932317550","","2015-05-16T08:15:15+0000","","","photo" 40 | "177057932317550_932914176731918","177057932317550","page_feed","177057932317550","","2015-06-08T11:10:56+0000","","5osh1","photo" 41 | "177057932317550_940984532591549","177057932317550","page_feed","177057932317550","","2015-06-25T02:16:30+0000","","F9OaB","photo" 42 | "177057932317550_990386480984687","177057932317550","page_feed","177057932317550","","2015-10-14T05:03:09+0000","","Qy6me","photo" 43 | "177057932317550_1043848705638464","177057932317550","page_feed","177057932317550","","2016-02-09T18:55:03+0000","","bllJY","status" 44 | "177057932317550_10207570586690352","177057932317550","page_feed","177057932317550","axmGB","2016-02-10T14:10:10+0000","V7keH","bXFqB","link" 45 | "177057932317550_1126854757337858","177057932317550","page_feed","177057932317550","","2016-06-22T06:23:24+0000","Md2lK","","video" 46 | "177057932317550_1379478452075486","177057932317550","page_feed","177057932317550","qycft","2017-04-06T20:27:49+0000","XM11E","Ysr2x","link" 47 | "177057932317550_10155365834019810","177057932317550","page_feed","177057932317550","","2017-05-25T22:38:42+0000","","m9Mef","photo" 48 | "177057932317550_177062472317096","177057932317550","page_feed","177057932317550","","2017-07-13T14:20:00+0000","","","photo" 49 | "177057932317550_1477131262310204","177057932317550","page_feed","177057932317550","","2017-07-13T14:42:10+0000","","2qulO","photo" 50 | "177057932317550_1478486035508060","177057932317550","page_feed","177057932317550","","2017-07-14T17:50:48+0000","","XU1Pt","photo" 51 | "177057932317550_200219200511028","177057932317550","page_feed","177057932317550","BvW1U","2017-07-19T03:17:27+0000","XskdM","zSiDU","event" 52 | "177057932317550_1920132124904149","177057932317550","page_feed","177057932317550","XTs78","2017-09-04T11:15:47+0000","ISHXC","8EIgi","event" 53 | "177057932317550_1528270307196299","177057932317550","page_feed","177057932317550","","2017-09-06T20:53:56+0000","","","photo" 54 | -------------------------------------------------------------------------------- /test/keboola/snapshots/feed/out/tables/feed_comments.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["id","ex-account-id","fb-graph-node","parent-id","created_time","from_id","from_name","message"],"incremental":true,"primary_key":["parent_id","id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/feed/out/tables/feed_comments/1516621526976: -------------------------------------------------------------------------------- 1 | "932914176731918_967071379982864","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-08-23T02:03:58+0000","","","LdJiP" 2 | "932914176731918_933521926671143","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-09T13:24:55+0000","177057932317550","fiEi7","FpZoh" 3 | "932914176731918_933191330037536","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T22:37:41+0000","","","6UEzz" 4 | "932914176731918_933149790041690","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T20:04:40+0000","","","rp0IC" 5 | "932914176731918_933149403375062","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T20:02:57+0000","","","grvPh" 6 | "932914176731918_933067250049944","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T18:36:51+0000","","","GIemL" 7 | "932914176731918_933056966717639","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T17:51:05+0000","","","NIIOY" 8 | "932914176731918_933055396717796","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T17:44:02+0000","","","spHHi" 9 | "932914176731918_933055010051168","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T17:42:19+0000","","","Idp0c" 10 | "932914176731918_933048243385178","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T17:22:09+0000","","","ZSyUO" 11 | "932914176731918_933040360052633","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T16:50:01+0000","","","bmhBP" 12 | "932914176731918_932938540062815","177057932317550","page_feed_comments","177057932317550_932914176731918","2015-06-08T12:22:04+0000","","","SOu8i" 13 | "10155365834019810_10155365853274810","177057932317550","page_feed_comments","177057932317550_10155365834019810","2017-05-25T22:46:44+0000","","","ifSve" 14 | "1477131262310204_1477461532277177","177057932317550","page_feed_comments","177057932317550_1477131262310204","2017-07-13T16:51:54+0000","","","08glB" 15 | "1478486035508060_1478596215497042","177057932317550","page_feed_comments","177057932317550_1478486035508060","2017-07-14T20:05:10+0000","177057932317550","2bh8M","VSZDW" 16 | -------------------------------------------------------------------------------- /test/keboola/snapshots/feed/test_feed.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.feed.test-feed 2 | (:require [keboola.snapshots.feed.apicalls :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest feed-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "feed"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "test/keboola/snapshots/feed") 17 | (test-utils/copy-config-tmp "test/keboola/snapshots/feed" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "test/keboola/snapshots/feed" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/snapshots/feedsummary/apicalls.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.feedsummary.apicalls) 2 | (def recorded 3 | { 4 | {:method :get, 5 | :address 6 | "https://graph.facebook.com/v2.8/177057932317550/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&since=15+months+ago&access_token=XXTOKENXX&limit=25&until=1491510469&__paging_token=enc_AdD1oKv62UShBGNFboAc4BJQnLMmQe5XbMXmjpmZAZCaEHTBgxi82ZCTM1ZB7nLSbl6f36azCbkjDLTupuX2RnnefrSEwI2oqRAkV0bPk89hPV1fGgZDZD", 7 | :as :json} 8 | (fn [req]{:status 200, :body "{\"data\":[]}"} 9 | ){:method :get, 10 | :address 11 | "https://graph.facebook.com/v2.8/222838661196260/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&limit=25&since=15+months+ago&__paging_token=enc_AdD6exG837adWHWfiNZCh9E5jFhpIWpZCyUchp2ZBtGYx3mWG93ylygF8GHU2uHa16T04GVCDubzzswdTnLRnZBanz0W1yxhdzw5saXUF6xFyPrSjAZDZD&access_token=XXTOKENXX&until=1477415790", 12 | :as :json} 13 | (fn [req]{:status 200, :body "{\"data\":[]}"} 14 | ){:method :get, 15 | :address 16 | "https://graph.facebook.com/v2.8/222838661196260/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&since=15+months+ago&access_token=XXTOKENXX&limit=25&until=1489004402&__paging_token=enc_AdCuBoNYLUHevwT3vXrsR6y8qggK9gZC0h6niHFIdwpcHSEEJE4ZA0XrureBrHVIj16XPvG6zCxZCOuXTIF6Ouan6OlE8VYnUM2rbKv7isdwcR0ZBgZDZD", 17 | :as :json} 18 | (fn [req]{:status 200, 19 | :body 20 | "{\"data\":[{\"id\":\"222838661196260_1102748413205276\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1102622523217865\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":4}}},{\"id\":\"222838661196260_1101117760035008\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":2}}},{\"id\":\"222838661196260_1095366693943448\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":9}}},{\"id\":\"222838661196260_1088426111304173\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":3}}},{\"id\":\"222838661196260_1085738481572936\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1082965645183553\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1077902842356500\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1071468542999930\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":2}}},{\"id\":\"222838661196260_1369870156416993\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":2}}},{\"id\":\"222838661196260_1047848388695279\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1024914097655375\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":5}}},{\"id\":\"222838661196260_1020713021408816\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":4}}},{\"id\":\"222838661196260_1019388544874597\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1005566356256816\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":11}}},{\"id\":\"222838661196260_1341134989232514\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}}],\"paging\":{\"previous\":\"https://graph.facebook.com/v2.8/222838661196260/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&limit=25&since=1489002753&__paging_token=enc_AdCf79If9Q1cYmZBtUEYntE6ZA9GoN9XDLweRMPp4y4bB5t1GsHpIzSYb3ofnYIp8W9n4QkxuqFwpxJJp8uu2B2TAMGqjMYVd0UtbqS8v0ErqStQZDZD&access_token=XXTOKENXX&__previous=1\",\"next\":\"https://graph.facebook.com/v2.8/222838661196260/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&limit=25&since=15+months+ago&__paging_token=enc_AdD6exG837adWHWfiNZCh9E5jFhpIWpZCyUchp2ZBtGYx3mWG93ylygF8GHU2uHa16T04GVCDubzzswdTnLRnZBanz0W1yxhdzw5saXUF6xFyPrSjAZDZD&access_token=XXTOKENXX&until=1477415790\"}}"} 21 | ){:method :get, 22 | :address "https://graph.facebook.com/v2.8/feed", 23 | :as :json, 24 | :query-params 25 | {:since "15 months ago", 26 | :until "now", 27 | :path "feed", 28 | :fields "reactions.type(LIKE).summary(total_count).limit(0)", 29 | :ids "222838661196260,177057932317550", 30 | :access_token "XXTOKENXX"}} 31 | (fn [req]{:status 200, 32 | :body 33 | "{\"222838661196260\":{\"data\":[{\"id\":\"222838661196260_1297263780420404\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":5}}},{\"id\":\"222838661196260_1295611950585587\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1284921258321323\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":5}}},{\"id\":\"222838661196260_1275430289270420\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":3}}},{\"id\":\"222838661196260_1272432166236899\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1272431919570257\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1272431639570285\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":3}}},{\"id\":\"222838661196260_144808662915978\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1268621089951340\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":6}}},{\"id\":\"222838661196260_1261976987282417\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1261140494032733\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":6}}},{\"id\":\"222838661196260_1257750477705068\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1256173724529410\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":5}}},{\"id\":\"222838661196260_1219971791482937\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":1}}},{\"id\":\"222838661196260_1169662839847166\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1166746836805433\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1166746623472121\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1117831915030259\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":2}}},{\"id\":\"222838661196260_1116959565117494\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1108005789346205\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":1}}},{\"id\":\"222838661196260_1107482036065247\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1107481849398599\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_1260350687419830\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"222838661196260_412927135725672\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":3}}},{\"id\":\"222838661196260_1102759903204127\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}}],\"paging\":{\"previous\":\"https://graph.facebook.com/v2.8/222838661196260/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&since=1513250622&access_token=XXTOKENXX&limit=25&__paging_token=enc_AdBMth1ZBGix7nrlHzm9YYnFHOoJaRthpYdeNNpZC3lRIIfQwSnPWyo8xaCR30FbKOZCUv2Qq8nHFaruopjLaez3E2lBFQPuOmZByacHJRqtLHP69QZDZD&__previous=1\",\"next\":\"https://graph.facebook.com/v2.8/222838661196260/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&since=15+months+ago&access_token=XXTOKENXX&limit=25&until=1489004402&__paging_token=enc_AdCuBoNYLUHevwT3vXrsR6y8qggK9gZC0h6niHFIdwpcHSEEJE4ZA0XrureBrHVIj16XPvG6zCxZCOuXTIF6Ouan6OlE8VYnUM2rbKv7isdwcR0ZBgZDZD\"}},\"177057932317550\":{\"data\":[{\"id\":\"177057932317550_1528270233862973\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"177057932317550_1920132124904149\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":4}}},{\"id\":\"177057932317550_200219200511028\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"177057932317550_1478486035508060\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":5}}},{\"id\":\"177057932317550_1477131262310204\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"177057932317550_177062472317096\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":0}}},{\"id\":\"177057932317550_10155365834019810\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":22}}},{\"id\":\"177057932317550_1379478452075486\",\"reactions\":{\"data\":[],\"summary\":{\"total_count\":2}}}],\"paging\":{\"previous\":\"https://graph.facebook.com/v2.8/177057932317550/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&since=1504731236&access_token=XXTOKENXX&limit=25&__paging_token=enc_AdCblm5DxAYalGbkvNQeoMZAf7xBZBUGbpBvocQACIvFinGIcpelcRK4BKIpZA94s9uraoZBBJPKGV0T7cOgH3ej2UyQxJz1j47ZBK9ZB7WJHzXplCtwZDZD&__previous=1\",\"next\":\"https://graph.facebook.com/v2.8/177057932317550/feed?fields=reactions.type%28LIKE%29.summary%28total_count%29.limit%280%29&since=15+months+ago&access_token=XXTOKENXX&limit=25&until=1491510469&__paging_token=enc_AdD1oKv62UShBGNFboAc4BJQnLMmQe5XbMXmjpmZAZCaEHTBgxi82ZCTM1ZB7nLSbl6f36azCbkjDLTupuX2RnnefrSEwI2oqRAkV0bPk89hPV1fGgZDZD\"}}}"} 34 | ) 35 | }) -------------------------------------------------------------------------------- /test/keboola/snapshots/feedsummary/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "222838661196260" : { 6 | "id" : "222838661196260", 7 | "name" : "hudebny sh", 8 | "category" : "entertaiment" 9 | }, 10 | "177057932317550" : { 11 | "id" : "177057932317550", 12 | "name" : "keboola", 13 | "category" : "software" 14 | } 15 | }, 16 | "api-version" : "v2.8", 17 | "queries" : [ { 18 | "name" : "feed_likes", 19 | "type" : "nested-query", 20 | "disabled" : false, 21 | "query" : { 22 | "since" : "15 months ago", 23 | "until" : "now", 24 | "path" : "feed", 25 | "fields" : "reactions.type(LIKE).summary(total_count).limit(0)", 26 | "ids" : null 27 | } 28 | } ] 29 | }, 30 | "authorization" : { 31 | "oauth_api" : { 32 | "id" : "keboola.ex-facebook", 33 | "credentials" : { 34 | "id" : "main", 35 | "authorizedFor" : "Myself", 36 | "creator" : { 37 | "id" : "1234", 38 | "description" : "me@keboola.com" 39 | }, 40 | "created" : "2016-01-31 00:13:30", 41 | "oauthVersion" : "facebook", 42 | "appKey" : "xxx", 43 | "#data" : "{\"token\":\"XXTOKENXX\"}", 44 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 45 | } 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/feedsummary/out/tables/accounts: -------------------------------------------------------------------------------- 1 | category,name,id 2 | entertaiment,hudebny sh,222838661196260 3 | software,keboola,177057932317550 4 | -------------------------------------------------------------------------------- /test/keboola/snapshots/feedsummary/out/tables/feed_likes_summary.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["ex-account-id","fb-graph-node","parent-id","total_count"],"incremental":true,"primary_key":["parent_id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/feedsummary/out/tables/feed_likes_summary/1516621525270: -------------------------------------------------------------------------------- 1 | "177057932317550","page_feed_reactions","177057932317550_1379478452075486","2" 2 | "177057932317550","page_feed_reactions","177057932317550_10155365834019810","22" 3 | "177057932317550","page_feed_reactions","177057932317550_177062472317096","0" 4 | "177057932317550","page_feed_reactions","177057932317550_1477131262310204","0" 5 | "177057932317550","page_feed_reactions","177057932317550_1478486035508060","5" 6 | "177057932317550","page_feed_reactions","177057932317550_200219200511028","0" 7 | "177057932317550","page_feed_reactions","177057932317550_1920132124904149","4" 8 | "177057932317550","page_feed_reactions","177057932317550_1528270233862973","0" 9 | "222838661196260","page_feed_reactions","222838661196260_1341134989232514","0" 10 | "222838661196260","page_feed_reactions","222838661196260_1005566356256816","11" 11 | "222838661196260","page_feed_reactions","222838661196260_1019388544874597","0" 12 | "222838661196260","page_feed_reactions","222838661196260_1020713021408816","4" 13 | "222838661196260","page_feed_reactions","222838661196260_1024914097655375","5" 14 | "222838661196260","page_feed_reactions","222838661196260_1047848388695279","0" 15 | "222838661196260","page_feed_reactions","222838661196260_1369870156416993","2" 16 | "222838661196260","page_feed_reactions","222838661196260_1071468542999930","2" 17 | "222838661196260","page_feed_reactions","222838661196260_1077902842356500","0" 18 | "222838661196260","page_feed_reactions","222838661196260_1082965645183553","0" 19 | "222838661196260","page_feed_reactions","222838661196260_1085738481572936","0" 20 | "222838661196260","page_feed_reactions","222838661196260_1088426111304173","3" 21 | "222838661196260","page_feed_reactions","222838661196260_1095366693943448","9" 22 | "222838661196260","page_feed_reactions","222838661196260_1101117760035008","2" 23 | "222838661196260","page_feed_reactions","222838661196260_1102622523217865","4" 24 | "222838661196260","page_feed_reactions","222838661196260_1102748413205276","0" 25 | "222838661196260","page_feed_reactions","222838661196260_1102759903204127","0" 26 | "222838661196260","page_feed_reactions","222838661196260_412927135725672","3" 27 | "222838661196260","page_feed_reactions","222838661196260_1260350687419830","0" 28 | "222838661196260","page_feed_reactions","222838661196260_1107481849398599","0" 29 | "222838661196260","page_feed_reactions","222838661196260_1107482036065247","0" 30 | "222838661196260","page_feed_reactions","222838661196260_1108005789346205","1" 31 | "222838661196260","page_feed_reactions","222838661196260_1116959565117494","0" 32 | "222838661196260","page_feed_reactions","222838661196260_1117831915030259","2" 33 | "222838661196260","page_feed_reactions","222838661196260_1166746623472121","0" 34 | "222838661196260","page_feed_reactions","222838661196260_1166746836805433","0" 35 | "222838661196260","page_feed_reactions","222838661196260_1169662839847166","0" 36 | "222838661196260","page_feed_reactions","222838661196260_1219971791482937","1" 37 | "222838661196260","page_feed_reactions","222838661196260_1256173724529410","5" 38 | "222838661196260","page_feed_reactions","222838661196260_1257750477705068","0" 39 | "222838661196260","page_feed_reactions","222838661196260_1261140494032733","6" 40 | "222838661196260","page_feed_reactions","222838661196260_1261976987282417","0" 41 | "222838661196260","page_feed_reactions","222838661196260_1268621089951340","6" 42 | "222838661196260","page_feed_reactions","222838661196260_144808662915978","0" 43 | "222838661196260","page_feed_reactions","222838661196260_1272431639570285","3" 44 | "222838661196260","page_feed_reactions","222838661196260_1272431919570257","0" 45 | "222838661196260","page_feed_reactions","222838661196260_1272432166236899","0" 46 | "222838661196260","page_feed_reactions","222838661196260_1275430289270420","3" 47 | "222838661196260","page_feed_reactions","222838661196260_1284921258321323","5" 48 | "222838661196260","page_feed_reactions","222838661196260_1295611950585587","0" 49 | "222838661196260","page_feed_reactions","222838661196260_1297263780420404","5" 50 | -------------------------------------------------------------------------------- /test/keboola/snapshots/feedsummary/test_feedsummary.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.feedsummary.test-feedsummary 2 | (:require [keboola.snapshots.feedsummary.apicalls :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest feedsummary-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "feedsummary"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "test/keboola/snapshots/feedsummary") 17 | (test-utils/copy-config-tmp "test/keboola/snapshots/feedsummary" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "test/keboola/snapshots/feedsummary" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/snapshots/outdirs_check.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.outdirs-check 2 | (:require [clojure.test :as t :refer :all] 3 | [cheshire.core :refer [parse-string]]) 4 | (:import java.io.File)) 5 | 6 | 7 | (defn list-dir [path] 8 | (let [dirs (.listFiles (File. path))] 9 | (zipmap (map #(.getName %) dirs) dirs))) 10 | 11 | (defn filter-dirs [dir] 12 | (filter #(.isDirectory (second %)) dir)) 13 | 14 | (defn is-same-ls [expected actual] 15 | (let [expected-names (keys expected) 16 | actual-names (keys actual)] 17 | (is (= (set expected-names) (set actual-names))))) 18 | 19 | (defn read-file [path] 20 | (with-open [r (clojure.java.io/reader path)] 21 | (doall (line-seq r)))) 22 | 23 | (defn parse-json [path] 24 | (let [file-content (slurp path)] 25 | (parse-string file-content true))) 26 | 27 | (defn compare-manifests [expected-path actual-path] 28 | (let [expected (parse-json expected-path) 29 | actual (parse-json actual-path) 30 | expected-pk (:primary_key expected) 31 | actual-pk (:primary_key actual) 32 | actual-pk-set (set actual-pk)] 33 | 34 | (is (:incremental actual)) 35 | (is (= (:columns expected) (:columns actual))) 36 | (is (= (count actual-pk) (count actual-pk-set))) 37 | (is (= expected-pk actual-pk)))) 38 | 39 | (defn compare-file-content [[f1 f2]] 40 | (let [f1path (.getAbsolutePath f1) 41 | f2path (.getAbsolutePath f2) 42 | f1-lines (set (read-file f1path)) 43 | f2-lines (set (read-file f2path)) 44 | diff (clojure.set/difference f1-lines f2-lines)] 45 | #_(println "comparing" f1path "vs." f2path) 46 | (is (empty? diff) (str "files are not same:" f1path " vs " f2path ". Difference:" diff)))) 47 | 48 | (defn compare-sliced-dir [expected actual] 49 | (let [a (sort-by #(.getName %) (.listFiles expected)) 50 | b (sort-by #(.getName %) (.listFiles actual)) 51 | pairs-to-compare (zipmap a b)] 52 | (is (= (count a) (count b))) 53 | (doseq [pair pairs-to-compare] 54 | (compare-file-content pair)))) 55 | 56 | (defn filter-manifests [dir] 57 | (into {} (filter #(re-matches #".*\.manifest$" (first %)) dir))) 58 | 59 | (defn is-equal [expected-dir actual-dir] 60 | (let 61 | [exptected-tables-path (str expected-dir "/out/tables") 62 | actual-tables-path (str actual-dir "/out/tables") 63 | ls-expected (list-dir exptected-tables-path) 64 | ls-actual (list-dir actual-tables-path)] 65 | (is-same-ls ls-expected ls-actual) 66 | (doseq [manifest-name (keys (filter-manifests ls-expected))] 67 | (compare-manifests (ls-expected manifest-name) (ls-actual manifest-name))) 68 | (doseq [dir-name (keys (filter-dirs ls-expected))] 69 | (compare-sliced-dir (ls-expected dir-name) (ls-actual dir-name))))) 70 | -------------------------------------------------------------------------------- /test/keboola/snapshots/pageinsights/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "177057932317550" : { 6 | "id" : "177057932317550", 7 | "name" : "keboola", 8 | "category" : "software" 9 | } 10 | }, 11 | "api-version" : "v2.8", 12 | "queries" : [ { 13 | "name" : "feed", 14 | "type" : "nested-query", 15 | "disabled" : true, 16 | "query" : { 17 | "path" : "feed", 18 | "fields" : "caption,message,created_time,type,description,likes{name,username},comments{message,created_time,from,likes{name,username},comments{message,created_time,from,likes{name,username}}}", 19 | "ids" : "177057932317550" 20 | } 21 | }, { 22 | "name" : "summarytest", 23 | "type" : "nested-query", 24 | "disabled" : false, 25 | "query" : { 26 | "path" : "", 27 | "fields" : "posts.limit(10){id,created_time,message,likes.summary(true).limit(0),reactions.summary(total_count).limit(0)}", 28 | "ids" : "177057932317550" 29 | } 30 | }, { 31 | "name" : "page", 32 | "type" : "nested-query", 33 | "disabled" : false, 34 | "query" : { 35 | "path" : "", 36 | "fields" : "insights.since(2 days ago).metric(page_views_by_age_gender_logged_in_unique,page_impressions_by_story_type, page_impressions_by_locale_unique, page_views_total, page_fans)", 37 | "ids" : null 38 | } 39 | }, { 40 | "name" : "", 41 | "type" : "nested-query", 42 | "disabled" : true, 43 | "query" : { 44 | "path" : "feed", 45 | "fields" : "insights.since(now).metric(post_negative_feedback, post_engaged_users, post_consumptions, post_impressions)", 46 | "ids" : "222838661196260" 47 | } 48 | } ] 49 | }, 50 | "authorization" : { 51 | "oauth_api" : { 52 | "id" : "keboola.ex-facebook", 53 | "credentials" : { 54 | "id" : "main", 55 | "authorizedFor" : "Myself", 56 | "creator" : { 57 | "id" : "1234", 58 | "description" : "me@keboola.com" 59 | }, 60 | "created" : "2016-01-31 00:13:30", 61 | "oauthVersion" : "facebook", 62 | "appKey" : "xxx", 63 | "#data" : "{\"token\":\"XXTOKENXX\"}", 64 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 65 | } 66 | } 67 | } 68 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/pageinsights/out/tables/accounts: -------------------------------------------------------------------------------- 1 | category,name,id 2 | software,keboola,177057932317550 3 | -------------------------------------------------------------------------------- /test/keboola/snapshots/pageinsights/out/tables/summarytest_posts.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["id","ex-account-id","fb-graph-node","parent-id","created_time","message"],"incremental":true,"primary_key":["parent_id","id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/pageinsights/out/tables/summarytest_summary.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["ex-account-id","fb-graph-node","parent-id","can_like","has_liked","total_count"],"incremental":true,"primary_key":["parent_id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/pageinsights/test_pageinsights.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.pageinsights.test-pageinsights 2 | (:require [clj-http.fake :refer :all] 3 | [clojure.test :as t :refer :all] 4 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 5 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.snapshots.outdirs-check :as outdirs-check] 8 | [keboola.snapshots.pageinsights.apicalls :as apicalls] 9 | [keboola.test-utils.core :as test-utils])) 10 | 11 | (deftest pageinsights-test 12 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "pageinsights"))] 13 | (disable-log-token) 14 | (println "testing dir:" tmp-dir) 15 | (println "expected dir:" "test/keboola/snapshots/pageinsights") 16 | (test-utils/copy-config-tmp "test/keboola/snapshots/pageinsights" tmp-dir) 17 | (with-global-fake-routes-in-isolation 18 | apicalls/recorded 19 | (reset-columns-map) 20 | (prepare-and-run tmp-dir) 21 | (outdirs-check/is-equal "test/keboola/snapshots/pageinsights" tmp-dir) 22 | ))) 23 | -------------------------------------------------------------------------------- /test/keboola/snapshots/postsinsights/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "177057932317550" : { 6 | "id" : "177057932317550", 7 | "name" : "keboola", 8 | "category" : "software" 9 | } 10 | }, 11 | "api-version" : "v2.8", 12 | "queries" : [ { 13 | "name" : "posts_insights", 14 | "type" : "nested-query", 15 | "disabled" : false, 16 | "query" : { 17 | "limit" : "40", 18 | "since" : "24 months ago", 19 | "until" : "now", 20 | "path" : "feed", 21 | "fields" : "insights.since(now).metric(post_video_view_time,post_engaged_fan,post_consumptions)", 22 | "ids" : null 23 | } 24 | } ] 25 | }, 26 | "authorization" : { 27 | "oauth_api" : { 28 | "id" : "keboola.ex-facebook", 29 | "credentials" : { 30 | "id" : "main", 31 | "authorizedFor" : "Myself", 32 | "creator" : { 33 | "id" : "1234", 34 | "description" : "me@keboola.com" 35 | }, 36 | "created" : "2016-01-31 00:13:30", 37 | "oauthVersion" : "facebook", 38 | "appKey" : "xxx", 39 | "#data" : "{\"token\":\"XXTOKENXX\"}", 40 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 41 | } 42 | } 43 | } 44 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/postsinsights/out/tables/accounts: -------------------------------------------------------------------------------- 1 | category,name,id 2 | software,keboola,177057932317550 3 | -------------------------------------------------------------------------------- /test/keboola/snapshots/postsinsights/out/tables/posts_insights.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["id","ex-account-id","fb-graph-node","parent-id","name","key1","key2","value","period","end_time","title","description"],"incremental":true,"primary_key":["parent_id","id","key1","key2","end_time"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/postsinsights/test_postsinsights.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.postsinsights.test-postsinsights 2 | (:require [clj-http.fake :refer :all] 3 | [clojure.test :as t :refer :all] 4 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 5 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.snapshots.outdirs-check :as outdirs-check] 8 | [keboola.snapshots.postsinsights.apicalls :as apicalls] 9 | [keboola.test-utils.core :as test-utils])) 10 | 11 | (deftest postsinsights-test 12 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "postsinsights"))] 13 | (disable-log-token) 14 | (println "testing dir:" tmp-dir) 15 | (println "expected dir:" "test/keboola/snapshots/postsinsights") 16 | (test-utils/copy-config-tmp "test/keboola/snapshots/postsinsights" tmp-dir) 17 | (with-global-fake-routes-in-isolation 18 | apicalls/recorded 19 | (reset-columns-map) 20 | (prepare-and-run tmp-dir) 21 | (outdirs-check/is-equal "test/keboola/snapshots/postsinsights" tmp-dir) 22 | ))) 23 | -------------------------------------------------------------------------------- /test/keboola/snapshots/runbyid/apicalls.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.runbyid.apicalls) 2 | (def recorded 3 | { 4 | {:method :get, 5 | :address "https://graph.facebook.com/v4.0/", 6 | :as :json, 7 | :query-params 8 | {:path "", 9 | :fields 10 | "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}", 11 | :ids "act_1146726535372240", 12 | :access_token "XXTOKENXX", 13 | :since "", 14 | :until ""}} 15 | (fn [req]{:status 200, 16 | :body 17 | "{\"act_1146726535372240\":{\"insights\":{\"data\":[{\"account_id\":\"1146726535372240\",\"account_name\":\"Rn4f8\",\"ad_name\":\"p84kw\",\"ad_id\":\"6164363071480\",\"impressions\":\"hLbV2\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"bepJ8\",\"ad_name\":\"WKjha\",\"ad_id\":\"6164363071680\",\"impressions\":\"pHulH\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"xTlvB\",\"ad_name\":\"5xRHT\",\"ad_id\":\"6164363072080\",\"impressions\":\"loLpz\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"oUxmM\",\"ad_name\":\"3VhKN\",\"ad_id\":\"6164363072280\",\"impressions\":\"cMF7L\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"qEb8h\",\"ad_name\":\"UyTj0\",\"ad_id\":\"6164363072880\",\"impressions\":\"Mh4jx\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"RiDsp\",\"ad_name\":\"k3L8Z\",\"ad_id\":\"6164363073280\",\"impressions\":\"i6URx\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"3gaaK\",\"ad_name\":\"7SbGa\",\"ad_id\":\"6164363071480\",\"impressions\":\"lQ5Cp\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"eJNaw\",\"ad_name\":\"5M1Bg\",\"ad_id\":\"6164363071680\",\"impressions\":\"Mx0hJ\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"KLzUd\",\"ad_name\":\"RLycB\",\"ad_id\":\"6164363072080\",\"impressions\":\"03zWv\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"bVMN4\",\"ad_name\":\"4hyrr\",\"ad_id\":\"6164363072280\",\"impressions\":\"mYTK4\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"Dyzf7\",\"ad_name\":\"I8lef\",\"ad_id\":\"6164363072880\",\"impressions\":\"RBPma\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"1146726535372240\",\"account_name\":\"viIso\",\"ad_name\":\"rqbRY\",\"ad_id\":\"6164363073280\",\"impressions\":\"RhNJN\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"}],\"paging\":{\"cursors\":{\"before\":\"MAZDZD\",\"after\":\"MTEZD\"}}},\"id\":\"act_1146726535372240\"}}"} 18 | ){:method :get, 19 | :address "https://graph.facebook.com/v4.0/", 20 | :as :json, 21 | :query-params 22 | {:path "", 23 | :fields 24 | "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}", 25 | :ids "act_108176966036258", 26 | :access_token "XXTOKENXX", 27 | :since "", 28 | :until ""}} 29 | (fn [req]{:status 200, 30 | :body 31 | "{\"act_108176966036258\":{\"insights\":{\"data\":[{\"account_id\":\"108176966036258\",\"account_name\":\"9OJDi\",\"ad_name\":\"cJ2fD\",\"ad_id\":\"6134702917855\",\"impressions\":\"801nU\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"QTiTQ\",\"ad_name\":\"Oq3jX\",\"ad_id\":\"6134760720255\",\"impressions\":\"Yp7Wh\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"ejkr5\",\"ad_name\":\"9Oh5V\",\"ad_id\":\"6134760720655\",\"impressions\":\"5UmnU\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"GCygF\",\"ad_name\":\"j7oPW\",\"ad_id\":\"6134760721055\",\"impressions\":\"k1Ila\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"xvb11\",\"ad_name\":\"ItqGO\",\"ad_id\":\"6134760721255\",\"impressions\":\"Ftcom\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"8e32I\",\"ad_name\":\"yHrYg\",\"ad_id\":\"6134760721455\",\"impressions\":\"oQSrN\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"KDsFR\",\"ad_name\":\"bJZaK\",\"ad_id\":\"6134760721655\",\"impressions\":\"PiS1l\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"UGHhF\",\"ad_name\":\"MCkrb\",\"ad_id\":\"6134760721855\",\"impressions\":\"ST4rD\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"1itYW\",\"ad_name\":\"V8pJn\",\"ad_id\":\"6134760722255\",\"impressions\":\"LFpYk\",\"date_start\":\"2019-09-29\",\"date_stop\":\"2019-09-29\"},{\"account_id\":\"108176966036258\",\"account_name\":\"8uHUi\",\"ad_name\":\"6Cgyi\",\"ad_id\":\"6134702917855\",\"impressions\":\"a5czK\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"1MXgy\",\"ad_name\":\"p2qRE\",\"ad_id\":\"6134760721255\",\"impressions\":\"g2Ude\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"OQ4HK\",\"ad_name\":\"G6SZI\",\"ad_id\":\"6134760721455\",\"impressions\":\"gZYka\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"dwBfc\",\"ad_name\":\"zYoRl\",\"ad_id\":\"6134760721655\",\"impressions\":\"RjeTM\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"},{\"account_id\":\"108176966036258\",\"account_name\":\"rjTMp\",\"ad_name\":\"CIP6S\",\"ad_id\":\"6134760722255\",\"impressions\":\"U7dqW\",\"date_start\":\"2019-09-30\",\"date_stop\":\"2019-09-30\"}],\"paging\":{\"cursors\":{\"before\":\"MAZDZD\",\"after\":\"MTMZD\"}}},\"id\":\"act_108176966036258\"}}"} 32 | ) 33 | }) -------------------------------------------------------------------------------- /test/keboola/snapshots/runbyid/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "act_108176966036258" : { 6 | "account_id" : "108176966036258", 7 | "business_name" : "account 2", 8 | "currency" : "GBP", 9 | "id" : "act_108176966036258", 10 | "name" : "account 2" 11 | }, 12 | "act_1146726535372240" : { 13 | "account_id" : "1146726535372240", 14 | "business_name" : "", 15 | "currency" : "GBP", 16 | "id" : "act_1146726535372240", 17 | "name" : "account 3" 18 | } 19 | }, 20 | "api-version" : "v4.0", 21 | "queries" : [ { 22 | "name" : "ads", 23 | "type" : "nested-query", 24 | "run-by-id" : true, 25 | "disabled" : false, 26 | "query" : { 27 | "path" : "", 28 | "fields" : "insights.time_range({\"since\":\"2019-09-29\",\"until\":\"2019-09-30\"}).level(ad).time_increment(1){account_id,account_name,ad_name,ad_id,impressions}", 29 | "ids" : "" 30 | } 31 | } ] 32 | }, 33 | "authorization" : { 34 | "oauth_api" : { 35 | "id" : "{OAUTH_API_ID}", 36 | "credentials" : { 37 | "id" : "main", 38 | "authorizedFor" : "Myself", 39 | "creator" : { 40 | "id" : "1234", 41 | "description" : "me@keboola.com" 42 | }, 43 | "created" : "2016-01-31 00:13:30", 44 | "oauthVersion" : "facebook", 45 | "appKey" : "", 46 | "#data" : "{\"token\":\"XXTOKENXX\"}", 47 | "#appSecret" : "" 48 | } 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/runbyid/out/tables/accounts: -------------------------------------------------------------------------------- 1 | account_id,name,business_name,currency,id 2 | 108176966036258,account 2,account 2,GBP,act_108176966036258 3 | 1146726535372240,account 3,,GBP,act_1146726535372240 4 | -------------------------------------------------------------------------------- /test/keboola/snapshots/runbyid/out/tables/ads_insights.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["ex-account-id","fb-graph-node","parent-id","account_id","account_name","ad_id","ad_name","date_start","date_stop","impressions"],"incremental":true,"primary_key":["parent_id","account_id","date_start","date_stop","ad_id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/runbyid/out/tables/ads_insights/1570782087866: -------------------------------------------------------------------------------- 1 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","viIso","6164363073280","rqbRY","2019-09-30","2019-09-30","RhNJN" 2 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","Dyzf7","6164363072880","I8lef","2019-09-30","2019-09-30","RBPma" 3 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","bVMN4","6164363072280","4hyrr","2019-09-30","2019-09-30","mYTK4" 4 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","KLzUd","6164363072080","RLycB","2019-09-30","2019-09-30","03zWv" 5 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","eJNaw","6164363071680","5M1Bg","2019-09-30","2019-09-30","Mx0hJ" 6 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","3gaaK","6164363071480","7SbGa","2019-09-30","2019-09-30","lQ5Cp" 7 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","RiDsp","6164363073280","k3L8Z","2019-09-29","2019-09-29","i6URx" 8 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","qEb8h","6164363072880","UyTj0","2019-09-29","2019-09-29","Mh4jx" 9 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","oUxmM","6164363072280","3VhKN","2019-09-29","2019-09-29","cMF7L" 10 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","xTlvB","6164363072080","5xRHT","2019-09-29","2019-09-29","loLpz" 11 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","bepJ8","6164363071680","WKjha","2019-09-29","2019-09-29","pHulH" 12 | "act_1146726535372240","page_insights","act_1146726535372240","1146726535372240","Rn4f8","6164363071480","p84kw","2019-09-29","2019-09-29","hLbV2" 13 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","rjTMp","6134760722255","CIP6S","2019-09-30","2019-09-30","U7dqW" 14 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","dwBfc","6134760721655","zYoRl","2019-09-30","2019-09-30","RjeTM" 15 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","OQ4HK","6134760721455","G6SZI","2019-09-30","2019-09-30","gZYka" 16 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","1MXgy","6134760721255","p2qRE","2019-09-30","2019-09-30","g2Ude" 17 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","8uHUi","6134702917855","6Cgyi","2019-09-30","2019-09-30","a5czK" 18 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","1itYW","6134760722255","V8pJn","2019-09-29","2019-09-29","LFpYk" 19 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","UGHhF","6134760721855","MCkrb","2019-09-29","2019-09-29","ST4rD" 20 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","KDsFR","6134760721655","bJZaK","2019-09-29","2019-09-29","PiS1l" 21 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","8e32I","6134760721455","yHrYg","2019-09-29","2019-09-29","oQSrN" 22 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","xvb11","6134760721255","ItqGO","2019-09-29","2019-09-29","Ftcom" 23 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","GCygF","6134760721055","j7oPW","2019-09-29","2019-09-29","k1Ila" 24 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","ejkr5","6134760720655","9Oh5V","2019-09-29","2019-09-29","5UmnU" 25 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","QTiTQ","6134760720255","Oq3jX","2019-09-29","2019-09-29","Yp7Wh" 26 | "act_108176966036258","page_insights","act_108176966036258","108176966036258","9OJDi","6134702917855","cJ2fD","2019-09-29","2019-09-29","801nU" 27 | -------------------------------------------------------------------------------- /test/keboola/snapshots/runbyid/test_runbyid.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.runbyid.test-runbyid 2 | (:require [keboola.snapshots.runbyid.apicalls :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest runbyid-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "runbyid"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "test/keboola/snapshots/runbyid") 17 | (test-utils/copy-config-tmp "test/keboola/snapshots/runbyid" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "test/keboola/snapshots/runbyid" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/snapshots/serializelists/apicalls.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.serializelists.apicalls) 2 | (def recorded 3 | { 4 | {:method :get, 5 | :address 6 | "https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&after=QVFIUkpnN2hjN2tvdjhPMGVhYVFzWU83a0c2U0d0bHFiSEFhTERHTnAxWEZA6MzBxbWZASN3JCTnVyVnN5ZAHRJeS00RHVGcnh1a1ZAabGdIeVEya0VwbDhmUGFR", 7 | :as :json} 8 | (fn [req]{:status 200, 9 | :body 10 | "{\"data\":[{\"id\":\"23844153993410682\"},{\"id\":\"23844148994770682\",\"frequency_control_specs\":[{\"event\":\"IMPRESSIONS\",\"interval_days\":1,\"max_frequency\":6}]},{\"id\":\"23844111320800682\"},{\"id\":\"23844103736930682\"},{\"id\":\"23844103698260682\"},{\"id\":\"23844102808490682\"},{\"id\":\"23844078885000682\"},{\"id\":\"23844028037580682\"},{\"id\":\"23844027913490682\"},{\"id\":\"23843980952820682\"},{\"id\":\"23843980782690682\"},{\"id\":\"23843888426700682\"},{\"id\":\"23843885056100682\"},{\"id\":\"23843840706700682\"},{\"id\":\"23843838944520682\"},{\"id\":\"23843663131530682\"},{\"id\":\"23843343442760682\"},{\"id\":\"23843240933280682\"},{\"id\":\"23843234244100682\"},{\"id\":\"23843222973130682\"},{\"id\":\"23843222967730682\"},{\"id\":\"23843159603400682\"},{\"id\":\"23843159283730682\"},{\"id\":\"23843121097860682\"},{\"id\":\"23843105682230682\"}],\"paging\":{\"cursors\":{\"before\":\"QVFIUl9sNGFhWG52Mi1xX1RxeHVjb05qOElfQl9wZA2dsSXNnZAVl6Q2FSeFlMaEFhYlZAJUE53R0lHM1V0M2cyTjFXS0E5Q1pGclBnNkFnOUJuSl9KNkpTYW53\",\"after\":\"QVFIUmt0VUhCTzlNT0N0XzZAzWmRGNTNMc3RlelVzQ2dUTnJlb0lZASmk1LXdWZAHd0QXBGY2V3emdGUkJLZAmpidG5Hbk1aTXJFNW5rWkhuWFVFeVdGSnpsZAXZA3\"},\"previous\":\"https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&before=QVFIUl9sNGFhWG52Mi1xX1RxeHVjb05qOElfQl9wZA2dsSXNnZAVl6Q2FSeFlMaEFhYlZAJUE53R0lHM1V0M2cyTjFXS0E5Q1pGclBnNkFnOUJuSl9KNkpTYW53\"}}"} 11 | ){:method :get, 12 | :address 13 | "https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&after=QVFIUlV5cDZAQdE1PaklVb1FqeVMyWGpjSXBWeFNpdEZAhZAlFxLWJpRWo1bmNKODFKX1BZAUk1neVlYNXgtRnRYa016N3NqcE9KczQ1NXBLcFRVeEZA2UTJjcDJB", 14 | :as :json} 15 | (fn [req]{:status 200, 16 | :body 17 | "{\"data\":[{\"id\":\"23845588576660682\"},{\"id\":\"23845510551760682\"},{\"id\":\"23845497626260682\"},{\"id\":\"23845458881630682\"},{\"id\":\"23845405656220682\"},{\"id\":\"23845271377410682\",\"frequency_control_specs\":[{\"event\":\"IMPRESSIONS\",\"interval_days\":30,\"max_frequency\":6}]},{\"id\":\"23845270930970682\"},{\"id\":\"23845201552990682\"},{\"id\":\"23845136125380682\"},{\"id\":\"23845063067650682\"},{\"id\":\"23845045896650682\"},{\"id\":\"23844890477370682\"},{\"id\":\"23844763735970682\"},{\"id\":\"23844751217210682\"},{\"id\":\"23844613385860682\"},{\"id\":\"23844596483280682\"},{\"id\":\"23844587024020682\"},{\"id\":\"23844582832290682\"},{\"id\":\"23844490747860682\"},{\"id\":\"23844184544430682\"},{\"id\":\"23844180554710682\"},{\"id\":\"23844178585150682\"},{\"id\":\"23844178324920682\"},{\"id\":\"23844167303090682\"},{\"id\":\"23844154012460682\"}],\"paging\":{\"cursors\":{\"before\":\"QVFIUlpHV0xYdzZAkZADYxaTFLSGRiUzN1Y2pyTklwdFo2TW93Ty14YkFtdTFWNGhIWE5WVHNhOE9KeFJQenNLdWh0ZA21yeEk4emxCZAk5CU2U0TjhPMzJHeGZAB\",\"after\":\"QVFIUkpnN2hjN2tvdjhPMGVhYVFzWU83a0c2U0d0bHFiSEFhTERHTnAxWEZA6MzBxbWZASN3JCTnVyVnN5ZAHRJeS00RHVGcnh1a1ZAabGdIeVEya0VwbDhmUGFR\"},\"next\":\"https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&after=QVFIUkpnN2hjN2tvdjhPMGVhYVFzWU83a0c2U0d0bHFiSEFhTERHTnAxWEZA6MzBxbWZASN3JCTnVyVnN5ZAHRJeS00RHVGcnh1a1ZAabGdIeVEya0VwbDhmUGFR\",\"previous\":\"https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&before=QVFIUlpHV0xYdzZAkZADYxaTFLSGRiUzN1Y2pyTklwdFo2TW93Ty14YkFtdTFWNGhIWE5WVHNhOE9KeFJQenNLdWh0ZA21yeEk4emxCZAk5CU2U0TjhPMzJHeGZAB\"}}"} 18 | ){:method :get, 19 | :address 20 | "https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&after=QVFIUlZApWmp5OTlIamFiUld2UWx5X2dQYmNVQ1pWNUlmbEJjVEp3c0Vqel9saTg5SHF6a09UUU1yTlBDZAUo5b0ZADek12V2RvTHZAMS0VBQ3hUY29YTkVaeXN3", 21 | :as :json} 22 | (fn [req]{:status 200, 23 | :body 24 | "{\"data\":[{\"id\":\"23847204132950682\"},{\"id\":\"23847148043930682\"},{\"id\":\"23847003783650682\"},{\"id\":\"23846895937800682\"},{\"id\":\"23846429372250682\"},{\"id\":\"23846373085740682\"},{\"id\":\"23846365734200682\"},{\"id\":\"23846337921540682\"},{\"id\":\"23846337921510682\"},{\"id\":\"23846337915070682\"},{\"id\":\"23846337857450682\"},{\"id\":\"23846329888800682\"},{\"id\":\"23846329827020682\"},{\"id\":\"23846328800180682\"},{\"id\":\"23846321778770682\"},{\"id\":\"23846296646780682\"},{\"id\":\"23846234243070682\"},{\"id\":\"23846155939880682\"},{\"id\":\"23846098443230682\"},{\"id\":\"23846052816590682\"},{\"id\":\"23846033839020682\"},{\"id\":\"23845984814360682\"},{\"id\":\"23845724849860682\"},{\"id\":\"23845593460530682\"},{\"id\":\"23845588673010682\"}],\"paging\":{\"cursors\":{\"before\":\"QVFIUmV0N3ltYlYtX3lpQnJNT1oxUGsxTmU5NG5VbjVaQmJzclJXWVRxX2lPdS1DZAzJjaExkemQwSWVzanZAjaVc2R2RWb1M4VFo4NTFEeE9tTG5oQVlLWXZAR\",\"after\":\"QVFIUlV5cDZAQdE1PaklVb1FqeVMyWGpjSXBWeFNpdEZAhZAlFxLWJpRWo1bmNKODFKX1BZAUk1neVlYNXgtRnRYa016N3NqcE9KczQ1NXBLcFRVeEZA2UTJjcDJB\"},\"next\":\"https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&after=QVFIUlV5cDZAQdE1PaklVb1FqeVMyWGpjSXBWeFNpdEZAhZAlFxLWJpRWo1bmNKODFKX1BZAUk1neVlYNXgtRnRYa016N3NqcE9KczQ1NXBLcFRVeEZA2UTJjcDJB\",\"previous\":\"https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&before=QVFIUmV0N3ltYlYtX3lpQnJNT1oxUGsxTmU5NG5VbjVaQmJzclJXWVRxX2lPdS1DZAzJjaExkemQwSWVzanZAjaVc2R2RWb1M4VFo4NTFEeE9tTG5oQVlLWXZAR\"}}"} 25 | ){:method :get, 26 | :address "https://graph.facebook.com/v11.0/adsets", 27 | :as :json, 28 | :query-params 29 | {:path "adsets", 30 | :fields "id,frequency_control_specs", 31 | :ids "act_522606278080331", 32 | :access_token "XXTOKENXX", 33 | :since "", 34 | :until ""}} 35 | (fn [req]{:status 200, 36 | :body 37 | "{\"act_522606278080331\":{\"data\":[{\"id\":\"23849106408410682\"},{\"id\":\"23849103477910682\",\"frequency_control_specs\":[{\"event\":\"IMPRESSIONS\",\"interval_days\":7,\"max_frequency\":1}]},{\"id\":\"23849067903270682\"},{\"id\":\"23849060995620682\"},{\"id\":\"23848962452680682\"},{\"id\":\"23848953603860682\"},{\"id\":\"23848953041860682\"},{\"id\":\"23848952915280682\"},{\"id\":\"23848927728330682\"},{\"id\":\"23848794070540682\"},{\"id\":\"23848686761460682\"},{\"id\":\"23848685259860682\"},{\"id\":\"23848685195570682\"},{\"id\":\"23848141469160682\"},{\"id\":\"23848105082980682\"},{\"id\":\"23848081830050682\"},{\"id\":\"23848056918680682\"},{\"id\":\"23847987759170682\"},{\"id\":\"23847936601890682\"},{\"id\":\"23847873997490682\"},{\"id\":\"23847669609830682\"},{\"id\":\"23847650185210682\"},{\"id\":\"23847602203250682\"},{\"id\":\"23847209514990682\"},{\"id\":\"23847209454990682\"}],\"paging\":{\"cursors\":{\"before\":\"QVFIUmNySG1OQkR0RUFhaWNoX1VIMlVxaVZAQRzNwNXRGaExSOVk0ajJQbXRVWkZA6U3ZAYVmFhRW9IR2MyUHI1eFhUZAGN1N1gwdWd3VFY2VHhKbVVlaTVOSTNR\",\"after\":\"QVFIUlZApWmp5OTlIamFiUld2UWx5X2dQYmNVQ1pWNUlmbEJjVEp3c0Vqel9saTg5SHF6a09UUU1yTlBDZAUo5b0ZADek12V2RvTHZAMS0VBQ3hUY29YTkVaeXN3\"},\"next\":\"https://graph.facebook.com/v11.0/act_522606278080331/adsets?access_token=XXTOKENXX&fields=id%2Cfrequency_control_specs&since&until&limit=25&after=QVFIUlZApWmp5OTlIamFiUld2UWx5X2dQYmNVQ1pWNUlmbEJjVEp3c0Vqel9saTg5SHF6a09UUU1yTlBDZAUo5b0ZADek12V2RvTHZAMS0VBQ3hUY29YTkVaeXN3\"}}}"} 38 | ) 39 | }) -------------------------------------------------------------------------------- /test/keboola/snapshots/serializelists/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "storage" : { }, 3 | "parameters" : { 4 | "accounts" : { 5 | "act_522606278080331" : { 6 | "account_id" : "522606278080331", 7 | "business_name" : "aaa.cz, s.r.o.", 8 | "currency" : "CZK", 9 | "id" : "act_522606278080331", 10 | "name" : "aaa" 11 | } 12 | }, 13 | "api-version" : "v11.0", 14 | "queries" : [ { 15 | "name" : "adsets", 16 | "type" : "nested-query", 17 | "disabled" : false, 18 | "query" : { 19 | "path" : "adsets", 20 | "fields" : "id,frequency_control_specs", 21 | "ids" : "" 22 | } 23 | } ] 24 | }, 25 | "authorization" : { 26 | "oauth_api" : { 27 | "id" : "keboola.ex-facebook-ads", 28 | "credentials" : { 29 | "id" : "main", 30 | "authorizedFor" : "Myself", 31 | "creator" : { 32 | "id" : "1234", 33 | "description" : "me@keboola.com" 34 | }, 35 | "created" : "2016-01-31 00:13:30", 36 | "oauthVersion" : "facebook", 37 | "appKey" : "xxx", 38 | "#data" : "{\"token\":\"XXTOKENXX\"}", 39 | "#appSecret" : "KBC::Encrypted==ENCODEDSTRING==" 40 | } 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /test/keboola/snapshots/serializelists/out/tables/accounts: -------------------------------------------------------------------------------- 1 | account_id,name,business_name,currency,id 2 | 522606278080331,aaa,"aaa.cz, s.r.o.",CZK,act_522606278080331 3 | -------------------------------------------------------------------------------- /test/keboola/snapshots/serializelists/out/tables/adsets.manifest: -------------------------------------------------------------------------------- 1 | {"columns":["id","ex-account-id","fb-graph-node","parent-id","frequency_control_specs"],"incremental":true,"primary_key":["parent_id","id"]} -------------------------------------------------------------------------------- /test/keboola/snapshots/serializelists/out/tables/adsets/1638290127158: -------------------------------------------------------------------------------- 1 | "23843105682230682","act_522606278080331","page_adsets","act_522606278080331","" 2 | "23843121097860682","act_522606278080331","page_adsets","act_522606278080331","" 3 | "23843159283730682","act_522606278080331","page_adsets","act_522606278080331","" 4 | "23843159603400682","act_522606278080331","page_adsets","act_522606278080331","" 5 | "23843222967730682","act_522606278080331","page_adsets","act_522606278080331","" 6 | "23843222973130682","act_522606278080331","page_adsets","act_522606278080331","" 7 | "23843234244100682","act_522606278080331","page_adsets","act_522606278080331","" 8 | "23843240933280682","act_522606278080331","page_adsets","act_522606278080331","" 9 | "23843343442760682","act_522606278080331","page_adsets","act_522606278080331","" 10 | "23843663131530682","act_522606278080331","page_adsets","act_522606278080331","" 11 | "23843838944520682","act_522606278080331","page_adsets","act_522606278080331","" 12 | "23843840706700682","act_522606278080331","page_adsets","act_522606278080331","" 13 | "23843885056100682","act_522606278080331","page_adsets","act_522606278080331","" 14 | "23843888426700682","act_522606278080331","page_adsets","act_522606278080331","" 15 | "23843980782690682","act_522606278080331","page_adsets","act_522606278080331","" 16 | "23843980952820682","act_522606278080331","page_adsets","act_522606278080331","" 17 | "23844027913490682","act_522606278080331","page_adsets","act_522606278080331","" 18 | "23844028037580682","act_522606278080331","page_adsets","act_522606278080331","" 19 | "23844078885000682","act_522606278080331","page_adsets","act_522606278080331","" 20 | "23844102808490682","act_522606278080331","page_adsets","act_522606278080331","" 21 | "23844103698260682","act_522606278080331","page_adsets","act_522606278080331","" 22 | "23844103736930682","act_522606278080331","page_adsets","act_522606278080331","" 23 | "23844111320800682","act_522606278080331","page_adsets","act_522606278080331","" 24 | "23844148994770682","act_522606278080331","page_adsets","act_522606278080331","[{""event"":""IMPRESSIONS"",""interval_days"":1,""max_frequency"":6}]" 25 | "23844153993410682","act_522606278080331","page_adsets","act_522606278080331","" 26 | "23844154012460682","act_522606278080331","page_adsets","act_522606278080331","" 27 | "23844167303090682","act_522606278080331","page_adsets","act_522606278080331","" 28 | "23844178324920682","act_522606278080331","page_adsets","act_522606278080331","" 29 | "23844178585150682","act_522606278080331","page_adsets","act_522606278080331","" 30 | "23844180554710682","act_522606278080331","page_adsets","act_522606278080331","" 31 | "23844184544430682","act_522606278080331","page_adsets","act_522606278080331","" 32 | "23844490747860682","act_522606278080331","page_adsets","act_522606278080331","" 33 | "23844582832290682","act_522606278080331","page_adsets","act_522606278080331","" 34 | "23844587024020682","act_522606278080331","page_adsets","act_522606278080331","" 35 | "23844596483280682","act_522606278080331","page_adsets","act_522606278080331","" 36 | "23844613385860682","act_522606278080331","page_adsets","act_522606278080331","" 37 | "23844751217210682","act_522606278080331","page_adsets","act_522606278080331","" 38 | "23844763735970682","act_522606278080331","page_adsets","act_522606278080331","" 39 | "23844890477370682","act_522606278080331","page_adsets","act_522606278080331","" 40 | "23845045896650682","act_522606278080331","page_adsets","act_522606278080331","" 41 | "23845063067650682","act_522606278080331","page_adsets","act_522606278080331","" 42 | "23845136125380682","act_522606278080331","page_adsets","act_522606278080331","" 43 | "23845201552990682","act_522606278080331","page_adsets","act_522606278080331","" 44 | "23845270930970682","act_522606278080331","page_adsets","act_522606278080331","" 45 | "23845271377410682","act_522606278080331","page_adsets","act_522606278080331","[{""event"":""IMPRESSIONS"",""interval_days"":30,""max_frequency"":6}]" 46 | "23845405656220682","act_522606278080331","page_adsets","act_522606278080331","" 47 | "23845458881630682","act_522606278080331","page_adsets","act_522606278080331","" 48 | "23845497626260682","act_522606278080331","page_adsets","act_522606278080331","" 49 | "23845510551760682","act_522606278080331","page_adsets","act_522606278080331","" 50 | "23845588576660682","act_522606278080331","page_adsets","act_522606278080331","" 51 | "23845588673010682","act_522606278080331","page_adsets","act_522606278080331","" 52 | "23845593460530682","act_522606278080331","page_adsets","act_522606278080331","" 53 | "23845724849860682","act_522606278080331","page_adsets","act_522606278080331","" 54 | "23845984814360682","act_522606278080331","page_adsets","act_522606278080331","" 55 | "23846033839020682","act_522606278080331","page_adsets","act_522606278080331","" 56 | "23846052816590682","act_522606278080331","page_adsets","act_522606278080331","" 57 | "23846098443230682","act_522606278080331","page_adsets","act_522606278080331","" 58 | "23846155939880682","act_522606278080331","page_adsets","act_522606278080331","" 59 | "23846234243070682","act_522606278080331","page_adsets","act_522606278080331","" 60 | "23846296646780682","act_522606278080331","page_adsets","act_522606278080331","" 61 | "23846321778770682","act_522606278080331","page_adsets","act_522606278080331","" 62 | "23846328800180682","act_522606278080331","page_adsets","act_522606278080331","" 63 | "23846329827020682","act_522606278080331","page_adsets","act_522606278080331","" 64 | "23846329888800682","act_522606278080331","page_adsets","act_522606278080331","" 65 | "23846337857450682","act_522606278080331","page_adsets","act_522606278080331","" 66 | "23846337915070682","act_522606278080331","page_adsets","act_522606278080331","" 67 | "23846337921510682","act_522606278080331","page_adsets","act_522606278080331","" 68 | "23846337921540682","act_522606278080331","page_adsets","act_522606278080331","" 69 | "23846365734200682","act_522606278080331","page_adsets","act_522606278080331","" 70 | "23846373085740682","act_522606278080331","page_adsets","act_522606278080331","" 71 | "23846429372250682","act_522606278080331","page_adsets","act_522606278080331","" 72 | "23846895937800682","act_522606278080331","page_adsets","act_522606278080331","" 73 | "23847003783650682","act_522606278080331","page_adsets","act_522606278080331","" 74 | "23847148043930682","act_522606278080331","page_adsets","act_522606278080331","" 75 | "23847204132950682","act_522606278080331","page_adsets","act_522606278080331","" 76 | "23847209454990682","act_522606278080331","page_adsets","act_522606278080331","" 77 | "23847209514990682","act_522606278080331","page_adsets","act_522606278080331","" 78 | "23847602203250682","act_522606278080331","page_adsets","act_522606278080331","" 79 | "23847650185210682","act_522606278080331","page_adsets","act_522606278080331","" 80 | "23847669609830682","act_522606278080331","page_adsets","act_522606278080331","" 81 | "23847873997490682","act_522606278080331","page_adsets","act_522606278080331","" 82 | "23847936601890682","act_522606278080331","page_adsets","act_522606278080331","" 83 | "23847987759170682","act_522606278080331","page_adsets","act_522606278080331","" 84 | "23848056918680682","act_522606278080331","page_adsets","act_522606278080331","" 85 | "23848081830050682","act_522606278080331","page_adsets","act_522606278080331","" 86 | "23848105082980682","act_522606278080331","page_adsets","act_522606278080331","" 87 | "23848141469160682","act_522606278080331","page_adsets","act_522606278080331","" 88 | "23848685195570682","act_522606278080331","page_adsets","act_522606278080331","" 89 | "23848685259860682","act_522606278080331","page_adsets","act_522606278080331","" 90 | "23848686761460682","act_522606278080331","page_adsets","act_522606278080331","" 91 | "23848794070540682","act_522606278080331","page_adsets","act_522606278080331","" 92 | "23848927728330682","act_522606278080331","page_adsets","act_522606278080331","" 93 | "23848952915280682","act_522606278080331","page_adsets","act_522606278080331","" 94 | "23848953041860682","act_522606278080331","page_adsets","act_522606278080331","" 95 | "23848953603860682","act_522606278080331","page_adsets","act_522606278080331","" 96 | "23848962452680682","act_522606278080331","page_adsets","act_522606278080331","" 97 | "23849060995620682","act_522606278080331","page_adsets","act_522606278080331","" 98 | "23849067903270682","act_522606278080331","page_adsets","act_522606278080331","" 99 | "23849103477910682","act_522606278080331","page_adsets","act_522606278080331","[{""event"":""IMPRESSIONS"",""interval_days"":7,""max_frequency"":1}]" 100 | "23849106408410682","act_522606278080331","page_adsets","act_522606278080331","" 101 | -------------------------------------------------------------------------------- /test/keboola/snapshots/serializelists/test_serializelists.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.snapshots.serializelists.test-serializelists 2 | (:require [keboola.snapshots.serializelists.apicalls :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest serializelists-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "serializelists"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "test/keboola/snapshots/serializelists") 17 | (test-utils/copy-config-tmp "test/keboola/snapshots/serializelists" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "test/keboola/snapshots/serializelists" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/snapshots/template.mustache: -------------------------------------------------------------------------------- 1 | (ns {{ns-name}} 2 | (:require [{{apicalls-ns}} :as apicalls] 3 | [clojure.test :as t :refer :all] 4 | [keboola.snapshots.outdirs-check :as outdirs-check] 5 | [keboola.test-utils.core :as test-utils] 6 | [keboola.facebook.extractor.sync-actions :refer [disable-log-token]] 7 | [keboola.facebook.extractor.output :refer [reset-columns-map]] 8 | [keboola.facebook.extractor.core :refer [prepare-and-run]] 9 | ) 10 | (:use clj-http.fake)) 11 | 12 | (deftest {{test-name}}-test 13 | (let [tmp-dir (.getPath (test-utils/mk-tmp-dir! "{{test-name}}"))] 14 | (disable-log-token) 15 | (println "testing dir:" tmp-dir) 16 | (println "expected dir:" "{{dir-path}}") 17 | (test-utils/copy-config-tmp "{{dir-path}}" tmp-dir) 18 | (with-global-fake-routes-in-isolation 19 | apicalls/recorded 20 | (reset-columns-map) 21 | (prepare-and-run tmp-dir) 22 | (outdirs-check/is-equal "{{dir-path}}" tmp-dir) 23 | ))) -------------------------------------------------------------------------------- /test/keboola/test_utils/core.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.test-utils.core 2 | (:require [clojure.string :as string] 3 | [clojure.spec.test.alpha :as stest] 4 | [clojure.java.io :as io] 5 | [clojure.test :refer [is]])) 6 | 7 | (defn test-and-check 8 | ([spec-test] (test-and-check spec-test 1000)) 9 | ([spec-test num-tests] 10 | (let [result (stest/summarize-results 11 | (stest/check spec-test {:clojure.spec.test.check/opts {:num-tests num-tests}}))] 12 | (= (:total result) (:check-passed result))))) 13 | 14 | (defn delete-recursively 15 | "Delete dir recursively" 16 | [fname] 17 | (let [func (fn [func f] 18 | (when (.isDirectory f) 19 | (doseq [f2 (.listFiles f)] 20 | (func func f2))) 21 | (clojure.java.io/delete-file f))] 22 | (func func (clojure.java.io/file fname)))) 23 | 24 | (defmacro with-err-str 25 | "Evaluates exprs in a context in which *err* is bound to a fresh 26 | StringWriter. Returns the string created by any nested printing 27 | calls." 28 | {:added "1.0"} 29 | [& body] 30 | `(let [s# (new java.io.StringWriter)] 31 | (binding [*err* s#] 32 | ~@body 33 | (str s#)))) 34 | 35 | (defn prints-error? [error-fn & params] 36 | (is (= (clojure.string/trim (with-err-str (apply error-fn params))) (clojure.string/join " " params)))) 37 | 38 | 39 | (defn prints-msg? [print-fn & params] 40 | (is (= (clojure.string/trim (with-out-str (apply print-fn params))) (clojure.string/join " " params)))) 41 | 42 | (defn mk-path [& args] 43 | (string/join "/" args)) 44 | 45 | (defn copy-config-tmp [dir-path tmp-path] 46 | (let [source-path (str dir-path "/config.json") 47 | dest-path (str tmp-path "/config.json")] 48 | (io/copy (io/file source-path) (io/file dest-path)))) 49 | 50 | (defn mk-tmp-dir! 51 | "Creates a unique temporary directory on the filesystem. Typically in /tmp on 52 | *NIX systems. Returns a File object pointing to the new directory. Raises an 53 | exception if the directory couldn't be created after 100 tries." 54 | [prefix] 55 | (let [base-dir (java.io.File. (System/getProperty "java.io.tmpdir")) 56 | base-name (str prefix (java.util.Date.) "-" (long (rand 100)) "-") 57 | tmp-base (mk-path base-dir base-name) 58 | max-attempts 100] 59 | (loop [num-attempts 1] 60 | (if (= num-attempts max-attempts) 61 | (throw (Exception. (str "Failed to create temporary directory after " max-attempts " attempts."))) 62 | (let [tmp-dir-name (str tmp-base num-attempts) 63 | tmp-dir (java.io.File. tmp-dir-name)] 64 | (if (.mkdir tmp-dir) 65 | tmp-dir 66 | (recur (inc num-attempts)))))))) 67 | 68 | 69 | (defn 70 | ^{ :doc "Deletes the given directory even if it contains files or subdirectories. This function will attempt to delete 71 | all of the files and directories in the given directory first, before deleting the directory. If the directory cannot be 72 | deleted, this function aborts and returns nil. If the delete finishes successfully, then this function returns true."} 73 | recursive-delete [directory] 74 | (if (.isDirectory directory) 75 | (when (reduce #(and %1 (recursive-delete %2)) true (.listFiles directory)) 76 | (.delete directory)) 77 | (.delete directory))) 78 | -------------------------------------------------------------------------------- /test/keboola/utils/json_to_csv_test.clj: -------------------------------------------------------------------------------- 1 | (ns keboola.utils.json-to-csv-test 2 | (:require [keboola.utils.json-to-csv :as sut] 3 | [keboola.test-utils.core :refer [test-and-check]] 4 | [clojure.test :refer :all])) 5 | 6 | (deftest test-replace-dash 7 | (is (test-and-check `sut/replace-dash 20))) 8 | 9 | (deftest test-prepare-kw-map 10 | (is (test-and-check `sut/prepare-kw-map 20))) 11 | 12 | (deftest test-underscorize 13 | (is (test-and-check `sut/underscorize 20))) 14 | --------------------------------------------------------------------------------