├── .gitignore ├── .idea ├── .gitignore ├── .name ├── compiler.xml ├── encodings.xml ├── jarRepositories.xml ├── misc.xml ├── uiDesigner.xml └── vcs.xml ├── DATA_POLICY.md ├── LICENSE ├── README.md ├── chronicle.iml ├── docker ├── .env ├── .gitignore ├── docker-compose.yaml ├── nginx-template-variables └── nginx.conf ├── pom.xml └── src ├── main ├── java │ └── com │ │ └── hackclub │ │ ├── bank │ │ ├── Main.java │ │ ├── commands │ │ │ └── GenerateProfilesCommand.java │ │ └── models │ │ │ ├── BankAccountDataEntry.java │ │ │ ├── BankAccountDocument.java │ │ │ └── BankAccountMetadata.java │ │ ├── clubs │ │ ├── GlobalData.java │ │ ├── Main.java │ │ ├── commands │ │ │ └── GenerateProfilesCommand.java │ │ ├── engagements │ │ │ ├── Blot.java │ │ │ ├── Onboard.java │ │ │ └── Sprig.java │ │ ├── events │ │ │ ├── Angelhacks.java │ │ │ ├── Assemble.java │ │ │ └── Outernet.java │ │ ├── github │ │ │ └── Github.java │ │ ├── models │ │ │ ├── ChannelDay.java │ │ │ ├── ChannelEvent.java │ │ │ ├── ClubInfo.java │ │ │ ├── ClubLeaderApplicationInfo.java │ │ │ ├── GeoPoint.java │ │ │ ├── GithubInfo.java │ │ │ ├── HackClubUser.java │ │ │ ├── OperationsInfo.java │ │ │ ├── PirateShipEntry.java │ │ │ ├── ScrapbookAccount.java │ │ │ ├── SlackInfo.java │ │ │ ├── StaffUsers.java │ │ │ ├── engagements │ │ │ │ ├── BlotEngagement.java │ │ │ │ ├── Engagement.java │ │ │ │ ├── OnboardEngagement.java │ │ │ │ └── SprigEngagement.java │ │ │ └── event │ │ │ │ ├── AngelhacksRegistration.java │ │ │ │ ├── AssembleRegistration.java │ │ │ │ ├── EventRegistration.java │ │ │ │ └── OuternetRegistration.java │ │ └── slack │ │ │ └── SlackUtils.java │ │ └── common │ │ ├── Utils.java │ │ ├── agg │ │ ├── Aggregator.java │ │ ├── DoubleAggregator.java │ │ ├── EntityDataExtractor.java │ │ └── EntityProcessor.java │ │ ├── conflation │ │ ├── MatchResult.java │ │ ├── MatchScorer.java │ │ └── Matcher.java │ │ ├── elasticsearch │ │ ├── ESCommand.java │ │ ├── ESUtils.java │ │ └── InitIndexCommand.java │ │ ├── file │ │ ├── BlobStore.java │ │ └── Cache.java │ │ └── geo │ │ └── Geocoder.java └── resources │ ├── es │ ├── geomapping.esquery │ └── settings.esquery │ └── github_user_query.graphql └── test └── java └── com └── hackclub └── common ├── agg ├── DoubleAggregatorTest.java └── EntityProcessorTest.java └── conflation └── MatcherTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | ### IntelliJ IDEA ### 2 | out/ 3 | !**/src/main/**/out/ 4 | !**/src/test/**/out/ 5 | 6 | ### Eclipse ### 7 | .apt_generated 8 | .classpath 9 | .factorypath 10 | .project 11 | .settings 12 | .springBeans 13 | .sts4-cache 14 | bin/ 15 | !**/src/main/**/bin/ 16 | !**/src/test/**/bin/ 17 | 18 | ### NetBeans ### 19 | /nbproject/private/ 20 | /nbbuild/ 21 | /dist/ 22 | /nbdist/ 23 | /.nb-gradle/ 24 | 25 | ### VS Code ### 26 | .vscode/ 27 | 28 | ### Mac OS ### 29 | .DS_Store 30 | 31 | target/ 32 | pom.xml.tag 33 | pom.xml.releaseBackup 34 | pom.xml.versionsBackup 35 | pom.xml.next 36 | release.properties 37 | dependency-reduced-pom.xml 38 | buildNumber.properties 39 | .mvn/timing.properties 40 | # https://github.com/takari/maven-wrapper#usage-without-binary-jar 41 | .mvn/wrapper/maven-wrapper.jar 42 | 43 | .idea/workspace.workspace.xml 44 | .env -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | chronicle -------------------------------------------------------------------------------- /.idea/compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /.idea/encodings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /.idea/jarRepositories.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 9 | 10 | 14 | 15 | 19 | 20 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /.idea/uiDesigner.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /DATA_POLICY.md: -------------------------------------------------------------------------------- 1 | Hack Club Data Policy 2 | ===================== 3 | 4 | Right now, at HQ, there are a bajillion different places where data is stored. Sprig saves game files in S3, HQ event registrations are spread across Airtable, Google Sheets, and Postgres DBs, club applications are in a separate Airtable DB, Slack activity is in the Slack analytics dashboard, there is stuff all over GitHub, HCB has its own separate Airtable DBs and Postgres, and a bunch more places too. 5 | 6 | This has made it very difficult to answer basic questions like: "who are active Hack Clubbers within driving distance of AngelHacks?" or "who came to Epoch and has contributed to Hack Club repos on GitHub?". 7 | 8 | We want to make some tools to make this a lot easier. To do this, we'd be pulling data that we already have in various places or that's already public together into one place. 9 | 10 | This project got us thinking about Hack Club's lack of a clear data policy in general, and we thought it'd be a good opportunity to start formalizing things. 11 | 12 | We're making a public proposal because we want your feedback before making anything official. We believe you should have a say in what we do with this data. 13 | 14 | Proposal 15 | -------- 16 | 17 | Our proposed data policy is simple: we will only use data you explicitly give us (ex. club application form, event registration) or information that's already public and relevant to Hack Club (ex. GitHub PRs, repos) in HQ systems. 18 | 19 | We won't use anything that falls within the realm of "metadata". For example, tracking whether or not you specifically clicked a link in an HQ-sent email, or figuring out your specific location by geocoding your IP address that you use to log into Slack. 20 | 21 | Example of explicitly giving us data: 22 | 23 | - If you're a club leader, you filled out a form to give us your address. That's how we can send you cool packages! You entered your address in a field and pressed a submit button with the intent of giving us that address, so this counts as explicitly giving us data. 24 | 25 | Example of not explicitly giving us data: 26 | 27 | - We won't use geolocated IP logs for form submissions because this was collected implicitly. The same applies for web analytics data, login cookies, and anything else that wasn't submitted to us with the physical intent of doing so. 28 | 29 | Some examples of what "public data" is and isn't: 30 | 31 | - When you submit a PR to a public GitHub repo, that's public data because anyone can see it. We might want to be able to ask questions like "who has submitted PRs to Hack Club repos and hasn't been to an HQ-led event yet?" 32 | 33 | - When you log in to Slack, there are access logs available to admins that show IP addresses. This info isn't public and you didn't explicitly give it to us, so it's off limits. We can only use IP addresses for bans. 34 | 35 | - Only publicly available data on Slack (such as information similar to [Slack Analytics](https://hackclub.slack.com/stats)), usually focusing on channels for HQ-led projects, such as Sprig, Haxidraw, and Burrow. The actual content of messages will not be used. Private channels and DMs are always off limits, [like we announced 5 years ago.](https://hackclub.slack.com/archives/C0266FRGT/p1521835388000021) 36 | 37 | Notable exceptions / caveats 38 | 39 | - There are certain systems (for example, HCB) that may require long-term logging of IP addresses and other metadata for purposes of security / fraud detection & deterrence / auditing. This data is and will not be used for marketing purposes, nor would it be conflated with other outside datasets such as what Chronicle uses, but instead is purely intended to ensure engineering stability and compliance with financial standards / laws. 40 | 41 | Other aspects of data policy: 42 | 43 | - Outside of core operations (ex. we have to give emails to Stripe for people to get HCB cards mailed to them, or we have to give your email to GitHub to invite you to a HC GitHub repo), Hack Club HQ won't share data with external parties without people opting-in first. Ex. If people want invites for early access to GitHub's new code search, we will ask who wants invites first before sharing emails. 44 | 45 | - Hack Club HQ won't sell data to anyone 46 | 47 | We'd like to propose this for a starting place for the Hack Club data policy. It will probably evolve as more HQ systems get built up and we run into more complicated scenarios. 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Hack Club 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The Hack Club Dashboard! 2 | 3 | ## Purpose 4 | 5 | Chronicle is the “Hack Club dashboard”, and is meant to: 6 | 7 | 1. Provide metrics around HQ programs, like “How many people have completed a ‘you ship, we ship’ project like Sprig this month?” 8 | 2. Enable Hack Club staff to do direct outreach to Hack Clubbers who are involved in one part of Hack Club, but not others yet (ex. “Who has published a Sprig game, but not yet come to an HQ hackathon?“). 9 | 10 | Some examples of questions that Chronicle should be able to answer: 11 | 12 | * Who are active Hack Clubbers within driving distance of AngelHacks? 13 | * Who came to Epoch and has contributed to Hack Club repos on GitHub? 14 | * Who is new to Slack, but hasn’t contributed a Sprig game? 15 | * Who has a lot of expertise with Rust, but hasn’t contributed to the Burrow project nor joined its channel? 16 | 17 | Key to this project is the fact that Hack Club is volunteer-led, and doesn’t want to mandate usage of universal systems (ex. unified hackathon registration) across the organization. 18 | 19 | ## Components 20 | Chronicle is split into two major components: 21 | 1. A suite of command line tools that load, transform, conflate, and ultimately sync data in a target Elasticsearch cluster 22 | 1. An Elasticsearch cluster w/ Kibana used for creating dashboards 23 | 24 | ## Usage 25 | Access to Chronicle will be limited to a very small set of Hack Club HQ employees with a set of very strict use cases. 26 | 27 | ## Data 28 | #### Data sources 29 | Chronicle uses data from a variety of sources, including but not limited to: 30 | * Airtable (leaders table, ops address table) 31 | * Scrapbook DB 32 | * Slack APIs 33 | * Google Geocoding APIs 34 | * Github APIs 35 | * Pirateship 36 | * Raw slack data exports (**ONLY** public data) 37 | 38 | #### Data freshness 39 | By design, Chronicle will not be designed in a way where new changes in underlying data will be synced to Elasticsearch promptly. Instead, snapshots will be generated periodically from origin datasources, and then subsequently be consumed when we perform our next sync. 40 | 41 | #### Data privacy 42 | Given the sensitive nature of the data Chronicle uses, Chronicle is **not** a tool made for public consumption. All data posted here in this repo is for purposes of testing and development is mock data not relating to any real person. 43 | 44 | Please refer to Hack Club's official data policy guidelines [here](https://github.com/hackclub/chronicle/blob/main/DATA_POLICY.md). 45 | 46 | ## Local development environment 47 | 48 | #### Bring up the local Elasticsearch stack 49 | ``` 50 | cd docker 51 | docker-compose up 52 | ``` 53 | 54 | #### Package the CLI tool into fat jar (dependencies included) form 55 | ``` 56 | mvn package 57 | java -jar target/chronicle-1.0-SNAPSHOT-jar-with-dependencies.jar -h 58 | ``` 59 | 60 | ## Running tests locally 61 | ``` 62 | mvn test 63 | ``` 64 | 65 | ## Contribution guide 66 | All contributions must... 67 | * ... be sent in pull request form... 68 | * ... have at least one reviewer approving from the 'infra' team... 69 | * ... not cause any test to fail... 70 | 71 | ...before merging code to the main branch. 72 | -------------------------------------------------------------------------------- /chronicle.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /docker/.env: -------------------------------------------------------------------------------- 1 | STACK_VERSION=8.7.0 2 | ELASTIC_PASSWORD=change_me_please 3 | KIBANA_PASSWORD=change_me_please 4 | ES_PORT=9200 5 | CLUSTER_NAME=es-cluster 6 | LICENSE=basic 7 | MEM_LIMIT=4g 8 | KIBANA_PORT=5601 9 | ENTERPRISE_SEARCH_PORT=3002 10 | ENCRYPTION_KEYS=6d04ab8fe85116538a7c55b4ffa5436108f57fde90b63f0d1a752a4005af368f 11 | HOSTNAME=localhost -------------------------------------------------------------------------------- /docker/.gitignore: -------------------------------------------------------------------------------- 1 | ssl 2 | -------------------------------------------------------------------------------- /docker/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "2.2" 2 | 3 | services: 4 | setup: 5 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 6 | volumes: 7 | - certs:/usr/share/elasticsearch/config/certs 8 | user: "0" 9 | networks: 10 | - chronicle-backend 11 | command: > 12 | bash -c ' 13 | if [ x${ELASTIC_PASSWORD} == x ]; then 14 | echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; 15 | exit 1; 16 | elif [ x${KIBANA_PASSWORD} == x ]; then 17 | echo "Set the KIBANA_PASSWORD environment variable in the .env file"; 18 | exit 1; 19 | fi; 20 | if [ ! -f certs/ca.zip ]; then 21 | echo "Creating CA"; 22 | bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; 23 | unzip config/certs/ca.zip -d config/certs; 24 | fi; 25 | if [ ! -f certs/certs.zip ]; then 26 | echo "Creating certs"; 27 | echo -ne \ 28 | "instances:\n"\ 29 | " - name: es01\n"\ 30 | " dns:\n"\ 31 | " - es01\n"\ 32 | " - localhost\n"\ 33 | " ip:\n"\ 34 | " - 127.0.0.1\n"\ 35 | > config/certs/instances.yml; 36 | bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; 37 | unzip config/certs/certs.zip -d config/certs; 38 | fi; 39 | echo "Setting file permissions" 40 | chown -R root:root config/certs; 41 | find . -type d -exec chmod 750 \{\} \;; 42 | find . -type f -exec chmod 640 \{\} \;; 43 | echo "Waiting for Elasticsearch availability"; 44 | until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; 45 | echo "Setting kibana_system password"; 46 | until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; 47 | echo "All done!"; 48 | ' 49 | healthcheck: 50 | test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] 51 | interval: 1s 52 | timeout: 5s 53 | retries: 120 54 | 55 | es01: 56 | depends_on: 57 | setup: 58 | condition: service_healthy 59 | image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} 60 | volumes: 61 | - certs:/usr/share/elasticsearch/config/certs 62 | - esdata01:/usr/share/elasticsearch/data 63 | networks: 64 | - chronicle-backend 65 | ports: 66 | - ${ES_PORT}:9200 67 | environment: 68 | - node.name=es01 69 | - cluster.name=${CLUSTER_NAME} 70 | - cluster.initial_master_nodes=es01 71 | - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} 72 | - bootstrap.memory_lock=true 73 | - xpack.security.enabled=true 74 | - xpack.security.http.ssl.enabled=true 75 | - xpack.security.http.ssl.key=certs/es01/es01.key 76 | - xpack.security.http.ssl.certificate=certs/es01/es01.crt 77 | - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt 78 | - xpack.security.http.ssl.verification_mode=certificate 79 | - xpack.security.transport.ssl.enabled=true 80 | - xpack.security.transport.ssl.key=certs/es01/es01.key 81 | - xpack.security.transport.ssl.certificate=certs/es01/es01.crt 82 | - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt 83 | - xpack.security.transport.ssl.verification_mode=certificate 84 | - xpack.license.self_generated.type=${LICENSE} 85 | mem_limit: ${MEM_LIMIT} 86 | ulimits: 87 | memlock: 88 | soft: -1 89 | hard: -1 90 | healthcheck: 91 | test: 92 | [ 93 | "CMD-SHELL", 94 | "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'", 95 | ] 96 | interval: 10s 97 | timeout: 10s 98 | retries: 120 99 | 100 | kibana: 101 | depends_on: 102 | es01: 103 | condition: service_healthy 104 | image: docker.elastic.co/kibana/kibana:${STACK_VERSION} 105 | volumes: 106 | - certs:/usr/share/kibana/config/certs 107 | - kibanadata:/usr/share/kibana/data 108 | networks: 109 | - chronicle-backend 110 | environment: 111 | - SERVERNAME=kibana 112 | - ELASTICSEARCH_HOSTS=https://es01:9200 113 | - ELASTICSEARCH_USERNAME=kibana_system 114 | - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} 115 | - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt 116 | - ENTERPRISESEARCH_HOST=http://enterprisesearch:${ENTERPRISE_SEARCH_PORT} 117 | mem_limit: ${MEM_LIMIT} 118 | healthcheck: 119 | test: 120 | [ 121 | "CMD-SHELL", 122 | "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'", 123 | ] 124 | interval: 10s 125 | timeout: 10s 126 | retries: 120 127 | 128 | enterprisesearch: 129 | depends_on: 130 | es01: 131 | condition: service_healthy 132 | kibana: 133 | condition: service_healthy 134 | image: docker.elastic.co/enterprise-search/enterprise-search:${STACK_VERSION} 135 | volumes: 136 | - certs:/usr/share/enterprise-search/config/certs 137 | - enterprisesearchdata:/usr/share/enterprise-search/config 138 | ports: 139 | - ${ENTERPRISE_SEARCH_PORT}:3002 140 | environment: 141 | - SERVERNAME=enterprisesearch 142 | - secret_management.encryption_keys=[${ENCRYPTION_KEYS}] 143 | - allow_es_settings_modification=true 144 | - elasticsearch.host=https://es01:9200 145 | - elasticsearch.username=elastic 146 | - elasticsearch.password=${ELASTIC_PASSWORD} 147 | - elasticsearch.ssl.enabled=true 148 | - elasticsearch.ssl.certificate_authority=/usr/share/enterprise-search/config/certs/ca/ca.crt 149 | - kibana.external_url=http://kibana:5601 150 | mem_limit: ${MEM_LIMIT} 151 | networks: 152 | - chronicle-backend 153 | healthcheck: 154 | test: 155 | [ 156 | "CMD-SHELL", 157 | "curl -s -I http://localhost:3002 | grep -q 'HTTP/1.1 302 Found'", 158 | ] 159 | interval: 10s 160 | timeout: 10s 161 | retries: 120 162 | 163 | nginx: 164 | depends_on: 165 | es01: 166 | condition: service_healthy 167 | kibana: 168 | condition: service_healthy 169 | image: nginx:latest 170 | container_name: webserver 171 | restart: unless-stopped 172 | environment: 173 | HOSTNAME: ${HOSTNAME} 174 | ports: 175 | - 80:80 176 | - 443:443 177 | volumes: 178 | - ./nginx.conf:/etc/nginx/nginx.conf 179 | - ./ssl/:/ssl/ 180 | - ./nginx-template-variables:/etc/nginx/templates/chronicle.conf.template:ro 181 | networks: 182 | - chronicle-backend 183 | 184 | 185 | volumes: 186 | certs: 187 | driver: local 188 | enterprisesearchdata: 189 | driver: local 190 | esdata01: 191 | driver: local 192 | kibanadata: 193 | driver: local 194 | 195 | networks: 196 | chronicle-backend: 197 | -------------------------------------------------------------------------------- /docker/nginx-template-variables: -------------------------------------------------------------------------------- 1 | map $host $hostname { 2 | default "$HOSTNAME"; 3 | } 4 | -------------------------------------------------------------------------------- /docker/nginx.conf: -------------------------------------------------------------------------------- 1 | events { } 2 | 3 | http { 4 | 5 | map $http_upgrade $connection_upgrade { 6 | default upgrade; 7 | '' close; 8 | } 9 | 10 | server { 11 | listen 80; 12 | server_name $hostname; 13 | return 301 https://$hostname$request_uri; 14 | } 15 | 16 | server { 17 | listen 443 ssl; 18 | server_name $hostname; 19 | 20 | ssl_certificate /ssl/ssl_cert.pem; 21 | ssl_certificate_key /ssl/ssl_private_key.pem; 22 | 23 | access_log /var/log/nginx/data-access.log combined; 24 | 25 | location / { 26 | proxy_pass http://kibana:5601/; 27 | proxy_set_header X-Real-IP $remote_addr; 28 | proxy_set_header X-Forwarded-For $remote_addr; 29 | proxy_set_header Host $host; 30 | proxy_set_header X-Forwarded-Proto $scheme; 31 | proxy_redirect http://kibana:5601/ $scheme://$http_host/; 32 | proxy_http_version 1.1; 33 | proxy_set_header Upgrade $http_upgrade; 34 | proxy_set_header Connection $connection_upgrade; 35 | proxy_read_timeout 20d; 36 | proxy_buffering off; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | groupId 8 | chronicle 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 20 13 | 20 14 | UTF-8 15 | 16 | 17 | 18 | 19 | com.google.code.gson 20 | gson 21 | 2.10.1 22 | 23 | 24 | info.picocli 25 | picocli 26 | 4.6.3 27 | 28 | 29 | com.opencsv 30 | opencsv 31 | 5.5 32 | 33 | 34 | co.elastic.clients 35 | elasticsearch-java 36 | 8.7.0 37 | 38 | 39 | com.fasterxml.jackson.core 40 | jackson-databind 41 | 2.12.3 42 | 43 | 44 | com.google.maps 45 | google-maps-services 46 | 2.2.0 47 | 48 | 49 | org.slf4j 50 | slf4j-simple 51 | 1.7.25 52 | 53 | 54 | org.junit.jupiter 55 | junit-jupiter 56 | 5.7.1 57 | 58 | 59 | org.kohsuke 60 | github-api 61 | 1.315 62 | 63 | 64 | com.jayway.jsonpath 65 | json-path 66 | 2.8.0 67 | 68 | 69 | com.slack.api 70 | slack-api-client 71 | 1.30.0 72 | 73 | 74 | 75 | 76 | 77 | 78 | maven-assembly-plugin 79 | 80 | 81 | 82 | com.hackclub.clubs.Main 83 | 84 | 85 | 86 | jar-with-dependencies 87 | 88 | 89 | 90 | 91 | make-assembly 92 | package 93 | 94 | single 95 | 96 | 97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/bank/Main.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.bank; 2 | 3 | import com.hackclub.bank.commands.GenerateProfilesCommand; 4 | import com.hackclub.common.elasticsearch.InitIndexCommand; 5 | import picocli.CommandLine; 6 | import picocli.CommandLine.Command; 7 | 8 | @Command(name = "bank-chronicle", 9 | mixinStandardHelpOptions = true, 10 | version = "bank-chronicle 1.0", 11 | description = "Data conflation and indexing tool", 12 | subcommands = { GenerateProfilesCommand.class, InitIndexCommand.class }) 13 | public class Main { 14 | public static void main(String... args) { 15 | int exitCode = new CommandLine(new Main()).execute(args); 16 | System.exit(exitCode); 17 | } 18 | } -------------------------------------------------------------------------------- /src/main/java/com/hackclub/bank/commands/GenerateProfilesCommand.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.bank.commands; 2 | 3 | import com.hackclub.bank.Main; 4 | import com.hackclub.bank.models.BankAccountDataEntry; 5 | import com.hackclub.bank.models.BankAccountDocument; 6 | import com.hackclub.bank.models.BankAccountMetadata; 7 | import co.elastic.clients.elasticsearch.ElasticsearchClient; 8 | import com.opencsv.CSVReader; 9 | import com.opencsv.CSVReaderBuilder; 10 | import com.opencsv.exceptions.CsvValidationException; 11 | import com.hackclub.common.elasticsearch.ESCommand; 12 | import com.hackclub.common.elasticsearch.ESUtils; 13 | import com.hackclub.common.geo.Geocoder; 14 | import picocli.CommandLine; 15 | 16 | import java.io.File; 17 | import java.io.FileReader; 18 | import java.io.IOException; 19 | import java.util.ArrayList; 20 | import java.util.HashMap; 21 | import java.util.Optional; 22 | import java.util.stream.Stream; 23 | 24 | /** 25 | * Generates user profiles and syncs them to an elasticsearch cluster 26 | */ 27 | @CommandLine.Command(name = "genprofiles") 28 | public class GenerateProfilesCommand extends ESCommand { 29 | @CommandLine.ParentCommand 30 | private Main mainCmd; 31 | 32 | @CommandLine.Parameters(index = "0", description = "A .csv file with a mapping of all bank accounts") 33 | private File bankAccountsCsvFile; 34 | 35 | @CommandLine.Parameters(index = "1", description = "A .csv file with a mapping of all bank account financial data for a time period") 36 | private File bankAccountEntryCsvFile; 37 | 38 | @Override 39 | public Integer call() throws Exception { 40 | ElasticsearchClient esClient = ESUtils.createElasticsearchClient(esHostname, esPort, esUsername, esPassword, esFingerprint); 41 | 42 | ESUtils.clearIndex(esClient, esIndex); 43 | 44 | HashMap bankAccounts = loadBankAccounts(); 45 | ArrayList dataEntries = loadBankData(); 46 | 47 | Stream docs = dataEntries.stream() 48 | .flatMap(entry -> 49 | getAccountForEntry(bankAccounts, entry).stream() 50 | .map(acc -> new BankAccountDocument(acc, entry))); 51 | 52 | writeBankAccountsToES(esClient, docs); 53 | 54 | Geocoder.shutdown(); 55 | return 0; 56 | } 57 | 58 | private Optional getAccountForEntry(HashMap bankAccounts, BankAccountDataEntry entry) { 59 | for (BankAccountMetadata md : bankAccounts.values()) { 60 | if (entry.getEmailAddresses().contains(md.getEmailAddress())) 61 | return Optional.of(md); 62 | } 63 | return Optional.empty(); 64 | } 65 | 66 | private HashMap loadBankAccounts() throws IOException, CsvValidationException { 67 | CSVReader reader = new CSVReaderBuilder(new FileReader(bankAccountsCsvFile)) 68 | .withSkipLines(1) 69 | .build(); 70 | 71 | String[] columns = "Event Name,Assignee,Org Type,Created At,Status,First Name,Last Name,Email Address,Phone Number,Date of Birth,Formatted Mailing Address,Mailing Address,Event Website,Event Location,Tell us about your event,Have you used Hack Club Bank for any previous events?,Comments,HCB account URL,Address Country,How did you hear about HCB?,Pending,Transparent".split(","); 72 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 73 | 74 | String [] nextLine; 75 | 76 | HashMap allBankAccounts = new HashMap<>(); 77 | while ((nextLine = reader.readNext()) != null) 78 | { 79 | BankAccountMetadata account = BankAccountMetadata.fromCsv(nextLine, columnIndices); 80 | if (account.getEmailAddress().length() > 0) 81 | allBankAccounts.put(account.getEmailAddress(), account); 82 | } 83 | return allBankAccounts; 84 | } 85 | 86 | private ArrayList loadBankData() throws IOException, CsvValidationException { 87 | CSVReader reader = new CSVReaderBuilder(new FileReader(bankAccountEntryCsvFile)) 88 | .withSkipLines(1) 89 | .build(); 90 | 91 | String[] columns = "slug,balance,amount_transacted,amount_raised,array_agg".split(","); 92 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 93 | 94 | String [] nextLine; 95 | 96 | ArrayList allBankData = new ArrayList<>(); 97 | while ((nextLine = reader.readNext()) != null) 98 | { 99 | BankAccountDataEntry data = BankAccountDataEntry.fromCsv(nextLine, columnIndices); 100 | allBankData.add(data); 101 | } 102 | return allBankData; 103 | } 104 | 105 | private void writeBankAccountsToES(ElasticsearchClient esClient, Stream docs) { 106 | // TODO: Batch writes are much faster 107 | docs.forEach(acc -> { 108 | try { 109 | if (acc.getMailingAddress() != null) { 110 | try { 111 | Geocoder.geocode(acc.getMailingAddress()).ifPresent(acc::setGeolocation); 112 | } catch (Throwable t) { 113 | System.out.printf("Issue geocoding: %s\n", t.getMessage()); 114 | } 115 | } 116 | 117 | esClient.index(i -> i 118 | .index(esIndex) 119 | .id(acc.getName()) 120 | .document(acc)); 121 | } catch (Throwable t) { 122 | System.out.println(String.format("Issue writing account %s to ES - %s", acc.toString(), t.getMessage())); 123 | } 124 | }); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/bank/models/BankAccountDataEntry.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.bank.models; 2 | 3 | import java.util.HashMap; 4 | import java.util.HashSet; 5 | 6 | /** 7 | * Model object representing a single HCB account, including its balance, transacted funds, and money raised 8 | */ 9 | public class BankAccountDataEntry { 10 | private Double amountRaised; 11 | private Double amountTransacted; 12 | private Double balance; 13 | private HashSet emailAddresses; 14 | 15 | public static BankAccountDataEntry fromCsv(String[] nextLine, HashMap columnIndices) { 16 | BankAccountDataEntry data = new BankAccountDataEntry(); 17 | data.amountRaised = Double.parseDouble(nextLine[columnIndices.get("amount_raised")]); 18 | data.amountTransacted = Double.parseDouble(nextLine[columnIndices.get("amount_transacted")]); 19 | data.balance = Double.parseDouble(nextLine[columnIndices.get("balance")]); 20 | data.emailAddresses = parseEmails(nextLine[columnIndices.get("array_agg")]); 21 | return data; 22 | } 23 | 24 | public Double getAmountRaised() { 25 | return amountRaised; 26 | } 27 | 28 | public void setAmountRaised(Double amountRaised) { 29 | this.amountRaised = amountRaised; 30 | } 31 | 32 | public Double getAmountTransacted() { 33 | return amountTransacted; 34 | } 35 | 36 | public void setAmountTransacted(Double amountTransacted) { 37 | this.amountTransacted = amountTransacted; 38 | } 39 | 40 | public Double getBalance() { 41 | return balance; 42 | } 43 | 44 | public void setBalance(Double balance) { 45 | this.balance = balance; 46 | } 47 | 48 | public HashSet getEmailAddresses() { 49 | return emailAddresses; 50 | } 51 | 52 | @Override 53 | public String toString() { 54 | return "BankAccountDataEntry{" + 55 | "amountRaised=" + amountRaised + 56 | ", amountTransacted=" + amountTransacted + 57 | ", balance=" + balance + 58 | ", emailAddresses=" + emailAddresses + 59 | '}'; 60 | } 61 | 62 | private static HashSet parseEmails(String emails) { 63 | HashSet ret = new HashSet<>(); 64 | for(String email : emails.split(",")) { 65 | email = email.replace("{", "").replace("}", ""); 66 | ret.add(email); 67 | } 68 | return ret; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/bank/models/BankAccountDocument.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.bank.models; 2 | 3 | import com.hackclub.clubs.models.GeoPoint; 4 | 5 | /** 6 | * The model object that represents the document we actually want to index in Elasticsearch 7 | */ 8 | public class BankAccountDocument { 9 | private String name; 10 | private String orgType; 11 | private String emailAddress; 12 | private String firstName; 13 | private String lastName; 14 | private String phoneNumber; 15 | private String dateOfBirth; 16 | private String mailingAddress; 17 | private String website; 18 | private String eventLocation; 19 | private GeoPoint geolocation; 20 | private String transparency; 21 | private String hcbUrl; 22 | private String pendingState; 23 | private Double amountRaised; 24 | private Double amountTransacted; 25 | private Double balance; 26 | private boolean hasGeo = false; 27 | 28 | public BankAccountDocument(BankAccountMetadata account, BankAccountDataEntry data) { 29 | name = account.getName(); 30 | orgType = account.getOrgType(); 31 | emailAddress = account.getEmailAddress(); 32 | firstName = account.getFirstName(); 33 | lastName = account.getLastName(); 34 | phoneNumber = account.getPhoneNumber(); 35 | dateOfBirth = account.getDateOfBirth(); 36 | mailingAddress = account.getMailingAddress(); 37 | website = account.getWebsite(); 38 | eventLocation = account.getEventLocation(); 39 | geolocation = account.getGeolocation(); 40 | transparency = account.getTransparency(); 41 | hcbUrl = account.getHcbUrl(); 42 | pendingState = account.getPendingState(); 43 | amountRaised = data.getAmountRaised() / 100.0; 44 | amountTransacted = data.getAmountTransacted() / 100.0; 45 | balance = data.getBalance() / 100.0; 46 | } 47 | 48 | public String getName() { 49 | return name; 50 | } 51 | 52 | public void setName(String name) { 53 | this.name = name; 54 | } 55 | 56 | public String getOrgType() { 57 | return orgType; 58 | } 59 | 60 | public void setOrgType(String orgType) { 61 | this.orgType = orgType; 62 | } 63 | 64 | public String getEmailAddress() { 65 | return emailAddress; 66 | } 67 | 68 | public void setEmailAddress(String emailAddress) { 69 | this.emailAddress = emailAddress; 70 | } 71 | 72 | public String getFirstName() { 73 | return firstName; 74 | } 75 | 76 | public void setFirstName(String firstName) { 77 | this.firstName = firstName; 78 | } 79 | 80 | public String getLastName() { 81 | return lastName; 82 | } 83 | 84 | public void setLastName(String lastName) { 85 | this.lastName = lastName; 86 | } 87 | 88 | public String getPhoneNumber() { 89 | return phoneNumber; 90 | } 91 | 92 | public void setPhoneNumber(String phoneNumber) { 93 | this.phoneNumber = phoneNumber; 94 | } 95 | 96 | public String getDateOfBirth() { 97 | return dateOfBirth; 98 | } 99 | 100 | public void setDateOfBirth(String dateOfBirth) { 101 | this.dateOfBirth = dateOfBirth; 102 | } 103 | 104 | public String getMailingAddress() { 105 | return mailingAddress; 106 | } 107 | 108 | public void setMailingAddress(String mailingAddress) { 109 | this.mailingAddress = mailingAddress; 110 | } 111 | 112 | public String getWebsite() { 113 | return website; 114 | } 115 | 116 | public void setWebsite(String website) { 117 | this.website = website; 118 | } 119 | 120 | public String getEventLocation() { 121 | return eventLocation; 122 | } 123 | 124 | public void setEventLocation(String eventLocation) { 125 | this.eventLocation = eventLocation; 126 | } 127 | 128 | public GeoPoint getGeolocation() { 129 | return geolocation; 130 | } 131 | 132 | public void setGeolocation(GeoPoint geolocation) { 133 | this.geolocation = geolocation; 134 | this.hasGeo = geolocation != null; 135 | } 136 | 137 | public String getTransparency() { 138 | return transparency; 139 | } 140 | 141 | public void setTransparency(String transparency) { 142 | this.transparency = transparency; 143 | } 144 | 145 | public String getHcbUrl() { 146 | return hcbUrl; 147 | } 148 | 149 | public void setHcbUrl(String hcbUrl) { 150 | this.hcbUrl = hcbUrl; 151 | } 152 | 153 | public String getPendingState() { 154 | return pendingState; 155 | } 156 | 157 | public void setPendingState(String pendingState) { 158 | this.pendingState = pendingState; 159 | } 160 | 161 | public Double getAmountRaised() { 162 | return amountRaised; 163 | } 164 | 165 | public void setAmountRaised(Double amountRaised) { 166 | this.amountRaised = amountRaised; 167 | } 168 | 169 | public Double getAmountTransacted() { 170 | return amountTransacted; 171 | } 172 | 173 | public void setAmountTransacted(Double amountTransacted) { 174 | this.amountTransacted = amountTransacted; 175 | } 176 | 177 | public Double getBalance() { 178 | return balance; 179 | } 180 | 181 | public void setBalance(Double balance) { 182 | this.balance = balance; 183 | } 184 | 185 | @Override 186 | public String toString() { 187 | return "BankAccountDocument{" + 188 | "name='" + name + '\'' + 189 | ", orgType='" + orgType + '\'' + 190 | ", emailAddress='" + emailAddress + '\'' + 191 | ", firstName='" + firstName + '\'' + 192 | ", lastName='" + lastName + '\'' + 193 | ", phoneNumber='" + phoneNumber + '\'' + 194 | ", dateOfBirth='" + dateOfBirth + '\'' + 195 | ", mailingAddress='" + mailingAddress + '\'' + 196 | ", website='" + website + '\'' + 197 | ", eventLocation='" + eventLocation + '\'' + 198 | ", geolocation=" + geolocation + 199 | ", transparency='" + transparency + '\'' + 200 | ", hcbUrl='" + hcbUrl + '\'' + 201 | ", pendingState='" + pendingState + '\'' + 202 | ", amountRaised=" + amountRaised + 203 | ", amountTransacted=" + amountTransacted + 204 | ", balance=" + balance + 205 | '}'; 206 | } 207 | 208 | public boolean isHasGeo() { 209 | return hasGeo; 210 | } 211 | 212 | public void setHasGeo(boolean hasGeo) { 213 | this.hasGeo = hasGeo; 214 | } 215 | } 216 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/bank/models/BankAccountMetadata.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.bank.models; 2 | 3 | import com.hackclub.clubs.models.GeoPoint; 4 | 5 | import java.util.HashMap; 6 | 7 | /** 8 | * Model object representing bank account metadata 9 | */ 10 | public class BankAccountMetadata { 11 | private String name; 12 | private String orgType; 13 | private String emailAddress; 14 | private String firstName; 15 | private String lastName; 16 | private String phoneNumber; 17 | private String dateOfBirth; 18 | private String mailingAddress; 19 | private String website; 20 | private String eventLocation; 21 | private GeoPoint geolocation; 22 | private String transparency; 23 | private String hcbUrl; 24 | private String pendingState; 25 | 26 | // Not publicly instantiable - use factory methods (fromCsv) 27 | private BankAccountMetadata() { 28 | } 29 | 30 | public static BankAccountMetadata fromCsv(String[] nextLine, HashMap columnIndices) { 31 | BankAccountMetadata bam = new BankAccountMetadata(); 32 | bam.name = nextLine[columnIndices.get("Event Name")]; 33 | bam.orgType = nextLine[columnIndices.get("Org Type")]; 34 | bam.emailAddress = nextLine[columnIndices.get("Email Address")]; 35 | bam.firstName = nextLine[columnIndices.get("First Name")]; 36 | bam.lastName = nextLine[columnIndices.get("Last Name")]; 37 | bam.phoneNumber = nextLine[columnIndices.get("Phone Number")]; 38 | bam.dateOfBirth = nextLine[columnIndices.get("Date of Birth")]; 39 | bam.mailingAddress = nextLine[columnIndices.get("Formatted Mailing Address")]; 40 | bam.website = nextLine[columnIndices.get("Event Website")]; 41 | bam.eventLocation = nextLine[columnIndices.get("Event Location")]; 42 | bam.transparency = nextLine[columnIndices.get("Event Name")]; 43 | bam.hcbUrl = nextLine[columnIndices.get("HCB account URL")]; 44 | return bam; 45 | } 46 | 47 | public String getName() { 48 | return name; 49 | } 50 | 51 | public void setName(String name) { 52 | this.name = name; 53 | } 54 | 55 | public String getOrgType() { 56 | return orgType; 57 | } 58 | 59 | public void setOrgType(String orgType) { 60 | this.orgType = orgType; 61 | } 62 | 63 | public String getEmailAddress() { 64 | return emailAddress; 65 | } 66 | 67 | public void setEmailAddress(String emailAddress) { 68 | this.emailAddress = emailAddress; 69 | } 70 | 71 | public String getFirstName() { 72 | return firstName; 73 | } 74 | 75 | public void setFirstName(String firstName) { 76 | this.firstName = firstName; 77 | } 78 | 79 | public String getLastName() { 80 | return lastName; 81 | } 82 | 83 | public void setLastName(String lastName) { 84 | this.lastName = lastName; 85 | } 86 | 87 | public String getPhoneNumber() { 88 | return phoneNumber; 89 | } 90 | 91 | public void setPhoneNumber(String phoneNumber) { 92 | this.phoneNumber = phoneNumber; 93 | } 94 | 95 | public String getDateOfBirth() { 96 | return dateOfBirth; 97 | } 98 | 99 | public void setDateOfBirth(String dateOfBirth) { 100 | this.dateOfBirth = dateOfBirth; 101 | } 102 | 103 | public String getMailingAddress() { 104 | return mailingAddress; 105 | } 106 | 107 | public void setMailingAddress(String mailingAddress) { 108 | this.mailingAddress = mailingAddress; 109 | } 110 | 111 | public String getWebsite() { 112 | return website; 113 | } 114 | 115 | public void setWebsite(String website) { 116 | this.website = website; 117 | } 118 | 119 | public String getEventLocation() { 120 | return eventLocation; 121 | } 122 | 123 | public void setEventLocation(String eventLocation) { 124 | this.eventLocation = eventLocation; 125 | } 126 | 127 | public GeoPoint getGeolocation() { 128 | return geolocation; 129 | } 130 | 131 | public void setGeolocation(GeoPoint geolocation) { 132 | this.geolocation = geolocation; 133 | } 134 | 135 | public String getHcbUrl() { 136 | return hcbUrl; 137 | } 138 | 139 | public void setHcbUrl(String hcbUrl) { 140 | this.hcbUrl = hcbUrl; 141 | } 142 | 143 | public String getTransparency() { 144 | return transparency; 145 | } 146 | 147 | public void setTransparency(String transparency) { 148 | this.transparency = transparency; 149 | } 150 | 151 | public String getPendingState() { 152 | return pendingState; 153 | } 154 | 155 | public void setPendingState(String pendingState) { 156 | this.pendingState = pendingState; 157 | } 158 | 159 | @Override 160 | public String toString() { 161 | return "BankAccountMetadata{" + 162 | "name='" + name + '\'' + 163 | ", orgType='" + orgType + '\'' + 164 | ", emailAddress='" + emailAddress + '\'' + 165 | ", firstName='" + firstName + '\'' + 166 | ", lastName='" + lastName + '\'' + 167 | ", phoneNumber='" + phoneNumber + '\'' + 168 | ", dateOfBirth='" + dateOfBirth + '\'' + 169 | ", mailingAddress='" + mailingAddress + '\'' + 170 | ", website='" + website + '\'' + 171 | ", eventLocation='" + eventLocation + '\'' + 172 | ", geolocation=" + geolocation + 173 | ", transparency='" + transparency + '\'' + 174 | ", hcbUrl='" + hcbUrl + '\'' + 175 | ", pendingState='" + pendingState + '\'' + 176 | '}'; 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/GlobalData.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs; 2 | 3 | import java.util.HashSet; 4 | 5 | /** 6 | * A place of great evil 7 | */ 8 | public class GlobalData { 9 | public static HashSet staffUserIds = new HashSet<>(); 10 | public static HashSet validTokens = new HashSet<>(); 11 | static { 12 | validTokens.add("rust"); 13 | validTokens.add("sprig"); 14 | validTokens.add("c++"); 15 | validTokens.add("python"); 16 | validTokens.add("java"); 17 | validTokens.add("javascript"); 18 | validTokens.add("tcl"); 19 | validTokens.add("php"); 20 | validTokens.add("scala"); 21 | validTokens.add("ruby"); 22 | validTokens.add("swift"); 23 | validTokens.add("ios"); 24 | validTokens.add("android"); 25 | validTokens.add("web"); 26 | validTokens.add("chrome"); 27 | validTokens.add("firefox"); 28 | validTokens.add("edge"); 29 | validTokens.add("html"); 30 | validTokens.add("game"); 31 | validTokens.add("gamedev"); 32 | validTokens.add("games"); 33 | validTokens.add("firmware"); 34 | validTokens.add("asm"); 35 | validTokens.add("assembly"); 36 | validTokens.add("epoch"); 37 | validTokens.add("hackathon"); 38 | validTokens.add("fundraising"); 39 | validTokens.add("bank"); 40 | validTokens.add("sinerider"); 41 | validTokens.add("zephyr"); 42 | validTokens.add("horizon"); 43 | validTokens.add("leader"); 44 | validTokens.add("money"); 45 | validTokens.add("debug"); 46 | validTokens.add("debugging"); 47 | validTokens.add("kernel"); 48 | validTokens.add("network"); 49 | validTokens.add("develop"); 50 | validTokens.add("development"); 51 | validTokens.add("deploy"); 52 | validTokens.add("hacker"); 53 | validTokens.add("vercel"); 54 | validTokens.add("heroku"); 55 | validTokens.add("aws"); 56 | validTokens.add("cloud"); 57 | validTokens.add("distributed"); 58 | validTokens.add("club"); 59 | validTokens.add("teacher"); 60 | validTokens.add("school"); 61 | validTokens.add("advisor"); 62 | validTokens.add("workshop"); 63 | validTokens.add("meeting"); 64 | validTokens.add("members"); 65 | validTokens.add("stickers"); 66 | validTokens.add("register"); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/Main.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs; 2 | 3 | import com.hackclub.clubs.commands.GenerateProfilesCommand; 4 | import com.hackclub.common.elasticsearch.InitIndexCommand; 5 | import picocli.CommandLine; 6 | import picocli.CommandLine.Command; 7 | 8 | @Command(name = "clubs-chronicle", 9 | mixinStandardHelpOptions = true, 10 | version = "clubs-chronicle 1.0", 11 | description = "Data conflation and indexing tool", 12 | subcommands = { GenerateProfilesCommand.class, InitIndexCommand.class }) 13 | public class Main { 14 | public static void main(String... args) { 15 | int exitCode = new CommandLine(new Main()).execute(args); 16 | System.exit(exitCode); 17 | } 18 | } -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/commands/GenerateProfilesCommand.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.commands; 2 | 3 | import com.hackclub.clubs.Main; 4 | import com.hackclub.clubs.GlobalData; 5 | import com.hackclub.clubs.engagements.Blot; 6 | import com.hackclub.clubs.engagements.Onboard; 7 | import com.hackclub.clubs.engagements.Sprig; 8 | import com.hackclub.clubs.events.Angelhacks; 9 | import com.hackclub.clubs.events.Assemble; 10 | import com.hackclub.clubs.events.Outernet; 11 | import com.hackclub.clubs.github.Github; 12 | import com.hackclub.clubs.models.*; 13 | import com.hackclub.clubs.slack.SlackUtils; 14 | import co.elastic.clients.elasticsearch.ElasticsearchClient; 15 | import com.fasterxml.jackson.databind.ObjectMapper; 16 | import com.google.gson.Gson; 17 | import com.google.gson.reflect.TypeToken; 18 | import com.hackclub.common.Utils; 19 | import com.hackclub.common.agg.DoubleAggregator; 20 | import com.hackclub.common.agg.EntityDataExtractor; 21 | import com.hackclub.common.agg.EntityProcessor; 22 | import com.hackclub.common.conflation.MatchResult; 23 | import com.hackclub.common.conflation.MatchScorer; 24 | import com.hackclub.common.conflation.Matcher; 25 | import com.opencsv.CSVReader; 26 | import com.opencsv.CSVReaderBuilder; 27 | import com.opencsv.exceptions.CsvValidationException; 28 | import com.hackclub.common.agg.Aggregator; 29 | import com.hackclub.common.elasticsearch.ESCommand; 30 | import com.hackclub.common.elasticsearch.ESUtils; 31 | import com.hackclub.common.file.BlobStore; 32 | import com.hackclub.common.geo.Geocoder; 33 | import org.apache.commons.lang3.StringUtils; 34 | import picocli.CommandLine; 35 | 36 | import java.io.*; 37 | import java.net.URI; 38 | import java.nio.file.Files; 39 | import java.nio.file.Paths; 40 | import java.util.*; 41 | import java.util.concurrent.atomic.AtomicLong; 42 | import java.util.stream.Collectors; 43 | import java.util.stream.Stream; 44 | 45 | import static com.hackclub.common.elasticsearch.ESUtils.clearIndex; 46 | 47 | /** 48 | * Generates user profiles and syncs them to an elasticsearch cluster 49 | */ 50 | @CommandLine.Command(name = "genprofiles") 51 | public class GenerateProfilesCommand extends ESCommand { 52 | private static final AtomicLong counter = new AtomicLong(0); 53 | private static volatile boolean running = true; 54 | 55 | @CommandLine.ParentCommand 56 | private Main mainCmd; 57 | 58 | @CommandLine.Parameters(index = "0", description = "URI to csv file with a mapping of all slack users") 59 | private URI inputUsersUri; 60 | 61 | @CommandLine.Parameters(index = "1", description = "URI to json file with a mapping of all staff slack users") 62 | private URI staffJsonUri; 63 | 64 | @CommandLine.Parameters(index = "2", description = "URI to directory containing slack data export(s)") 65 | private URI inputDirUri; 66 | 67 | @CommandLine.Parameters(index = "3", description = "URO to scrapbook account data csv file") 68 | private URI scrapbookAccountDataCsvUri; 69 | 70 | @CommandLine.Parameters(index = "4", description = "URI to potential leaders (club applications) csv file") 71 | private URI clubApplicationsCsvUri; 72 | 73 | @CommandLine.Parameters(index = "5", description = "URI to active clubs csv file") 74 | private URI activeClubsCsvUri; 75 | 76 | @CommandLine.Parameters(index = "6", description = "URI to pirate ship shipment csv file") 77 | private URI pirateshipShipmentCsvUri; 78 | 79 | @CommandLine.Parameters(index = "7", description = "URI to assemble registration csv file") 80 | private URI assembleRegistrationCsvUri; 81 | 82 | @CommandLine.Parameters(index = "8", description = "URI to outernet registration csv file") 83 | private URI outernetRegistrationCsvUri; 84 | 85 | @CommandLine.Parameters(index = "9", description = "URI to angelhacks registration csv file") 86 | private URI angelhacksRegistrationCsvUri; 87 | 88 | @CommandLine.Parameters(index = "10", description = "URI to blot engagements csv file") 89 | private URI blotEngagementsCsvUri; 90 | 91 | @CommandLine.Parameters(index = "11", description = "URI to sprig engagements csv file") 92 | private URI sprigEngagementsCsvUri; 93 | 94 | @CommandLine.Parameters(index = "12", description = "URI to onboard engagements csv file") 95 | private URI onboardEngagementsCsvUri; 96 | 97 | @Override 98 | public Integer call() throws Exception { 99 | System.out.println("Initializing geocoder"); 100 | Geocoder.initialize(googleGeocodingApiKey); 101 | System.out.printf("Initializing elasticsearch client (host: %s port: %d)%n", esHostname, esPort); 102 | ElasticsearchClient esClient = ESUtils.createElasticsearchClient(esHostname, esPort, esUsername, esPassword, esFingerprint); 103 | 104 | System.out.printf("Clearing the \"%s\" index%n", esIndex); 105 | clearIndex(esClient, esIndex); 106 | 107 | final long startTimeMs = System.currentTimeMillis(); 108 | Thread monitoringThread = startMonitoringThread(); 109 | 110 | GlobalData.staffUserIds = loadStaffUsers(BlobStore.load(staffJsonUri)); 111 | loadUsers(BlobStore.load(inputUsersUri)); 112 | 113 | System.out.println("AGGREGATION PHASE"); 114 | //aggregate(); 115 | 116 | System.out.printf("Ignored user accounts: %s", String.join(", ", ChannelEvent.ignoredAccounts.keySet())); 117 | 118 | System.out.println("CONFLATION PHASE"); 119 | conflate(); 120 | System.out.println("POST-PROCESSING PHASE"); 121 | postProcess(); 122 | System.out.println("UPLOAD PHASE"); 123 | writeUsersToES(esClient); 124 | 125 | final long totalTimeMs = System.currentTimeMillis() - startTimeMs; 126 | System.out.printf("Total time in seconds: %.02f\n", totalTimeMs / 1000.0f); 127 | running = false; 128 | monitoringThread.join(); 129 | System.out.println("Monitoring thread complete!"); 130 | Geocoder.shutdown(); 131 | return 0; 132 | } 133 | 134 | private static void postProcess() { 135 | HackClubUser.getAllUsers().forEach((userId, hackClubUser) -> hackClubUser.finish()); 136 | } 137 | 138 | private Map loadOperationsInfo() { 139 | return Collections.emptyMap(); 140 | } 141 | 142 | private Stream aggregate() throws Exception { 143 | Stream days = getDayStream(BlobStore.load(inputDirUri)); 144 | Stream dayEntries = days 145 | //.limit(500) 146 | //.filter(day -> day.getLocalDate().isAfter(LocalDate.now().minusMonths(12))) 147 | .flatMap(day -> day.getEntries(false)) 148 | .filter(de -> de.getUser() != null) 149 | .peek(ChannelEvent::tokenize) 150 | .peek(ChannelEvent::onComplete); 151 | 152 | EntityDataExtractor channelMessageCountWithStaffExtractor = entity -> (double) entity.getEntries(false).count(); 153 | EntityDataExtractor uniqueUsersCountWithStaffExtractor = entity -> (double)entity.getEntries(false) 154 | .map(ChannelEvent::getUser).distinct().count(); 155 | EntityDataExtractor uniqueUsersCountWithoutStaffExtractor = entity -> (double) entity.getEntries(true) 156 | .map(ChannelEvent::getUser).distinct().count(); 157 | EntityDataExtractor> keywordsExtractor = ChannelEvent::getTokenOccurrences; 158 | 159 | EntityProcessor channelMessageCountsWithStaffByDay = new EntityProcessor<>(new DoubleAggregator<>(day -> String.format("%d_%d_%d_%s", day.getYear(), day.getMonth(), day.getDay(), day.getChannelName())), channelMessageCountWithStaffExtractor); 160 | EntityProcessor channelMessageCountsWithStaffByMonth = new EntityProcessor<>(new DoubleAggregator<>(day -> String.format("%d_%d_%s", day.getYear(), day.getMonth(), day.getChannelName())), channelMessageCountWithStaffExtractor); 161 | EntityProcessor channelMessageCountsWithStaffByYear = new EntityProcessor<>(new DoubleAggregator<>(day -> String.format("%d_%s", day.getYear(), day.getChannelName())), channelMessageCountWithStaffExtractor); 162 | EntityProcessor channelMessageCountsWithStaffByAlltime = new EntityProcessor<>(new DoubleAggregator<>(day -> String.format("%s", day.getChannelName())), channelMessageCountWithStaffExtractor); 163 | EntityProcessor channelUniqueUserCountsByDay = new EntityProcessor<>(new DoubleAggregator<>(day -> String.format("%d_%d_%d_%s", day.getYear(), day.getMonth(), day.getDay(), day.getChannelName())), uniqueUsersCountWithStaffExtractor); 164 | EntityProcessor channelUniqueUserCountsNonStaffByDay = new EntityProcessor<>(new DoubleAggregator<>(day -> String.format("%d_%d_%d_%s", day.getYear(), day.getMonth(), day.getDay(), day.getChannelName())), uniqueUsersCountWithoutStaffExtractor); 165 | EntityProcessor channelUniqueUserCountsNonStaffByMonth = new EntityProcessor<>(new DoubleAggregator<>(day -> String.format("%d_%d_%s", day.getYear(), day.getMonth(), day.getChannelName())), uniqueUsersCountWithoutStaffExtractor); 166 | EntityProcessor> userKeywordAssociations = new EntityProcessor<>(keywordAggregator, keywordsExtractor); 167 | 168 | // Message counts 169 | // days = channelMessageCountsWithStaffByDay.process(days); 170 | // days = channelMessageCountsWithStaffByMonth.process(days); 171 | // days = channelMessageCountsWithStaffByYear.process(days); 172 | // days = channelMessageCountsWithStaffByAlltime.process(days); 173 | 174 | // Unique user counts 175 | // days = channelUniqueUserCountsByDay.process(days); 176 | // days = channelUniqueUserCountsNonStaffByDay.process(days); 177 | // days = channelUniqueUserCountsNonStaffByMonth.process(days); 178 | 179 | dayEntries = userKeywordAssociations.process(dayEntries); 180 | System.out.printf("Finished keyword pipeline\n", dayEntries.count()); 181 | 182 | return days; 183 | } 184 | 185 | private ArrayList loadShipmentInfo() throws IOException, CsvValidationException { 186 | CSVReader reader = new CSVReader(new FileReader(BlobStore.load(pirateshipShipmentCsvUri))); 187 | String [] nextLine; 188 | 189 | String[] columns = "Created Date,Recipient,Company,Email,Tracking Number,Cost,Status,Batch,Label Size,Saved Package,Ship From,Ship Date,Estimated Delivery Time,Weight (oz),Zone,Package Type,Package Length,Package Width,Package Height,Tracking Status,Tracking Info,Tracking Date,Address Line 1,Address Line 2,City,State,Zipcode,Country,Carrier,Service,Order ID,Rubber Stamp 1,Rubber Stamp 2,Rubber Stamp 3,Order Value".split(","); 190 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 191 | 192 | ArrayList pirateShipEntries = new ArrayList<>(); 193 | while ((nextLine = reader.readNext()) != null) 194 | { 195 | pirateShipEntries.add(PirateShipEntry.fromCsv(nextLine, columnIndices)); 196 | } 197 | return pirateShipEntries; 198 | } 199 | 200 | public Stream getDayStream(File dir) throws Exception { 201 | File[] files = dir.listFiles(); 202 | if (files == null) { 203 | throw new RuntimeException("Whoa - no files there"); 204 | } 205 | 206 | return getAllAbsoluteFilePathsInDirectory(new ArrayList<>(), files) 207 | .parallel() 208 | .flatMap(this::processDayFile); 209 | } 210 | 211 | private static Thread startMonitoringThread() { 212 | Thread t = new Thread(() -> { 213 | while (running) { 214 | try { 215 | Thread.sleep(1000); 216 | //System.out.println("Number: " + counter.get()); 217 | } catch (InterruptedException e) { 218 | } 219 | } 220 | }); 221 | t.start(); 222 | 223 | return t; 224 | } 225 | 226 | public static Stream getAllAbsoluteFilePathsInDirectory(ArrayList paths, File[] files) { 227 | for (File file : files) { 228 | if (file.isDirectory()) { 229 | getAllAbsoluteFilePathsInDirectory(paths, file.listFiles()); // Calls same method again. 230 | } else { 231 | paths.add(file.getAbsolutePath()); 232 | } 233 | } 234 | return paths.stream(); 235 | } 236 | 237 | private Stream processDayFile(String filePath) { 238 | Reader reader = null; 239 | try { 240 | // create Gson instance 241 | Gson gson = new Gson(); 242 | 243 | // create a reader 244 | reader = Files.newBufferedReader(Paths.get(filePath)); 245 | 246 | // convert a JSON string to a clubs.pojo.User object 247 | List entries = gson.fromJson(reader, new TypeToken>() { 248 | }.getType()); 249 | counter.addAndGet(entries.size()); 250 | return Stream.of(new ChannelDay(filePath, entries.toArray(new ChannelEvent[0]))); 251 | } catch (Throwable t) { 252 | return Stream.empty(); 253 | } finally { 254 | // close reader 255 | if (reader != null) { 256 | try { 257 | reader.close(); 258 | } catch (IOException e) { 259 | } 260 | } 261 | } 262 | } 263 | 264 | private HashMap loadUsers(File userCsv) throws IOException, CsvValidationException { 265 | CSVReader reader = new CSVReader(new FileReader(userCsv)); 266 | String [] nextLine; 267 | 268 | HashMap allUsers = new HashMap<>(); 269 | while ((nextLine = reader.readNext()) != null) 270 | { 271 | HackClubUser hackClubUser = HackClubUser.fromSlackCsv(nextLine); 272 | hackClubUser.setStaff(GlobalData.staffUserIds.contains(hackClubUser.getSlackUserId())); 273 | allUsers.put(hackClubUser.getSlackUserId(), hackClubUser); 274 | } 275 | return allUsers; 276 | } 277 | 278 | private HashMap loadScrapbookAccounts() throws IOException, CsvValidationException { 279 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(scrapbookAccountDataCsvUri))) 280 | .withSkipLines(1) 281 | .build(); 282 | 283 | String[] columns = "lastusernameupdatedtime,github,website,timezone,displaystreak,avatar,webhookurl,customdomain,newmember,fullslackmember,cssurl,timezoneoffset,slackid,pronouns,streakcount,maxstreaks,customaudiourl,streakstoggledoff,webring,email,username,_airbyte_ab_id,_airbyte_emitted_at,_airbyte_normalized_at,_airbyte_synced_scrapbook_accounts_hashid".split(","); 284 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 285 | 286 | String [] nextLine; 287 | 288 | HashMap allScrapbookAccounts = new HashMap<>(); 289 | while ((nextLine = reader.readNext()) != null) 290 | { 291 | ScrapbookAccount account = ScrapbookAccount.fromCsv(nextLine, columnIndices); 292 | if (account.getSlackId().length() > 0) 293 | allScrapbookAccounts.put(account.getSlackId(), account); 294 | } 295 | return allScrapbookAccounts; 296 | } 297 | 298 | private HashMap loadClubLeaderApplicationInfoByEmail() throws IOException, CsvValidationException { 299 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(clubApplicationsCsvUri))) 300 | .withSkipLines(1) 301 | .build(); 302 | 303 | String[] columns = "ID,Application,Email,Logins,Application ID,Log In Path,Completed,Full Name,Birthday,School Year,Code,Phone,Address,Address Line 1,Address Line 2,Address City,Address State,Address Zip,Address Country,Address Formatted,Gender,Ethnicity,Website,Twitter,GitHub,Other,Hacker Story,Achievement,Technicality,Accepted Tokens,New Fact,Clubs Dashboard,Birth Year,Turnover ID,Turnover Invite?,Turnover".split(","); 304 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 305 | 306 | String [] nextLine; 307 | 308 | HashMap allLeaderApplications = new HashMap<>(); 309 | while ((nextLine = reader.readNext()) != null) 310 | { 311 | ClubLeaderApplicationInfo leader = ClubLeaderApplicationInfo.fromCsv(nextLine, columnIndices); 312 | if (leader.getEmail().length() > 0) 313 | allLeaderApplications.put(leader.getEmail(), leader); 314 | } 315 | return allLeaderApplications; 316 | } 317 | 318 | private void conflate() throws CsvValidationException, IOException { 319 | System.out.println("Loading slack keyword counts..."); 320 | Map> userKeywordCounts = Collections.emptyMap(); 321 | System.out.println("Loading scrapbook data..."); 322 | Map userScrapbookData = loadScrapbookAccounts(); 323 | System.out.println("Loading clubs data..."); 324 | Map userClubInfo = loadAllUsersClubInfo(); 325 | System.out.println("Loading clubs application data..."); 326 | Map userClubLeaderApplicationInfo = loadClubLeaderApplicationInfoByEmail(); 327 | 328 | System.out.println("Conflating keywords, scrapbook, and leader data..."); 329 | HackClubUser.getAllUsers().forEach((userId, hackClubUser) -> { 330 | hackClubUser.setKeywords(userKeywordCounts.getOrDefault(userId, Collections.emptyMap())); 331 | hackClubUser.setScrapbookAccount(Optional.ofNullable(userScrapbookData.getOrDefault(userId, null))); 332 | hackClubUser.setLeaderInfo(Optional.ofNullable(userClubInfo.get(hackClubUser)), Optional.ofNullable(userClubLeaderApplicationInfo.getOrDefault(hackClubUser.getEmail(), null))); 333 | }); 334 | 335 | System.out.println("Conflating event data..."); 336 | conflateEventData(); 337 | System.out.println("Conflating engagement data..."); 338 | conflateEngagementData(); 339 | System.out.println("Conflating slack data..."); 340 | conflateSlackData(); 341 | System.out.println("Conflating github data..."); 342 | conflateGithubData(); 343 | } 344 | 345 | private Map loadAllUsersClubInfo() throws IOException, CsvValidationException { 346 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(activeClubsCsvUri))) 347 | .withSkipLines(1) 348 | .build(); 349 | 350 | String[] columns = "Venue,Application Link,Current Leader(s),Current Leaders' Emails,Notes,Status,Location,Slack ID,Leader Address,Address Line 1,Address Line 2,Address City,Address State,Address Zip,Address Country,Address Formatted,Last Check-In,Tier,T1-Engaged-Super,T1-Engaged,T1-Super,T1,On Bank,Latitude,Longitude,Last Outreach,Next check-In,Ambassador,Club Leaders,Prospective Leaders,Email (from Prospective Leaders),Full Name (from Prospective Leaders),Phone (from Prospective Leaders),Current Leaders' Phones,Leader Phone,Leader-Club Join,Leader Birthday,Continent".split(","); 351 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 352 | 353 | String [] nextLine; 354 | 355 | HashSet allClubs = new HashSet<>(); 356 | while ((nextLine = reader.readNext()) != null) 357 | { 358 | allClubs.add(ClubInfo.fromCsv(nextLine, columnIndices)); 359 | } 360 | 361 | Matcher clubMatcher = new Matcher<>("Slack users -> clubs", new HashSet<>(HackClubUser.getAllUsers().values()), allClubs, clubScorer); 362 | Set> results = clubMatcher.getResults(); 363 | HashMap allUsersClubInfo = new HashMap<>(); 364 | for(MatchResult result : results) { 365 | allUsersClubInfo.put(result.getFrom(), result.getTo()); 366 | } 367 | 368 | return allUsersClubInfo; 369 | } 370 | 371 | private static MatchScorer clubScorer = new MatchScorer<>() { 372 | @Override 373 | public double score(HackClubUser from, ClubInfo to) { 374 | boolean exactSlackIdMatch = to.getSlackId() != null && StringUtils.equals(to.getSlackId(), from.getSlackUserId()); 375 | boolean hasEitherEmailOrSlackId = exactSlackIdMatch || to.hasEmail(from.getEmail()) || to.hasSlackId(from.getSlackUserId()); 376 | return hasEitherEmailOrSlackId ? 1.0f : 0.0f; 377 | } 378 | 379 | @Override 380 | public double getThreshold() { 381 | return 0.99; 382 | } 383 | }; 384 | 385 | private void conflateEngagementData() throws CsvValidationException, IOException { 386 | Blot.conflate(blotEngagementsCsvUri); 387 | Onboard.conflate(onboardEngagementsCsvUri); 388 | Sprig.conflate(sprigEngagementsCsvUri); 389 | } 390 | 391 | private void conflateEventData() throws CsvValidationException, IOException { 392 | Outernet.conflate(outernetRegistrationCsvUri); 393 | Assemble.conflate(assembleRegistrationCsvUri); 394 | Angelhacks.conflate(angelhacksRegistrationCsvUri); 395 | } 396 | 397 | private void conflateSlackData() { 398 | final AtomicLong lastReportTime = new AtomicLong(System.currentTimeMillis()); 399 | Set> allUsers = HackClubUser.getAllUsers().entrySet(); 400 | final AtomicLong totalEntries = new AtomicLong(allUsers.size()); 401 | final AtomicLong processedEntries = new AtomicLong(0); 402 | 403 | // Iterate over all users 404 | allUsers.parallelStream().forEach(entry -> { 405 | String slackId = entry.getValue().getSlackUserId(); 406 | entry.getValue().setSlackInfo(SlackUtils.getSlackInfo(slackId, slackApiKey)); 407 | processedEntries.incrementAndGet(); 408 | reportProgressIfNeeded(lastReportTime, totalEntries, processedEntries); 409 | }); 410 | } 411 | 412 | private void reportProgressIfNeeded(AtomicLong lastReportTime, AtomicLong totalEntries, AtomicLong processedEntries) { 413 | Utils.doEvery(lastReportTime, 1000, () -> { 414 | float percent = ((float) processedEntries.get() / (float) totalEntries.get()) * 100.0f; 415 | String formattedTime = getFormattedSlackTimeLeft(processedEntries.get(), totalEntries.get()); 416 | System.out.printf("%.2f%% complete (%d/%d) %s%n", percent, processedEntries.get(), totalEntries.get(), formattedTime); 417 | }); 418 | } 419 | 420 | private String getFormattedSlackTimeLeft(float current, long total) { 421 | // We take advantage of knowing that we are rate limited to 100 RPM per https://api.slack.com/methods/users.profile.get 422 | long minutesLeft = (long)((total - current) / 100.0f); 423 | return String.format("%d minutes left", minutesLeft); 424 | } 425 | 426 | private void conflateGithubData() { 427 | HackClubUser.getAllUsers().entrySet().parallelStream().forEach(entry -> { 428 | HackClubUser user = entry.getValue(); 429 | String githubUsername = user.getGithubUsername(); 430 | if (StringUtils.isNotEmpty(githubUsername)) { 431 | Github.getUserData(githubUsername, githubApiKey).ifPresent(user::setGithubInfo); 432 | } 433 | }); 434 | } 435 | 436 | private void writeUsersToES(ElasticsearchClient esClient) { 437 | // TODO: Batch writes are much faster 438 | HackClubUser.getAllUsers().forEach((userId, hackClubUser) -> { 439 | try { 440 | esClient.index(i -> i 441 | .index(esIndex) 442 | .id(hackClubUser.getRootId()) 443 | .document(hackClubUser)); 444 | } catch (Throwable t) { 445 | System.out.printf("Warning - %s%n", t.getMessage().substring(0, 50)); 446 | /* 447 | t.printStackTrace(); 448 | System.out.println(String.format("Issue writing user %s (%s) to ES - %s", userId, hackClubUser.getFullRealName(), t.getMessage())); 449 | */ 450 | } 451 | }); 452 | } 453 | 454 | private HashSet loadStaffUsers(File staffJsonFile) throws IOException { 455 | ObjectMapper mapper = new ObjectMapper(); 456 | StaffUsers pojo = mapper.readValue(staffJsonFile, StaffUsers.class); 457 | return new HashSet<>(pojo.getUsers()); 458 | } 459 | 460 | private static Aggregator> keywordAggregator = new Aggregator>() { 461 | @Override 462 | public Map add(Map v1, Map v2) { 463 | return Stream.concat(v1.entrySet().stream(), v2.entrySet().stream()) 464 | .collect(Collectors.toMap( 465 | Map.Entry::getKey, 466 | Map.Entry::getValue, 467 | Integer::sum)); 468 | } 469 | 470 | @Override 471 | public String bucket(ChannelEvent entity) { 472 | String userId = entity.getUser(); 473 | if (!HackClubUser.getAllUsers().containsKey(userId)) { 474 | return null; 475 | } 476 | return userId; 477 | } 478 | 479 | @Override 480 | public Map getDefaultValue() { 481 | return Collections.emptyMap(); 482 | } 483 | }; 484 | } 485 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/engagements/Blot.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.engagements; 2 | 3 | import com.hackclub.clubs.models.HackClubUser; 4 | import com.hackclub.clubs.models.engagements.BlotEngagement; 5 | import com.hackclub.common.Utils; 6 | import com.hackclub.common.conflation.MatchScorer; 7 | import com.hackclub.common.conflation.Matcher; 8 | import com.hackclub.common.elasticsearch.ESUtils; 9 | import com.hackclub.common.file.BlobStore; 10 | import com.opencsv.CSVReader; 11 | import com.opencsv.CSVReaderBuilder; 12 | import com.opencsv.exceptions.CsvValidationException; 13 | 14 | import java.io.FileReader; 15 | import java.io.IOException; 16 | import java.net.URI; 17 | import java.util.HashMap; 18 | import java.util.HashSet; 19 | 20 | public class Blot { 21 | private static String[] columns = "Email,Name,Address Line 1,Address Line 2,Address City,Address State,Address Country,Address Zip,Phone Number,Student Proof,Is Slack User?,Slack ID,Needs Printed Parts?,Status,Created At".split(","); 22 | public static void conflate(URI uri) throws CsvValidationException, IOException { 23 | HashSet hackClubUsers = new HashSet<>(HackClubUser.getAllUsers().values()); 24 | HashSet registrations = load(uri); 25 | Matcher matcher = new Matcher<>("Blot engagements -> Hack Clubbers", registrations, hackClubUsers, scorer); 26 | matcher.getResults().forEach(result -> result.getTo().setBlotEngagement(result.getFrom())); 27 | matcher.getUnmatchedFrom().forEach(blotEngagement -> { 28 | HackClubUser newUser = new HackClubUser("blot-" + blotEngagement.getEmail()); 29 | newUser.setBlotEngagement(blotEngagement); 30 | }); 31 | } 32 | 33 | private static HashSet load(URI uri) throws IOException, CsvValidationException { 34 | HashSet registrations = new HashSet<>(); 35 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(uri))).withSkipLines(1).build(); 36 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 37 | String [] nextLine; 38 | while ((nextLine = reader.readNext()) != null) { 39 | registrations.add(BlotEngagement.fromCsv(nextLine, columnIndices)); 40 | } 41 | return registrations; 42 | } 43 | 44 | private static MatchScorer scorer = new MatchScorer<>() { 45 | @Override 46 | public double score(BlotEngagement from, HackClubUser to) { 47 | return Utils.normalizedLevenshtein(from.getEmail(), to.getEmail(), 2); 48 | } 49 | 50 | @Override 51 | public double getThreshold() { 52 | return 0.49; 53 | } 54 | }; 55 | } 56 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/engagements/Onboard.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.engagements; 2 | 3 | import com.hackclub.clubs.models.HackClubUser; 4 | import com.hackclub.clubs.models.engagements.OnboardEngagement; 5 | import com.hackclub.common.Utils; 6 | import com.hackclub.common.conflation.MatchScorer; 7 | import com.hackclub.common.conflation.Matcher; 8 | import com.hackclub.common.elasticsearch.ESUtils; 9 | import com.hackclub.common.file.BlobStore; 10 | import com.opencsv.CSVReader; 11 | import com.opencsv.CSVReaderBuilder; 12 | import com.opencsv.exceptions.CsvValidationException; 13 | 14 | import java.io.FileReader; 15 | import java.io.IOException; 16 | import java.net.URI; 17 | import java.util.HashMap; 18 | import java.util.HashSet; 19 | 20 | public class Onboard { 21 | private static String[] columns = "Full Name,Email,Proof of High School Enrollment,GitHub handle,Country,Status,Commented on Github? ,On HCB? ,Birthdate,1st line of shipping address,Zip/Postal code of shipping address,2nd line of shipping address,City (shipping address),State,Referral category,How did you hear about OnBoard?,Created,Is this the first PCB you've made?".split(","); 22 | public static void conflate(URI uri) throws CsvValidationException, IOException { 23 | HashSet hackClubUsers = new HashSet<>(HackClubUser.getAllUsers().values()); 24 | HashSet registrations = load(uri); 25 | Matcher matcher = new Matcher<>("Onboard engagements -> Hack Clubbers", registrations, hackClubUsers, scorer); 26 | matcher.getResults().forEach(result -> result.getTo().setOnboardEngagement(result.getFrom())); 27 | matcher.getUnmatchedFrom().forEach(onboardEngagement -> { 28 | HackClubUser newUser = new HackClubUser("onboard-" + onboardEngagement.getEmail()); 29 | newUser.setOnboardEngagement(onboardEngagement); 30 | }); 31 | } 32 | 33 | private static HashSet load(URI uri) throws IOException, CsvValidationException { 34 | HashSet engagements = new HashSet<>(); 35 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(uri))).withSkipLines(1).build(); 36 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 37 | String [] nextLine; 38 | while ((nextLine = reader.readNext()) != null) { 39 | engagements.add(OnboardEngagement.fromCsv(nextLine, columnIndices)); 40 | } 41 | return engagements; 42 | } 43 | 44 | private static MatchScorer scorer = new MatchScorer<>() { 45 | @Override 46 | public double score(OnboardEngagement from, HackClubUser to) { 47 | double normalizedEmailDistance = Utils.normalizedLevenshtein(Utils.safeToLower(from.getEmail()), Utils.safeToLower(to.getEmail()), 2); 48 | if (normalizedEmailDistance > 0.49) return 1.0; 49 | double normalizedFullnameDistance = Utils.normalizedLevenshtein(Utils.safeToLower(from.getFullName()), Utils.safeToLower(to.getFullRealName()), 2); 50 | if (normalizedFullnameDistance > 0.49) 51 | return 1.0; 52 | return (normalizedFullnameDistance + normalizedEmailDistance) / 2; 53 | } 54 | 55 | @Override 56 | public double getThreshold() { 57 | return 0.49; 58 | } 59 | }; 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/engagements/Sprig.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.engagements; 2 | 3 | import com.hackclub.clubs.models.HackClubUser; 4 | import com.hackclub.clubs.models.engagements.SprigEngagement; 5 | import com.hackclub.common.Utils; 6 | import com.hackclub.common.conflation.MatchScorer; 7 | import com.hackclub.common.conflation.Matcher; 8 | import com.hackclub.common.elasticsearch.ESUtils; 9 | import com.hackclub.common.file.BlobStore; 10 | import com.opencsv.CSVReader; 11 | import com.opencsv.CSVReaderBuilder; 12 | import com.opencsv.exceptions.CsvValidationException; 13 | import org.apache.commons.lang3.StringUtils; 14 | 15 | import java.io.FileReader; 16 | import java.io.IOException; 17 | import java.net.URI; 18 | import java.util.HashMap; 19 | import java.util.HashSet; 20 | 21 | public class Sprig { 22 | private static String[] columns = "GitHub Username,Submitted AT,Pull Request,Email,Proof of Student,Birthday,Authentication ID,Name,Address line 1,Address line 2,City,State or Province,Zip,Country,Phone (optional),Hack Club Slack ID (optional),Color,In a club?,Sprig Status,Club name,Sprig seeds mailed?,How did you hear about Sprig?,Address Formatted,Status,Notes,Tracking,Carrier,Tracking Base Link,Tracking Emailed,Referral Source,Age (years)".split(","); 23 | public static void conflate(URI uri) throws CsvValidationException, IOException { 24 | HashSet hackClubUsers = new HashSet<>(HackClubUser.getAllUsers().values()); 25 | HashSet registrations = load(uri); 26 | Matcher matcher = new Matcher<>("Sprig engagements -> Hack Clubbers", registrations, hackClubUsers, scorer); 27 | 28 | System.out.printf("matches: %d unmatchedFrom: %d%n", matcher.getResults().size(), matcher.getUnmatchedFrom().size()); 29 | matcher.getResults().forEach(result -> result.getTo().setSprigEngagement(result.getFrom())); 30 | matcher.getUnmatchedFrom().forEach(sprigEngagement -> { 31 | String rootId = String.format("sprig-email-%s-slackid-%s", sprigEngagement.getEmail(), sprigEngagement.getSlackId()); 32 | HackClubUser newUser = new HackClubUser(rootId); 33 | newUser.setSprigEngagement(sprigEngagement); 34 | }); 35 | } 36 | 37 | private static HashSet load(URI uri) throws IOException, CsvValidationException { 38 | HashSet registrations = new HashSet<>(); 39 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(uri))).withSkipLines(1).build(); 40 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 41 | String [] nextLine; 42 | while ((nextLine = reader.readNext()) != null) { 43 | registrations.add(SprigEngagement.fromCsv(nextLine, columnIndices)); 44 | } 45 | return registrations; 46 | } 47 | 48 | private static MatchScorer scorer = new MatchScorer<>() { 49 | @Override 50 | public double score(SprigEngagement from, HackClubUser to) { 51 | if(to.getSlackUserId() != null && StringUtils.equals(from.getSlackId(), to.getSlackUserId())) { 52 | return 1.0f; 53 | } 54 | 55 | double emailDistance = Utils.normalizedLevenshtein(from.getEmail(), to.getEmail(), 2); 56 | if (emailDistance > getThreshold()) return emailDistance; 57 | return Utils.normalizedLevenshtein(from.getName(), to.getFullRealName(), 2); 58 | } 59 | 60 | @Override 61 | public double getThreshold() { 62 | return 0.49; 63 | } 64 | }; 65 | } 66 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/events/Angelhacks.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.events; 2 | 3 | import com.hackclub.clubs.models.HackClubUser; 4 | import com.hackclub.clubs.models.event.AngelhacksRegistration; 5 | import com.hackclub.common.Utils; 6 | import com.hackclub.common.conflation.MatchResult; 7 | import com.hackclub.common.conflation.MatchScorer; 8 | import com.hackclub.common.conflation.Matcher; 9 | import com.hackclub.common.elasticsearch.ESUtils; 10 | import com.hackclub.common.file.BlobStore; 11 | import com.opencsv.CSVReader; 12 | import com.opencsv.CSVReaderBuilder; 13 | import com.opencsv.exceptions.CsvValidationException; 14 | 15 | import java.io.FileReader; 16 | import java.io.IOException; 17 | import java.net.URI; 18 | import java.util.HashMap; 19 | import java.util.HashSet; 20 | import java.util.Set; 21 | 22 | public class Angelhacks { 23 | private static String[] columns = "Name,Email,Phone number,School,Grade,Pronouns,T-shirt size,Duration,Skill Level,Game experience,Goals,Helping,Source,Waivers Done,Added to Postal,Checked in,Checked out".split(","); 24 | public static void conflate(URI uri) throws CsvValidationException, IOException { 25 | HashSet hackClubUsers = new HashSet<>(HackClubUser.getAllUsers().values()); 26 | HashSet registrations = loadRegistrations(uri); 27 | Matcher matcher = new Matcher<>("Angelhacks attendance -> Hack Clubbers", registrations, hackClubUsers, scorer); 28 | matcher.getResults().forEach(result -> result.getTo().setAngelhacksRegistration(result.getFrom())); 29 | matcher.getUnmatchedFrom().forEach(angelhacksRegistration -> { 30 | String rootId = String.format("angelhacks-email-%s-school-%s", angelhacksRegistration.getEmail(), angelhacksRegistration.getSchool()); 31 | HackClubUser newUser = new HackClubUser(rootId); 32 | newUser.setAngelhacksRegistration(angelhacksRegistration); 33 | }); 34 | } 35 | 36 | private static HashSet loadRegistrations(URI uri) throws IOException, CsvValidationException { 37 | HashSet registrations = new HashSet<>(); 38 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(uri))).withSkipLines(1).build(); 39 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 40 | String [] nextLine; 41 | while ((nextLine = reader.readNext()) != null) { 42 | registrations.add(AngelhacksRegistration.fromCsv(nextLine, columnIndices)); 43 | } 44 | return registrations; 45 | } 46 | 47 | private static MatchScorer scorer = new MatchScorer<>() { 48 | @Override 49 | public double score(AngelhacksRegistration from, HackClubUser to) { 50 | double normalizedEmailDistance = Utils.normalizedLevenshtein(Utils.safeToLower(from.getEmail()), Utils.safeToLower(to.getEmail()), 2); 51 | if (normalizedEmailDistance > 0.49) return 1.0; 52 | double normalizedFullnameDistance = Utils.normalizedLevenshtein(Utils.safeToLower(from.getName()), Utils.safeToLower(to.getFullRealName()), 2); 53 | if (normalizedFullnameDistance > 0.49) 54 | return 1.0; 55 | return (normalizedFullnameDistance + normalizedEmailDistance) / 2; 56 | } 57 | 58 | @Override 59 | public double getThreshold() { 60 | return 0.49; 61 | } 62 | }; 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/events/Assemble.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.events; 2 | 3 | import com.hackclub.clubs.models.HackClubUser; 4 | import com.hackclub.clubs.models.event.AssembleRegistration; 5 | import com.hackclub.common.Utils; 6 | import com.hackclub.common.conflation.MatchResult; 7 | import com.hackclub.common.conflation.MatchScorer; 8 | import com.hackclub.common.conflation.Matcher; 9 | import com.hackclub.common.elasticsearch.ESUtils; 10 | import com.hackclub.common.file.BlobStore; 11 | import com.opencsv.CSVReader; 12 | import com.opencsv.CSVReaderBuilder; 13 | import com.opencsv.exceptions.CsvValidationException; 14 | 15 | import java.io.FileReader; 16 | import java.io.IOException; 17 | import java.net.URI; 18 | import java.util.HashMap; 19 | import java.util.HashSet; 20 | import java.util.Set; 21 | 22 | public class Assemble { 23 | private static String[] columns = "//ID,Email,Log In Path,Full Name,Your Nearest Airport,Birthday,Vaccinated?,\"If you're not vaccinated, please explain why:\",Do you require a letter for visa applications?,Travel Stipend,Dietary Restrictions,\"At the moment, what is your estimated travel cost?\",Travel Stipend Cost INT,What would a travel stipend mean to you?,Skill Level,Would you be interested in hosting a workshop session at Assemble?,Workshop Topic,Shirt,Parent Name,Parent Email,Tabs or Spaces,Pineapple on Pizza,Submission Timestamp,Voted For,Team Notes,Stipend,Decision:,Follow Up,Estimated Cost(Hugo),Amount of Votes,Name (For Prefill),Follow Up (For Prefill),Vote *against*,18?,Serious Alum?,Pronouns,Password Code,Send 2 Weeks Out Email,Waiver,Freedom,Off Waitlist,Vaccinated,waiver_type,Send Wed 3 Email,Created at".split(","); 24 | public static void conflate(URI uri) throws CsvValidationException, IOException { 25 | HashSet hackClubUsers = new HashSet<>(HackClubUser.getAllUsers().values()); 26 | HashSet registrations = loadRegistrations(uri); 27 | Matcher matcher = new Matcher<>("Assemble attendance -> Hack Clubbers", registrations, hackClubUsers, scorer); 28 | matcher.getResults().forEach(result -> result.getTo().setAssembleRegistration(result.getFrom())); 29 | matcher.getUnmatchedFrom().forEach(assembleRegistration -> { 30 | String rootId = String.format("assemble-email-%s", assembleRegistration.getEmail()); 31 | HackClubUser newUser = new HackClubUser(rootId); 32 | newUser.setAssembleRegistration(assembleRegistration); 33 | }); 34 | } 35 | 36 | private static HashSet loadRegistrations(URI uri) throws IOException, CsvValidationException { 37 | HashSet registrations = new HashSet<>(); 38 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(uri))).withSkipLines(1).build(); 39 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 40 | String [] nextLine; 41 | while ((nextLine = reader.readNext()) != null) { 42 | registrations.add(AssembleRegistration.fromCsv(nextLine, columnIndices)); 43 | } 44 | return registrations; 45 | } 46 | 47 | private static MatchScorer scorer = new MatchScorer<>() { 48 | @Override 49 | public double score(AssembleRegistration from, HackClubUser to) { 50 | double normalizedEmailDistance = Utils.normalizedLevenshtein(Utils.safeToLower(from.getEmail()), Utils.safeToLower(to.getEmail()), 2); 51 | if (normalizedEmailDistance > 0.49) return 1.0; 52 | double normalizedFullnameDistance = Utils.normalizedLevenshtein(Utils.safeToLower(from.getFullName()), Utils.safeToLower(to.getFullRealName()), 2); 53 | if (normalizedFullnameDistance > 0.49) 54 | return 1.0; 55 | return (normalizedFullnameDistance + normalizedEmailDistance) / 2; 56 | } 57 | 58 | @Override 59 | public double getThreshold() { 60 | return 0.49; 61 | } 62 | }; 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/events/Outernet.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.events; 2 | 3 | import com.hackclub.clubs.models.HackClubUser; 4 | import com.hackclub.clubs.models.event.OuternetRegistration; 5 | import com.hackclub.common.Utils; 6 | import com.hackclub.common.conflation.MatchResult; 7 | import com.hackclub.common.conflation.MatchScorer; 8 | import com.hackclub.common.conflation.Matcher; 9 | import com.hackclub.common.elasticsearch.ESUtils; 10 | import com.hackclub.common.file.BlobStore; 11 | import com.opencsv.CSVReader; 12 | import com.opencsv.CSVReaderBuilder; 13 | import com.opencsv.exceptions.CsvValidationException; 14 | 15 | import java.io.FileReader; 16 | import java.io.IOException; 17 | import java.net.URI; 18 | import java.util.HashMap; 19 | import java.util.HashSet; 20 | import java.util.Set; 21 | 22 | public class Outernet { 23 | private static String[] columns = "ID,Name,Club Leader?,Workshop / Lightning Talk Focus,Email,Pronouns,Birthday,T-Shirt Size,Travel,Dietary Restrictions,Parent's Name,Parent's Email,GitHub,Example Project,Curiosity,Guild Interest,Guild Focus,ranking,Workshop / Lightning Talk Interest,Stipend Record,Cool ideas,Shuttle Record,migration,workshop status,Waiver Sent,Created,Stipend Approved,Pod,Checked In?,Notes,Contact's Phone number,Checked out,Accepted Stipends".split(","); 24 | public static void conflate(URI uri) throws CsvValidationException, IOException { 25 | HashSet hackClubUsers = new HashSet<>(HackClubUser.getAllUsers().values()); 26 | HashSet registrations = loadRegistrations(uri); 27 | Matcher matcher = new Matcher<>("Outernet registrations -> Hack Clubbers", registrations, hackClubUsers, scorer); 28 | matcher.getResults().forEach(result -> result.getTo().setOuternetRegistration(result.getFrom())); 29 | matcher.getUnmatchedFrom().forEach(outernetRegistration -> { 30 | String rootId = String.format("outernet-email-%s", outernetRegistration.getEmail()); 31 | HackClubUser newUser = new HackClubUser(rootId); 32 | newUser.setOuternetRegistration(outernetRegistration); 33 | }); 34 | } 35 | 36 | private static HashSet loadRegistrations(URI uri) throws IOException, CsvValidationException { 37 | HashSet registrations = new HashSet<>(); 38 | CSVReader reader = new CSVReaderBuilder(new FileReader(BlobStore.load(uri))).withSkipLines(1).build(); 39 | HashMap columnIndices = ESUtils.getIndexMapping(columns); 40 | String [] nextLine; 41 | while ((nextLine = reader.readNext()) != null) { 42 | registrations.add(OuternetRegistration.fromCsv(nextLine, columnIndices)); 43 | } 44 | return registrations; 45 | } 46 | 47 | private static MatchScorer scorer = new MatchScorer<>() { 48 | @Override 49 | public double score(OuternetRegistration from, HackClubUser to) { 50 | double normalizedEmailDistance = Utils.normalizedLevenshtein(Utils.safeToLower(from.getEmail()), Utils.safeToLower(to.getEmail()), 2); 51 | if (normalizedEmailDistance > 0.49) return 1.0; 52 | double normalizedFullnameDistance = Utils.normalizedLevenshtein(Utils.safeToLower(from.getName()), Utils.safeToLower(to.getFullRealName()), 2); 53 | if (normalizedFullnameDistance > 0.49) 54 | return 1.0; 55 | return (normalizedFullnameDistance + normalizedEmailDistance) / 2; 56 | } 57 | 58 | @Override 59 | public double getThreshold() { 60 | return 0.49; 61 | } 62 | }; 63 | } 64 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/github/Github.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.github; 2 | 3 | import com.hackclub.clubs.models.GithubInfo; 4 | import com.hackclub.clubs.models.SlackInfo; 5 | import com.fasterxml.jackson.core.JsonProcessingException; 6 | import com.fasterxml.jackson.databind.ObjectMapper; 7 | import com.jayway.jsonpath.DocumentContext; 8 | import com.jayway.jsonpath.JsonPath; 9 | import com.hackclub.common.Utils; 10 | import com.hackclub.common.file.Cache; 11 | import org.apache.commons.text.StringEscapeUtils; 12 | import org.xml.sax.SAXException; 13 | 14 | import javax.xml.parsers.ParserConfigurationException; 15 | import javax.xml.xpath.XPathExpressionException; 16 | import java.io.IOException; 17 | import java.net.URI; 18 | import java.net.http.HttpClient; 19 | import java.net.http.HttpRequest; 20 | import java.net.http.HttpResponse; 21 | import java.time.Duration; 22 | import java.util.HashMap; 23 | import java.util.List; 24 | import java.util.Optional; 25 | 26 | /** 27 | * Integrates with Github GraphQL APIs to retrieve data relating to PRs, commits, and other metadata 28 | */ 29 | public class Github { 30 | private static String userQueryTemplate; 31 | static { 32 | try { 33 | userQueryTemplate = Utils.getResourceFileAsString("github_user_query.graphql"); 34 | } catch (IOException e) { 35 | throw new RuntimeException(e); 36 | } 37 | } 38 | 39 | public static Optional getUserData(String username, String githubApiKey) { 40 | if (username == null) { 41 | //System.out.println("No username, skipping github data"); 42 | return Optional.empty(); 43 | } 44 | 45 | String cacheKey = username + "-github"; 46 | Optional cachedGithubInfo = loadFromCache(cacheKey); 47 | if (cachedGithubInfo.isPresent()) { 48 | cachedGithubInfo.get().setId(String.format("https://github.com/%s", username)); 49 | return cachedGithubInfo; 50 | } 51 | 52 | HttpResponse response = null; 53 | try { 54 | String template = "{\"query\": \"%s\"}"; 55 | String query = String.format(template, StringEscapeUtils.escapeJson(String.format(userQueryTemplate, username))); 56 | 57 | HttpClient client = HttpClient.newBuilder() 58 | .version(HttpClient.Version.HTTP_1_1) 59 | .followRedirects(HttpClient.Redirect.NORMAL) 60 | .connectTimeout(Duration.ofSeconds(20)) 61 | .build(); 62 | 63 | HttpRequest request = HttpRequest.newBuilder() 64 | .uri(URI.create("https://api.github.com/graphql")) 65 | .timeout(Duration.ofMinutes(2)) 66 | .header("Authorization", "bearer " + githubApiKey) 67 | .header("Content-Type", "application/json") 68 | .POST(HttpRequest.BodyPublishers.ofString(query)) 69 | .build(); 70 | 71 | response = client.send(request, HttpResponse.BodyHandlers.ofString()); 72 | 73 | Optional ghInfo = parseUserData(Utils.getPrettyJson(response.body())); 74 | cache(cacheKey, ghInfo); 75 | ghInfo.ifPresent(githubInfo -> githubInfo.setId(String.format("https://github.com/%s", username))); 76 | return ghInfo; 77 | } catch (Throwable t) { 78 | t.printStackTrace(); 79 | System.out.println("Failed to load github user data"); 80 | return Optional.empty(); 81 | } 82 | } 83 | 84 | private static Optional parseUserData(String data) throws ParserConfigurationException, IOException, SAXException, XPathExpressionException { 85 | DocumentContext root = JsonPath.parse(data); 86 | 87 | try { 88 | String userInfoPath = "$.data.search.edges[0].node"; 89 | HashMap userData = root.read(userInfoPath); 90 | if (userData.isEmpty()) return Optional.empty(); 91 | } catch (Throwable t) { 92 | // no users returned 93 | return Optional.empty(); 94 | } 95 | 96 | String topRepositoriesPath = "$.data.search.edges[0].node.topRepositories.edges[*].node"; 97 | List> topRepositories = root.read(topRepositoriesPath); 98 | 99 | String pullRequestPath = "$.data.search.edges[0].node.contributionsCollection.pullRequestContributions.nodes[*].pullRequest"; 100 | List> pullRequests = root.read(pullRequestPath); 101 | 102 | GithubInfo info = new GithubInfo(); 103 | HashMap prCountsByRepo = new HashMap<>(); 104 | HashMap prCountsByLanguage = new HashMap<>(); 105 | HashMap prCountsByOwner = new HashMap<>(); 106 | pullRequests.forEach(pr -> { 107 | try { 108 | HashMap repoInfo = (HashMap) pr.get("repository"); 109 | HashMap ownerInfo = (HashMap) repoInfo.get("owner"); 110 | String ownerName = (String) ownerInfo.get("login"); 111 | String repoName = (String) repoInfo.get("name"); 112 | HashMap primaryLanguageInfo = (HashMap) repoInfo.getOrDefault("primaryLanguage", "unknown"); 113 | String primaryLanguage = "unknown"; 114 | if (primaryLanguageInfo != null) { 115 | primaryLanguage = (String)primaryLanguageInfo.getOrDefault("name", "unknown"); 116 | } 117 | incrementKeyword(prCountsByRepo, String.format("%s/%s", ownerName, repoName)); 118 | incrementKeyword(prCountsByLanguage, primaryLanguage); 119 | incrementKeyword(prCountsByOwner, ownerName); 120 | //System.out.printf("owner: %s repo: %s language: %s\n", ownerName, repoName, primaryLanguage); 121 | } catch (Throwable t) { 122 | t.printStackTrace(); 123 | } 124 | }); 125 | 126 | info.setPullRequestCountsByLanguage(prCountsByLanguage); 127 | info.setPullRequestCountsByOwner(prCountsByOwner); 128 | info.setPullRequestCountsByRepo(prCountsByRepo); 129 | 130 | return Optional.of(info); 131 | } 132 | 133 | private static void incrementKeyword(HashMap countMapping, String key) { 134 | int currentCount = countMapping.getOrDefault(key, 0); 135 | countMapping.put(key, currentCount+1); 136 | } 137 | 138 | private static void cache(String key, Optional ret) { 139 | try { 140 | if (ret.isPresent()) { 141 | Cache.save(key, new ObjectMapper().writeValueAsString(ret.get())); 142 | } else { 143 | Cache.save(key, new ObjectMapper().writeValueAsString("")); 144 | } 145 | } catch (Throwable t) { 146 | t.printStackTrace(); 147 | } 148 | } 149 | 150 | private static Optional loadFromCache(String key) { 151 | Optional githubData = Cache.load(key); 152 | if (githubData.isPresent()) { 153 | if (githubData.get().length() == 2) { 154 | return Optional.empty(); 155 | } 156 | try { 157 | return Optional.of(new ObjectMapper().readValue(githubData.get(), GithubInfo.class)); 158 | } catch (JsonProcessingException e) { 159 | e.printStackTrace(); 160 | } 161 | } 162 | return Optional.empty(); 163 | } 164 | 165 | } 166 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/ChannelDay.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import com.hackclub.clubs.GlobalData; 4 | 5 | import java.time.LocalDate; 6 | import java.util.Arrays; 7 | import java.util.stream.Stream; 8 | 9 | /** 10 | * Model object that represents one day of channel data 11 | */ 12 | public class ChannelDay { 13 | private ChannelEvent[] entries; 14 | private String filename; 15 | private String channelName; 16 | private Integer month; 17 | private Integer day; 18 | private Integer year; 19 | public String getChannelName() { 20 | return channelName; 21 | } 22 | 23 | public Integer getMonth() { 24 | return month; 25 | } 26 | 27 | public Integer getDay() { 28 | return day; 29 | } 30 | 31 | public Integer getYear() { 32 | return year; 33 | } 34 | 35 | @Override 36 | public String toString() { 37 | return "clubs.pojo.Day{" + 38 | ", channelName='" + channelName + '\'' + 39 | ", month=" + month + 40 | ", day=" + day + 41 | ", year=" + year + 42 | ", numChats=" + entries.length + 43 | '}'; 44 | } 45 | 46 | public ChannelDay(String filename, ChannelEvent[] entries) { 47 | this.filename = filename; 48 | 49 | String[] parts = filename.split("/"); 50 | if (parts.length < 1) throw new RuntimeException("Whoa something is weird here"); 51 | 52 | String localFilename = parts[parts.length-1]; 53 | this.channelName = parts[parts.length-2]; 54 | 55 | if (!localFilename.endsWith(".json")) 56 | throw new RuntimeException("Whoa"); 57 | String prefix = localFilename.split("\\.")[0]; 58 | String[] dateComponents = prefix.split("-"); 59 | this.year = Integer.parseInt(dateComponents[0]); 60 | this.month = Integer.parseInt(dateComponents[1]); 61 | this.day = Integer.parseInt(dateComponents[2]); 62 | 63 | setEntries(entries); 64 | } 65 | public Stream getEntries(boolean excludeStaff) { 66 | Stream data = Stream.of(entries); 67 | 68 | if (excludeStaff) { 69 | data = data.filter(entry -> !GlobalData.staffUserIds.contains(entry.getUser())); 70 | } 71 | 72 | return data; 73 | } 74 | 75 | public void setEntries(ChannelEvent[] entries) { 76 | this.entries = entries; 77 | for(ChannelEvent entry : this.entries) { 78 | entry.setParent(this); 79 | } 80 | } 81 | 82 | public String getFilename() { 83 | return filename; 84 | } 85 | 86 | public void setFilename(String filename) { 87 | this.filename = filename; 88 | } 89 | 90 | Stream allPostingUsers() { 91 | return Arrays.stream(entries).flatMap(entry -> HackClubUser.getWithRootId(entry.getUser()).stream()); 92 | } 93 | 94 | public LocalDate getLocalDate() { 95 | final String dayStr = String.format("%d-%02d-%02d", getYear(), getMonth(), getDay()); 96 | return LocalDate.parse(dayStr); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/ChannelEvent.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import com.hackclub.clubs.GlobalData; 4 | 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.concurrent.ConcurrentHashMap; 8 | import java.util.stream.Stream; 9 | 10 | /** 11 | * Model object for events that occur in a channel, such as slack messages, emojis, etc 12 | */ 13 | public class ChannelEvent { 14 | private String type; 15 | private String subType; 16 | private String user; 17 | private String text; 18 | private ChannelDay parent; 19 | private HashMap tokenOccurrences = new HashMap<>(); 20 | public static ConcurrentHashMap ignoredAccounts = new ConcurrentHashMap<>(); 21 | 22 | @Override 23 | public String toString() { 24 | return "clubs.pojo.DayEntry{" + 25 | "type='" + type + '\'' + 26 | ", subType='" + subType + '\'' + 27 | ", user='" + user + '\'' + 28 | ", text='" + text + '\'' + 29 | '}'; 30 | } 31 | 32 | public String getType() { 33 | return type; 34 | } 35 | 36 | public void setType(String type) { 37 | this.type = type; 38 | } 39 | 40 | public String getSubType() { 41 | return subType; 42 | } 43 | 44 | public void setSubType(String subType) { 45 | this.subType = subType; 46 | } 47 | 48 | public String getUser() { 49 | return user; 50 | } 51 | 52 | public void setUser(String user) { 53 | this.user = user; 54 | } 55 | 56 | public String getText() { 57 | return text; 58 | } 59 | 60 | public void setText(String text) { 61 | this.text = text; 62 | } 63 | 64 | public void tokenize() { 65 | createTokens(text); 66 | } 67 | 68 | public void onComplete() { 69 | if (!HackClubUser.getAllUsers().containsKey(user)) { 70 | ignoredAccounts.put(user, true); 71 | return; 72 | } 73 | 74 | HackClubUser.getAllUsers().get(user).onSlackChatMessageProcessed(this); 75 | } 76 | 77 | private void createTokens(String text) { 78 | Stream.of(text.split(" ")).forEach(potentialToken -> { 79 | potentialToken = potentialToken.toLowerCase(); 80 | if (GlobalData.validTokens.contains(clean(potentialToken))) { 81 | if (!tokenOccurrences.containsKey(potentialToken)) { 82 | tokenOccurrences.put(potentialToken, 1); 83 | } else { 84 | tokenOccurrences.put(potentialToken, tokenOccurrences.get(potentialToken) + 1); 85 | } 86 | } 87 | }); 88 | } 89 | 90 | public Map getTokenOccurrences() { 91 | return tokenOccurrences; 92 | } 93 | 94 | public String clean(String str) { 95 | return str.toLowerCase(); 96 | } 97 | 98 | public ChannelDay getParent() { 99 | return parent; 100 | } 101 | 102 | public void setParent(ChannelDay parent) { 103 | this.parent = parent; 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/ClubInfo.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import java.util.Arrays; 4 | import java.util.HashMap; 5 | import java.util.stream.Collectors; 6 | import java.util.stream.Stream; 7 | 8 | public class ClubInfo { 9 | private String slackId; 10 | private String leaderSlackIds; 11 | private String leaderEmails; 12 | private String clubAddress; 13 | private String tier; 14 | private String notes; 15 | private String applicationLink; 16 | private String venue; 17 | private String lastCheckIn; 18 | private String status; 19 | 20 | // Not publicly instantiable - use factory methods (fromCsv) 21 | private ClubInfo() { 22 | } 23 | 24 | // Venue,Application Link,Current Leader(s),Current Leaders' Emails,Notes,Status,Location,Slack ID,Leader Address,Address Line 1,Address Line 2,Address City,Address State,Address Zip,Address Country,Address Formatted,Last Check-In,Tier,T1-Engaged-Super,T1-Engaged,T1-Super,T1,On Bank,Latitude,Longitude,Last Outreach,Next check-In,Ambassador,Club Leaders,Prospective Leaders,Email (from Prospective Leaders),Full Name (from Prospective Leaders),Phone (from Prospective Leaders),Current Leaders' Phones,Leader Phone,Leader-Club Join,Leader Birthday,Continent 25 | public static ClubInfo fromCsv(String[] nextLine, HashMap columnIndices) { 26 | ClubInfo ci = new ClubInfo(); 27 | 28 | ci.slackId = nextLine[columnIndices.get("Slack ID")]; 29 | ci.leaderEmails = nextLine[columnIndices.get("Current Leaders' Emails")]; 30 | ci.leaderSlackIds = nextLine[columnIndices.get("Slack ID")]; 31 | ci.clubAddress = nextLine[columnIndices.get("Address Formatted")]; 32 | ci.lastCheckIn = nextLine[columnIndices.get("Last Check-In")]; 33 | ci.tier = nextLine[columnIndices.get("Tier")]; 34 | ci.notes = nextLine[columnIndices.get("Notes")]; 35 | ci.applicationLink = nextLine[columnIndices.get("Application Link")]; 36 | ci.status = nextLine[columnIndices.get("Status")]; 37 | ci.venue = nextLine[columnIndices.get("Venue")]; 38 | 39 | return ci; 40 | } 41 | 42 | public boolean hasEmail(String email) { 43 | return getLeaderEmails().collect(Collectors.toSet()).contains(email); 44 | } 45 | 46 | public boolean hasSlackId(String slackId) { 47 | return getLeaderSlackIds().collect(Collectors.toSet()).contains(slackId); 48 | } 49 | 50 | public Stream getLeaderEmails() { 51 | return Arrays.stream(leaderEmails.replaceAll("\\s", "").split(",")).distinct(); 52 | } 53 | 54 | public Stream getLeaderSlackIds() { 55 | return Arrays.stream(leaderSlackIds.replaceAll("\\s", "").split(",")).distinct(); 56 | } 57 | 58 | public void setLeaderSlackIds(String leaderSlackIds) { 59 | this.leaderSlackIds = leaderSlackIds; 60 | } 61 | 62 | public void setLeaderEmails(String leaderEmails) { 63 | this.leaderEmails = leaderEmails; 64 | } 65 | 66 | public String getClubAddress() { 67 | return clubAddress; 68 | } 69 | 70 | public void setClubAddress(String clubAddress) { 71 | this.clubAddress = clubAddress; 72 | } 73 | 74 | public String getTier() { 75 | return tier; 76 | } 77 | 78 | public void setTier(String tier) { 79 | this.tier = tier; 80 | } 81 | 82 | public String getNotes() { 83 | return notes; 84 | } 85 | 86 | public void setNotes(String notes) { 87 | this.notes = notes; 88 | } 89 | 90 | public String getApplicationLink() { 91 | return applicationLink; 92 | } 93 | 94 | public void setApplicationLink(String applicationLink) { 95 | this.applicationLink = applicationLink; 96 | } 97 | 98 | public String getVenue() { 99 | return venue; 100 | } 101 | 102 | public void setVenue(String venue) { 103 | this.venue = venue; 104 | } 105 | 106 | public String getLastCheckIn() { 107 | return lastCheckIn; 108 | } 109 | 110 | public void setLastCheckIn(String lastCheckIn) { 111 | this.lastCheckIn = lastCheckIn; 112 | } 113 | 114 | public String getStatus() { 115 | return status; 116 | } 117 | 118 | public void setStatus(String status) { 119 | this.status = status; 120 | } 121 | 122 | public String getSlackId() { 123 | return slackId; 124 | } 125 | 126 | public void setSlackId(String slackId) { 127 | this.slackId = slackId; 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/ClubLeaderApplicationInfo.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | import java.util.HashMap; 6 | 7 | /** 8 | * Model object representing information that we know about Club Leaders 9 | */ 10 | public class ClubLeaderApplicationInfo { 11 | private String email; 12 | private boolean isOrWasLeader; 13 | private String fullName; 14 | private String birthday; 15 | private String schoolYear; 16 | private String phoneNumber; 17 | private String address; 18 | private String country; 19 | private String gender; 20 | private String ethnicity; 21 | private String prettyAddress; 22 | private String twitter; 23 | private String github; 24 | private Integer birthYear; 25 | 26 | // Not publicly instantiable - use factory methods (fromCsv) 27 | private ClubLeaderApplicationInfo() { 28 | } 29 | 30 | // ID,Application,Email,Logins,Application ID,Log In Path,Completed,Full Name,Birthday,School Year,Code,Phone,Address,Address Line 1,Address Line 2,Address City,Address State,Address Zip,Address Country,Address Formatted,Gender,Ethnicity,Website,Twitter,GitHub,Other,Hacker Story,Achievement,Technicality,Accepted Tokens,New Fact,Clubs Dashboard,Birth Year,Turnover ID,Turnover Invite?,Turnover 31 | public static ClubLeaderApplicationInfo fromCsv(String[] nextLine, HashMap columnIndices) { 32 | ClubLeaderApplicationInfo cli = new ClubLeaderApplicationInfo(); 33 | 34 | cli.email = nextLine[columnIndices.get("Email")]; 35 | cli.isOrWasLeader = nextLine[columnIndices.get("Completed")].equals("checked"); 36 | cli.fullName = nextLine[columnIndices.get("Full Name")]; 37 | cli.birthday = nextLine[columnIndices.get("Birthday")]; 38 | cli.schoolYear = nextLine[columnIndices.get("School Year")]; 39 | cli.phoneNumber = nextLine[columnIndices.get("Phone")]; 40 | cli.address = nextLine[columnIndices.get("Address")]; 41 | cli.country = nextLine[columnIndices.get("Address Country")]; 42 | cli.gender = nextLine[columnIndices.get("Gender")]; 43 | cli.ethnicity = nextLine[columnIndices.get("Ethnicity")]; 44 | cli.prettyAddress = nextLine[columnIndices.get("Address Formatted")]; 45 | cli.twitter = nextLine[columnIndices.get("Twitter")]; 46 | cli.github = nextLine[columnIndices.get("GitHub")]; 47 | String rawBirthYear = nextLine[columnIndices.get("Birth Year")]; 48 | cli.birthYear = (StringUtils.isEmpty(rawBirthYear) || rawBirthYear.equals("NaN")) ? null : Integer.parseInt(rawBirthYear); 49 | 50 | return cli; 51 | } 52 | 53 | public String getEmail() { 54 | return email; 55 | } 56 | 57 | public void setEmail(String email) { 58 | this.email = email; 59 | } 60 | 61 | public boolean isOrWasLeader() { 62 | return isOrWasLeader; 63 | } 64 | 65 | public void setOrWasLeader(boolean orWasLeader) { 66 | isOrWasLeader = orWasLeader; 67 | } 68 | 69 | public String getFullName() { 70 | return fullName; 71 | } 72 | 73 | public void setFullName(String fullName) { 74 | this.fullName = fullName; 75 | } 76 | 77 | public String getBirthday() { 78 | return birthday; 79 | } 80 | 81 | public void setBirthday(String birthday) { 82 | this.birthday = birthday; 83 | } 84 | 85 | public String getSchoolYear() { 86 | return schoolYear; 87 | } 88 | 89 | public void setSchoolYear(String schoolYear) { 90 | this.schoolYear = schoolYear; 91 | } 92 | 93 | public String getPhoneNumber() { 94 | return phoneNumber; 95 | } 96 | 97 | public void setPhoneNumber(String phoneNumber) { 98 | this.phoneNumber = phoneNumber; 99 | } 100 | 101 | public String getAddress() { 102 | return address; 103 | } 104 | 105 | public void setAddress(String address) { 106 | this.address = address; 107 | } 108 | 109 | public String getCountry() { 110 | return country; 111 | } 112 | 113 | public void setCountry(String country) { 114 | this.country = country; 115 | } 116 | 117 | public String getGender() { 118 | return gender; 119 | } 120 | 121 | public void setGender(String gender) { 122 | this.gender = gender; 123 | } 124 | 125 | public String getEthnicity() { 126 | return ethnicity; 127 | } 128 | 129 | public void setEthnicity(String ethnicity) { 130 | this.ethnicity = ethnicity; 131 | } 132 | 133 | public String getPrettyAddress() { 134 | return prettyAddress; 135 | } 136 | 137 | public void setPrettyAddress(String prettyAddress) { 138 | this.prettyAddress = prettyAddress; 139 | } 140 | 141 | public String getTwitter() { 142 | return twitter; 143 | } 144 | 145 | public void setTwitter(String twitter) { 146 | this.twitter = twitter; 147 | } 148 | 149 | public String getGithub() { 150 | return github; 151 | } 152 | 153 | public void setGithub(String github) { 154 | this.github = github; 155 | } 156 | 157 | public Integer getBirthYear() { 158 | return birthYear; 159 | } 160 | 161 | public void setBirthYear(Integer birthYear) { 162 | this.birthYear = birthYear; 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/GeoPoint.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | /** 4 | * Model for geo coordinates (lat/lng) 5 | */ 6 | public class GeoPoint { 7 | private double lat; 8 | private double lon; 9 | 10 | public GeoPoint() { 11 | } 12 | 13 | public GeoPoint(double lat, double lon) { 14 | this.lat = lat; 15 | this.lon = lon; 16 | } 17 | 18 | public double getLat() { 19 | return lat; 20 | } 21 | 22 | public void setLat(double lat) { 23 | this.lat = lat; 24 | } 25 | 26 | public double getLon() { 27 | return lon; 28 | } 29 | 30 | public void setLon(double lon) { 31 | this.lon = lon; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/GithubInfo.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import java.time.LocalDate; 4 | import java.util.Map; 5 | 6 | /** 7 | * Model object for information retrieved from Github via GraphQL 8 | */ 9 | public class GithubInfo { 10 | private String id = null; 11 | private String bio = null; 12 | private String companyName = null; 13 | private Map pullRequestCountsByLanguage = null; 14 | private Map pullRequestCountsByOwner = null; 15 | private Map pullRequestCountsByRepo = null; 16 | private Integer forkedRepoCount = null; 17 | private Integer ownedRepoCount = null; 18 | private Integer totalReceivedStarCount = null; 19 | private LocalDate lastPullRequestTime = null; 20 | 21 | public GithubInfo() { 22 | } 23 | 24 | public String getId() { 25 | return id; 26 | } 27 | 28 | public void setId(String id) { 29 | this.id = id; 30 | } 31 | 32 | public String getBio() { 33 | return bio; 34 | } 35 | 36 | public void setBio(String bio) { 37 | this.bio = bio; 38 | } 39 | 40 | public String getCompanyName() { 41 | return companyName; 42 | } 43 | 44 | public void setCompanyName(String companyName) { 45 | this.companyName = companyName; 46 | } 47 | 48 | public Integer getForkedRepoCount() { 49 | return forkedRepoCount; 50 | } 51 | 52 | public void setForkedRepoCount(Integer forkedRepoCount) { 53 | this.forkedRepoCount = forkedRepoCount; 54 | } 55 | 56 | public Integer getOwnedRepoCount() { 57 | return ownedRepoCount; 58 | } 59 | 60 | public void setOwnedRepoCount(Integer ownedRepoCount) { 61 | this.ownedRepoCount = ownedRepoCount; 62 | } 63 | 64 | public Integer getTotalReceivedStarCount() { 65 | return totalReceivedStarCount; 66 | } 67 | 68 | public void setTotalReceivedStarCount(Integer totalReceivedStarCount) { 69 | this.totalReceivedStarCount = totalReceivedStarCount; 70 | } 71 | 72 | public LocalDate getLastPullRequestTime() { 73 | return lastPullRequestTime; 74 | } 75 | 76 | public void setLastPullRequestTime(LocalDate lastPullRequestTime) { 77 | this.lastPullRequestTime = lastPullRequestTime; 78 | } 79 | 80 | public Map getPullRequestCountsByLanguage() { 81 | return pullRequestCountsByLanguage; 82 | } 83 | 84 | public void setPullRequestCountsByLanguage(Map pullRequestCountsByLanguage) { 85 | this.pullRequestCountsByLanguage = pullRequestCountsByLanguage; 86 | } 87 | 88 | public Map getPullRequestCountsByOwner() { 89 | return pullRequestCountsByOwner; 90 | } 91 | 92 | public void setPullRequestCountsByOwner(Map pullRequestCountsByOwner) { 93 | this.pullRequestCountsByOwner = pullRequestCountsByOwner; 94 | } 95 | 96 | public Map getPullRequestCountsByRepo() { 97 | return pullRequestCountsByRepo; 98 | } 99 | 100 | public void setPullRequestCountsByRepo(Map pullRequestCountsByRepo) { 101 | this.pullRequestCountsByRepo = pullRequestCountsByRepo; 102 | } 103 | 104 | @Override 105 | public String toString() { 106 | return "GithubInfo{" + 107 | "id='" + id + '\'' + 108 | ", bio='" + bio + '\'' + 109 | ", companyName='" + companyName + '\'' + 110 | ", pullRequestCountsByLanguage=" + pullRequestCountsByLanguage + 111 | ", pullRequestCountsByOwner=" + pullRequestCountsByOwner + 112 | ", pullRequestCountsByRepo=" + pullRequestCountsByRepo + 113 | ", forkedRepoCount=" + forkedRepoCount + 114 | ", ownedRepoCount=" + ownedRepoCount + 115 | ", totalReceivedStarCount=" + totalReceivedStarCount + 116 | ", lastPullRequestTime=" + lastPullRequestTime + 117 | '}'; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/HackClubUser.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import com.hackclub.clubs.models.engagements.BlotEngagement; 4 | import com.hackclub.clubs.models.engagements.OnboardEngagement; 5 | import com.hackclub.clubs.models.engagements.SprigEngagement; 6 | import com.hackclub.clubs.models.event.AngelhacksRegistration; 7 | import com.hackclub.clubs.models.event.AssembleRegistration; 8 | import com.hackclub.clubs.models.event.EventRegistration; 9 | import com.hackclub.clubs.models.event.OuternetRegistration; 10 | import com.hackclub.clubs.models.engagements.Engagement; 11 | import com.hackclub.common.geo.Geocoder; 12 | import com.hackclub.common.Utils; 13 | import org.apache.commons.lang3.StringUtils; 14 | 15 | import java.time.LocalDate; 16 | import java.time.ZoneId; 17 | import java.util.HashMap; 18 | import java.util.Map; 19 | import java.util.Optional; 20 | 21 | /** 22 | * Represents the data that we want to persist in ElasticSearch 23 | */ 24 | public class HackClubUser { 25 | private static HashMap allUsers = new HashMap<>(); 26 | 27 | /** 28 | * Fields we intend on serializing to JSON - BEGIN 29 | */ 30 | private String rootId = null; 31 | private String slackUserName = null; 32 | private String slackHandle = null; 33 | private String email = null; 34 | private String status = null; 35 | private String slackUserId = null; 36 | private String slackDisplayName = null; 37 | private LocalDate earliestPostDate = null; 38 | private LocalDate latestPostDate = null; 39 | private String githubUsername = null; 40 | private String timezone = null; 41 | private Integer maxScrapbookStreaks = null; 42 | private String pronouns = null; 43 | private String website = null; 44 | private String fullRealName = null; 45 | private String birthday = null; 46 | private String schoolYear = null; 47 | private String phoneNumber = null; 48 | private String address = null; 49 | private String country = null; 50 | private String gender = null; 51 | private String ethnicity = null; 52 | private String prettyAddress = null; 53 | private String twitter = null; 54 | private Boolean isStaff = false; 55 | private Boolean isAlumni = false; 56 | private Boolean hasGeo = false; 57 | private Boolean isScrapbookUser = false; 58 | private Boolean isOrWasLeader = false; 59 | private Boolean isActiveLeader = false; 60 | private Integer age = null; 61 | private Integer birthYear = null; 62 | private Long lastSlackActivity = null; 63 | private GeoPoint geolocation = null; 64 | private Map keywords = new HashMap<>(); 65 | private Optional slackInfo = Optional.empty(); 66 | private GithubInfo githubInfo = new GithubInfo(); 67 | private Map eventAttendance = new HashMap<>(); 68 | private Map engagements = new HashMap<>(); 69 | /** 70 | * Fields we intend on serializing to JSON - END 71 | */ 72 | 73 | // Constructor 74 | public HackClubUser(String rootId) { 75 | this.rootId = rootId; 76 | getAllUsers().put(rootId, this); 77 | } 78 | 79 | //username,email,status,billing-active,has-2fa,has-sso,userid,fullname,displayname,expiration-timestamp 80 | public static HackClubUser fromSlackCsv(String[] parts) { 81 | String slackUserId = parts[6]; 82 | HackClubUser newUser = new HackClubUser(slackUserId); 83 | newUser.setSlackData(slackUserId, parts[8], parts[0], parts[1], parts[2], parts[7]); 84 | return newUser; 85 | } 86 | 87 | public static HashMap getAllUsers() { 88 | return allUsers; 89 | } 90 | 91 | public static Optional get(String rootId) { 92 | return Optional.ofNullable(allUsers.getOrDefault(rootId, null)); 93 | } 94 | 95 | public void setSlackData(String slackUserId, String slackHandle, String slackUserName, String email, String status, String slackDisplayName) { 96 | this.slackUserId = slackUserId; 97 | this.slackHandle = slackHandle.length() == 0 ? null : slackHandle; 98 | this.slackUserName = slackUserName; 99 | this.email = email; 100 | this.status = status; 101 | this.slackDisplayName = slackDisplayName; 102 | } 103 | 104 | public static Optional getWithRootId(String rootId) { 105 | if (rootId == null) 106 | return Optional.empty(); 107 | 108 | return Optional.ofNullable(getAllUsers().getOrDefault(rootId, null)); 109 | } 110 | 111 | public String getSlackUserName() { 112 | return slackUserName; 113 | } 114 | 115 | public String getEmail() { 116 | return email; 117 | } 118 | 119 | public String getStatus() { 120 | return status; 121 | } 122 | 123 | public String getSlackUserId() { 124 | return slackUserId; 125 | } 126 | 127 | public String getSlackDisplayName() { 128 | return slackDisplayName; 129 | } 130 | 131 | public boolean isStaff() { 132 | return isStaff; 133 | } 134 | 135 | public void setStaff(boolean staff) { 136 | isStaff = staff; 137 | } 138 | 139 | public void onSlackChatMessageProcessed(ChannelEvent entry) { 140 | final ChannelDay day = entry.getParent(); 141 | final LocalDate date = day.getLocalDate(); 142 | 143 | // Let's be careful as this might be called from multiple threads... 144 | synchronized(this) { 145 | if(earliestPostDate == null || date.isBefore(earliestPostDate)) 146 | earliestPostDate = date; 147 | 148 | if (latestPostDate == null || date.isAfter(latestPostDate)) 149 | latestPostDate = date; 150 | } 151 | } 152 | 153 | public boolean isActiveSince(LocalDate date) { 154 | return latestPostDate.isAfter(date); 155 | } 156 | 157 | public void finish() { 158 | ZoneId zoneId = ZoneId.systemDefault(); // or: ZoneId.of("Europe/Oslo"); 159 | 160 | if (latestPostDate != null) { 161 | lastSlackActivity = latestPostDate.atStartOfDay(zoneId).toEpochSecond(); 162 | if (latestPostDate.isBefore(LocalDate.now().minusYears(5))) 163 | isAlumni = true; 164 | } 165 | 166 | if (birthYear != null) { 167 | age = LocalDate.now().getYear() - birthYear; 168 | } 169 | 170 | if (prettyAddress != null) { 171 | try { 172 | Geocoder.geocode(prettyAddress).ifPresent(this::setGeolocation); 173 | } catch (Throwable t) { 174 | System.out.printf("Issue geocoding: %s\n", t.getMessage()); 175 | } 176 | } 177 | } 178 | 179 | public void setLeaderInfo(Optional clubInfoOpt, Optional leaderApplicationInfoOpt) { 180 | if (leaderApplicationInfoOpt.isPresent()) { 181 | ClubLeaderApplicationInfo leaderApplicationInfo = leaderApplicationInfoOpt.get(); 182 | isOrWasLeader = leaderApplicationInfo.isOrWasLeader(); 183 | fullRealName = leaderApplicationInfo.getFullName(); 184 | 185 | birthday = Utils.sanitizeDate(leaderApplicationInfo.getBirthday()); 186 | schoolYear = leaderApplicationInfo.getSchoolYear(); 187 | phoneNumber = leaderApplicationInfo.getPhoneNumber(); 188 | address = leaderApplicationInfo.getAddress(); 189 | country = leaderApplicationInfo.getCountry(); 190 | gender = StringUtils.isEmpty(leaderApplicationInfo.getGender()) ? "Unknown" : leaderApplicationInfo.getGender(); 191 | ethnicity = StringUtils.isEmpty(leaderApplicationInfo.getEthnicity()) ? "Unknown" : leaderApplicationInfo.getEthnicity(); 192 | prettyAddress = leaderApplicationInfo.getPrettyAddress(); 193 | twitter = leaderApplicationInfo.getTwitter(); 194 | 195 | // Only slurp this if it doesn't exist already 196 | if (!StringUtils.isEmpty(leaderApplicationInfo.getGithub())) { 197 | githubUsername = Utils.getLastPathInUrl(leaderApplicationInfo.getGithub()); 198 | } 199 | birthYear = leaderApplicationInfo.getBirthYear(); 200 | } 201 | 202 | if (clubInfoOpt.isPresent()) { 203 | ClubInfo clubInfo = clubInfoOpt.get(); 204 | 205 | isOrWasLeader = true; 206 | isActiveLeader = StringUtils.equals(clubInfo.getStatus(), "active"); 207 | if (StringUtils.isEmpty(prettyAddress) && !StringUtils.isEmpty(clubInfo.getClubAddress())) 208 | prettyAddress = clubInfo.getClubAddress(); 209 | } 210 | } 211 | 212 | public void setScrapbookAccount(Optional scrapbookAccount) { 213 | if (scrapbookAccount.isEmpty()) 214 | return; 215 | 216 | ScrapbookAccount sba = scrapbookAccount.get(); 217 | 218 | String githubUrl = StringUtils.isEmpty(sba.getGithubUrl()) ? null : sba.getGithubUrl(); 219 | if (githubUrl != null) { 220 | githubUsername = Utils.getLastPathInUrl(githubUrl); 221 | } 222 | 223 | timezone = StringUtils.isEmpty(sba.getTimezone()) ? null : sba.getTimezone(); 224 | maxScrapbookStreaks = sba.getMaxStreaks(); 225 | pronouns = StringUtils.isEmpty(sba.getPronouns()) ? null : sba.getPronouns(); 226 | website = StringUtils.isEmpty(sba.getWebsite()) ? null : sba.getWebsite(); 227 | isScrapbookUser = true; 228 | } 229 | 230 | public void setKeywords(Map keywords) { 231 | this.keywords = keywords; 232 | } 233 | 234 | public Map getKeywords() { 235 | return keywords; 236 | } 237 | 238 | public String getSlackHandle() { 239 | return slackHandle; 240 | } 241 | 242 | public void setSlackHandle(String slackHandle) { 243 | this.slackHandle = slackHandle; 244 | } 245 | 246 | public String getTimezone() { 247 | return timezone; 248 | } 249 | 250 | public void setTimezone(String timezone) { 251 | this.timezone = timezone; 252 | } 253 | 254 | public Integer getMaxScrapbookStreaks() { 255 | return maxScrapbookStreaks; 256 | } 257 | 258 | public void setMaxScrapbookStreaks(Integer maxScrapbookStreaks) { 259 | this.maxScrapbookStreaks = maxScrapbookStreaks; 260 | } 261 | 262 | public String getPronouns() { 263 | return pronouns; 264 | } 265 | 266 | public void setPronouns(String pronouns) { 267 | this.pronouns = pronouns; 268 | } 269 | 270 | public String getWebsite() { 271 | return website; 272 | } 273 | 274 | public void setWebsite(String website) { 275 | this.website = website; 276 | } 277 | 278 | public boolean isScrapbookUser() { 279 | return isScrapbookUser; 280 | } 281 | 282 | public void setScrapbookUser(boolean scrapbookUser) { 283 | isScrapbookUser = scrapbookUser; 284 | } 285 | 286 | public boolean isOrWasLeader() { 287 | return isOrWasLeader; 288 | } 289 | 290 | public void setOrWasLeader(boolean orWasLeader) { 291 | isOrWasLeader = orWasLeader; 292 | } 293 | 294 | public String getFullRealName() { 295 | return fullRealName; 296 | } 297 | 298 | public void setFullRealName(String fullRealName) { 299 | this.fullRealName = fullRealName; 300 | } 301 | 302 | public String getBirthday() { 303 | return birthday; 304 | } 305 | 306 | public void setBirthday(String birthday) { 307 | this.birthday = Utils.sanitizeDate(birthday); 308 | } 309 | 310 | public String getSchoolYear() { 311 | return schoolYear; 312 | } 313 | 314 | public void setSchoolYear(String schoolYear) { 315 | this.schoolYear = schoolYear; 316 | } 317 | 318 | public String getPhoneNumber() { 319 | return phoneNumber; 320 | } 321 | 322 | public void setPhoneNumber(String phoneNumber) { 323 | this.phoneNumber = phoneNumber; 324 | } 325 | 326 | public String getAddress() { 327 | return address; 328 | } 329 | 330 | public void setAddress(String address) { 331 | this.address = address; 332 | } 333 | 334 | public String getCountry() { 335 | return country; 336 | } 337 | 338 | public void setCountry(String country) { 339 | this.country = country; 340 | } 341 | 342 | public String getGender() { 343 | return gender; 344 | } 345 | 346 | public void setGender(String gender) { 347 | this.gender = gender; 348 | } 349 | 350 | public String getEthnicity() { 351 | return ethnicity; 352 | } 353 | 354 | public void setEthnicity(String ethnicity) { 355 | this.ethnicity = ethnicity; 356 | } 357 | 358 | public String getPrettyAddress() { 359 | return prettyAddress; 360 | } 361 | 362 | public void setPrettyAddress(String prettyAddress) { 363 | this.prettyAddress = prettyAddress; 364 | } 365 | 366 | public String getTwitter() { 367 | return twitter; 368 | } 369 | 370 | public void setTwitter(String twitter) { 371 | this.twitter = twitter; 372 | } 373 | 374 | public Integer getBirthYear() { 375 | return birthYear; 376 | } 377 | 378 | public void setBirthYear(Integer birthYear) { 379 | this.birthYear = birthYear; 380 | } 381 | 382 | public Long getLastSlackActivity() { 383 | return lastSlackActivity; 384 | } 385 | 386 | public void setLastSlackActivity(Long lastSlackActivity) { 387 | this.lastSlackActivity = lastSlackActivity; 388 | } 389 | 390 | public boolean isAlumni() { 391 | return isAlumni; 392 | } 393 | 394 | public void setAlumni(boolean alumni) { 395 | isAlumni = alumni; 396 | } 397 | 398 | public Integer getAge() { 399 | return age; 400 | } 401 | 402 | public void setAge(Integer age) { 403 | this.age = age; 404 | } 405 | 406 | public GeoPoint getGeolocation() { 407 | return geolocation; 408 | } 409 | 410 | public void setGeolocation(GeoPoint geolocation) { 411 | this.geolocation = geolocation; 412 | hasGeo = (geolocation != null); 413 | } 414 | 415 | public boolean isHasGeo() { 416 | return hasGeo; 417 | } 418 | 419 | public void setHasGeo(boolean hasGeo) { 420 | this.hasGeo = hasGeo; 421 | } 422 | 423 | public GithubInfo getGithubInfo() { 424 | return githubInfo; 425 | } 426 | 427 | public void setGithubInfo(GithubInfo githubInfo) { 428 | this.githubInfo = githubInfo; 429 | } 430 | 431 | public void setOperationsInfo(OperationsInfo opsInfo) { 432 | } 433 | 434 | @Override 435 | public String toString() { 436 | return "HackClubUser{" + 437 | "slackUserName='" + slackUserName + '\'' + 438 | ", slackHandle='" + slackHandle + '\'' + 439 | ", email='" + email + '\'' + 440 | ", status='" + status + '\'' + 441 | ", slackUserId='" + slackUserId + '\'' + 442 | ", slackDisplayName='" + slackDisplayName + '\'' + 443 | ", earliestPostDate=" + earliestPostDate + 444 | ", latestPostDate=" + latestPostDate + 445 | ", isStaff=" + isStaff + 446 | ", timezone='" + timezone + '\'' + 447 | ", maxScrapbookStreaks=" + maxScrapbookStreaks + 448 | ", pronouns='" + pronouns + '\'' + 449 | ", website='" + website + '\'' + 450 | ", isScrapbookUser=" + isScrapbookUser + 451 | ", isOrWasLeader=" + isOrWasLeader + 452 | ", fullRealName='" + fullRealName + '\'' + 453 | ", birthday='" + birthday + '\'' + 454 | ", schoolYear='" + schoolYear + '\'' + 455 | ", phoneNumber='" + phoneNumber + '\'' + 456 | ", address='" + address + '\'' + 457 | ", country='" + country + '\'' + 458 | ", gender='" + gender + '\'' + 459 | ", ethnicity='" + ethnicity + '\'' + 460 | ", prettyAddress='" + prettyAddress + '\'' + 461 | ", twitter='" + twitter + '\'' + 462 | ", birthYear=" + birthYear + 463 | ", lastSlackActivity=" + lastSlackActivity + 464 | ", isAlumni=" + isAlumni + 465 | ", age=" + age + 466 | ", geolocation=" + geolocation + 467 | ", hasGeo=" + hasGeo + 468 | ", keywords=" + keywords + 469 | ", githubInfo=" + githubInfo + 470 | '}'; 471 | } 472 | 473 | public void setOuternetRegistration(OuternetRegistration reg) { 474 | EventRegistration data = new EventRegistration(); 475 | data.setRsvped(true); 476 | data.setAttended(reg.isCheckedIn()); 477 | data.setStipendRequested(reg.isReceivedStipend()); 478 | eventAttendance.put("outernet", data); 479 | 480 | if (StringUtils.isEmpty(githubUsername)) { 481 | githubUsername = Utils.getLastPathInUrl(reg.getGithub()); 482 | } 483 | 484 | if (StringUtils.isEmpty(email)) email = reg.getEmail(); 485 | if (StringUtils.isEmpty(fullRealName)) fullRealName = reg.getName(); 486 | if (StringUtils.isEmpty(githubUsername)) githubUsername = Utils.getLastPathInUrl(reg.getGithub()); 487 | } 488 | 489 | public void setAssembleRegistration(AssembleRegistration reg) { 490 | EventRegistration data = new EventRegistration(); 491 | data.setRsvped(true); 492 | data.setAttended(true); 493 | data.setStipendRequested(reg.isReceivedStipend()); 494 | 495 | if (StringUtils.isNotEmpty(reg.getNearestAirport())) { 496 | if (StringUtils.isEmpty(prettyAddress)) { 497 | prettyAddress = "Airport - " + reg.getNearestAirport(); 498 | } 499 | } 500 | eventAttendance.put("assemble", data); 501 | 502 | if (StringUtils.isEmpty(email)) email = reg.getEmail(); 503 | if (StringUtils.isEmpty(fullRealName)) fullRealName = reg.getFullName(); 504 | } 505 | 506 | public void setAngelhacksRegistration(AngelhacksRegistration reg) { 507 | EventRegistration data = new EventRegistration(); 508 | data.setAttended(reg.isCheckedIn()); 509 | data.setRsvped(true); 510 | data.setStipendRequested(false); 511 | eventAttendance.put("angelhacks", data); 512 | 513 | if (StringUtils.isEmpty(email)) email = reg.getEmail(); 514 | if (StringUtils.isEmpty(fullRealName)) fullRealName = reg.getName(); 515 | } 516 | 517 | public void setBlotEngagement(BlotEngagement reg) { 518 | Engagement data = new Engagement(); 519 | data.setImpressed(true); 520 | data.setRewarded(StringUtils.equals(reg.getStatus(), "Shipped")); 521 | engagements.put("blot", data); 522 | if (StringUtils.isEmpty(email)) email = reg.getEmail(); 523 | if (StringUtils.isEmpty(fullRealName)) fullRealName = reg.getName(); 524 | if (StringUtils.isEmpty(slackUserId)) slackUserId = reg.getSlackId(); 525 | String blotAddress = String.format("%s, %s, %s", reg.getCity(), reg.getState(), reg.getCountry()); 526 | if (StringUtils.isEmpty(address)) address = blotAddress; 527 | if (StringUtils.isEmpty(prettyAddress)) prettyAddress = blotAddress; 528 | } 529 | 530 | public void setSprigEngagement(SprigEngagement reg) { 531 | Engagement data = new Engagement(); 532 | data.setImpressed(true); 533 | data.setRewarded(StringUtils.equals(reg.getStatus(), "Shipped")); 534 | engagements.put("sprig", data); 535 | 536 | if (StringUtils.isEmpty(email)) email = reg.getEmail(); 537 | String sprigAddress = String.format("%s, %s, %s", reg.getCity(), reg.getState(), reg.getCountry()); 538 | if (StringUtils.isEmpty(address)) address = sprigAddress; 539 | if (StringUtils.isEmpty(prettyAddress)) prettyAddress = sprigAddress; 540 | if (StringUtils.isEmpty(slackUserId)) slackUserId = reg.getSlackId(); 541 | if (StringUtils.isEmpty(fullRealName)) fullRealName = reg.getName(); 542 | } 543 | 544 | public void setOnboardEngagement(OnboardEngagement reg) { 545 | Engagement data = new Engagement(); 546 | data.setImpressed(true); 547 | data.setRewarded(StringUtils.equals(reg.getStatus(), "Approved")); 548 | engagements.put("onboard", data); 549 | String onboardAddress = String.format("%s, %s", reg.getCity(), reg.getState()); 550 | if (StringUtils.isEmpty(email)) email = reg.getEmail(); 551 | if (StringUtils.isEmpty(address)) address = onboardAddress; 552 | if (StringUtils.isEmpty(prettyAddress)) prettyAddress = onboardAddress; 553 | if (StringUtils.isEmpty(fullRealName)) fullRealName = reg.getFullName(); 554 | if (StringUtils.isEmpty(birthday)) birthday = Utils.sanitizeDate(reg.getBirthDate()); 555 | } 556 | 557 | public Optional getSlackInfo() { 558 | return slackInfo; 559 | } 560 | 561 | public void setSlackInfo(Optional slackInfo) { 562 | this.slackInfo = slackInfo; 563 | } 564 | 565 | public String getGithubUsername() { 566 | return githubUsername; 567 | } 568 | 569 | public void setGithubUsername(String githubUsername) { 570 | this.githubUsername = githubUsername; 571 | } 572 | 573 | public void setSlackUserName(String slackUserName) { 574 | this.slackUserName = slackUserName; 575 | } 576 | 577 | public void setEmail(String email) { 578 | this.email = email; 579 | } 580 | 581 | public void setSlackUserId(String slackUserId) { 582 | this.slackUserId = slackUserId; 583 | } 584 | 585 | public void setSlackDisplayName(String slackDisplayName) { 586 | this.slackDisplayName = slackDisplayName; 587 | } 588 | 589 | public Map getEventAttendance() { 590 | return eventAttendance; 591 | } 592 | 593 | public void setEventAttendance(Map eventAttendance) { 594 | this.eventAttendance = eventAttendance; 595 | } 596 | 597 | public boolean isActiveLeader() { 598 | return isActiveLeader; 599 | } 600 | 601 | public void setActiveLeader(boolean activeLeader) { 602 | isActiveLeader = activeLeader; 603 | } 604 | 605 | public Map getEngagements() { 606 | return engagements; 607 | } 608 | 609 | public void setEngagements(Map engagements) { 610 | this.engagements = engagements; 611 | } 612 | public String getRootId() { 613 | return rootId; 614 | } 615 | 616 | public void setRootId(String rootId) { 617 | this.rootId = rootId; 618 | } 619 | 620 | } 621 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/OperationsInfo.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | /** 4 | * Model for information kept in the Operations Airtable 5 | */ 6 | public class OperationsInfo { 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/PirateShipEntry.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import java.util.HashMap; 4 | 5 | /** 6 | * Model object for address data contained within PirateShip 7 | */ 8 | public class PirateShipEntry { 9 | private String recipient; 10 | private String addressLine1; 11 | private String addressLine2; 12 | private String city; 13 | private String state; 14 | private String zipCode; 15 | private String country; 16 | private String concatenatedAddress; 17 | 18 | private PirateShipEntry() {} 19 | 20 | // Created Date,Recipient,Company,Email,Tracking Number,Cost,Status,Batch,Label Size,Saved Package,Ship From,Ship Date,Estimated Delivery Time,Weight (oz),Zone,Package Type,Package Length,Package Width,Package Height,Tracking Status,Tracking Info,Tracking Date,Address Line 1,Address Line 2,City,State,Zipcode,Country,Carrier,Service,Order ID,Rubber Stamp 1,Rubber Stamp 2,Rubber Stamp 3,Order Value 21 | public static PirateShipEntry fromCsv(String[] nextLine, HashMap columnIndices) { 22 | PirateShipEntry e = new PirateShipEntry(); 23 | e.recipient = nextLine[columnIndices.get("Recipient")]; 24 | 25 | e.addressLine1 = nextLine[columnIndices.get("Address Line 1")]; 26 | e.addressLine2 = nextLine[columnIndices.get("Address Line 2")]; 27 | e.city = nextLine[columnIndices.get("City")]; 28 | e.state = nextLine[columnIndices.get("State")]; 29 | e.zipCode = nextLine[columnIndices.get("Zipcode")]; 30 | e.country = nextLine[columnIndices.get("Country")]; 31 | e.concatenatedAddress = String.format("%s %s %s %s %s %s", 32 | e.addressLine1, e.addressLine2, e.city, e.state, e.zipCode, e.country); 33 | 34 | return e; 35 | } 36 | 37 | public String getRecipient() { 38 | return recipient; 39 | } 40 | 41 | public void setRecipient(String recipient) { 42 | this.recipient = recipient; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/ScrapbookAccount.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import java.util.HashMap; 4 | import java.util.Objects; 5 | 6 | /** 7 | * Model object for data relating to a Scrapbook account 8 | */ 9 | public class ScrapbookAccount { 10 | private String githubUrl = null; 11 | private String timezone = null; 12 | 13 | private boolean fullSlackMember; 14 | 15 | private String pronouns = null; 16 | 17 | private String slackId = null; 18 | 19 | private Integer maxStreaks = null; 20 | 21 | private String email = null; 22 | 23 | private String website = null; 24 | 25 | // Not publicly instantiable - use factory methods (fromCsv) 26 | private ScrapbookAccount() { 27 | } 28 | public static ScrapbookAccount fromCsv(String[] nextLine, HashMap columnIndices) { 29 | ScrapbookAccount sba = new ScrapbookAccount(); 30 | sba.githubUrl = nextLine[columnIndices.get("github")]; 31 | if (sba.githubUrl != null && !sba.githubUrl.startsWith("https://github.com/")) 32 | sba.githubUrl = null; 33 | sba.timezone = nextLine[columnIndices.get("timezone")]; 34 | sba.fullSlackMember = Objects.equals(nextLine[columnIndices.get("github")], "t"); 35 | 36 | String rawMaxStreaks = nextLine[columnIndices.get("maxstreaks")]; 37 | sba.maxStreaks = rawMaxStreaks.length() > 0 ? Integer.parseInt(rawMaxStreaks) : null; 38 | sba.email = nextLine[columnIndices.get("email")]; 39 | sba.slackId = nextLine[columnIndices.get("slackid")]; 40 | sba.pronouns = nextLine[columnIndices.get("pronouns")]; 41 | sba.website = nextLine[columnIndices.get("website")]; 42 | return sba; 43 | } 44 | 45 | public String getGithubUrl() { 46 | return githubUrl; 47 | } 48 | 49 | public void setGithubUrl(String githubUrl) { 50 | this.githubUrl = githubUrl; 51 | } 52 | 53 | public String getTimezone() { 54 | return timezone; 55 | } 56 | 57 | public void setTimezone(String timezone) { 58 | this.timezone = timezone; 59 | } 60 | 61 | public boolean isFullSlackMember() { 62 | return fullSlackMember; 63 | } 64 | 65 | public void setFullSlackMember(boolean fullSlackMember) { 66 | this.fullSlackMember = fullSlackMember; 67 | } 68 | 69 | public String getPronouns() { 70 | return pronouns; 71 | } 72 | 73 | public void setPronouns(String pronouns) { 74 | this.pronouns = pronouns; 75 | } 76 | 77 | public String getSlackId() { 78 | return slackId; 79 | } 80 | 81 | public void setSlackId(String slackId) { 82 | this.slackId = slackId; 83 | } 84 | 85 | public Integer getMaxStreaks() { 86 | return maxStreaks; 87 | } 88 | 89 | public void setMaxStreaks(Integer maxStreaks) { 90 | this.maxStreaks = maxStreaks; 91 | } 92 | 93 | public String getEmail() { 94 | return email; 95 | } 96 | 97 | public void setEmail(String email) { 98 | this.email = email; 99 | } 100 | 101 | public String getWebsite() { 102 | return website; 103 | } 104 | 105 | public void setWebsite(String website) { 106 | this.website = website; 107 | } 108 | 109 | @Override 110 | public String toString() { 111 | return "ScrapbookAccount{" + 112 | "githubUrl='" + githubUrl + '\'' + 113 | ", timezone='" + timezone + '\'' + 114 | ", fullSlackMember=" + fullSlackMember + 115 | ", pronouns='" + pronouns + '\'' + 116 | ", slackId='" + slackId + '\'' + 117 | ", maxStreaks=" + maxStreaks + 118 | ", email='" + email + '\'' + 119 | ", website='" + website + '\'' + 120 | '}'; 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/SlackInfo.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | /** 4 | * Model object for information retrieved from Slack API 5 | */ 6 | public class SlackInfo { 7 | private String githubUrl = null; 8 | private String githubUsername = null; 9 | 10 | public SlackInfo() { 11 | } 12 | 13 | public String getGithubUrl() { 14 | return githubUrl; 15 | } 16 | 17 | public void setGithubUrl(String githubUrl) { 18 | this.githubUrl = githubUrl; 19 | } 20 | 21 | public String getGithubUsername() { 22 | return githubUsername; 23 | } 24 | 25 | public void setGithubUsername(String githubUsername) { 26 | this.githubUsername = githubUsername; 27 | } 28 | 29 | @Override 30 | public String toString() { 31 | return "SlackInfo{" + 32 | "githubUrl='" + githubUrl + '\'' + 33 | ", githubUsername='" + githubUsername + '\'' + 34 | '}'; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/StaffUsers.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Model object for a collection of Hack Club HQ staff members 7 | */ 8 | public class StaffUsers { 9 | private boolean ok; 10 | private List users; 11 | 12 | public List getUsers() { 13 | return users; 14 | } 15 | 16 | public void setUsers(List users) { 17 | this.users = users; 18 | } 19 | 20 | public boolean isOk() { 21 | return ok; 22 | } 23 | 24 | public void setOk(boolean ok) { 25 | this.ok = ok; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/engagements/BlotEngagement.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models.engagements; 2 | 3 | import java.util.HashMap; 4 | 5 | public class BlotEngagement { 6 | private String email; 7 | private String name; 8 | private String city; 9 | private String state; 10 | private String country; 11 | private String slackId; 12 | private String status; 13 | 14 | // Not publicly instantiable - use factory methods (fromCsv) 15 | private BlotEngagement() { 16 | } 17 | 18 | // Email,Name,Address Line 1,Address Line 2,Address City,Address State,Address Country,Address Zip,Phone Number,Student Proof,Is Slack User?,Slack ID,Needs Printed Parts?,Status,Created At 19 | public static BlotEngagement fromCsv(String[] nextLine, HashMap columnIndices) { 20 | BlotEngagement eng = new BlotEngagement(); 21 | 22 | eng.email = nextLine[columnIndices.get("Email")]; 23 | eng.name = nextLine[columnIndices.get("Name")]; 24 | eng.city = nextLine[columnIndices.get("Address City")]; 25 | eng.state = nextLine[columnIndices.get("Address State")]; 26 | eng.country = nextLine[columnIndices.get("Address Country")]; 27 | eng.slackId = nextLine[columnIndices.get("Slack ID")]; 28 | eng.status = nextLine[columnIndices.get("Status")]; 29 | 30 | return eng; 31 | } 32 | 33 | public String getEmail() { 34 | return email; 35 | } 36 | 37 | public void setEmail(String email) { 38 | this.email = email; 39 | } 40 | 41 | public String getName() { 42 | return name; 43 | } 44 | 45 | public void setName(String name) { 46 | this.name = name; 47 | } 48 | 49 | public String getCity() { 50 | return city; 51 | } 52 | 53 | public void setCity(String city) { 54 | this.city = city; 55 | } 56 | 57 | public String getState() { 58 | return state; 59 | } 60 | 61 | public void setState(String state) { 62 | this.state = state; 63 | } 64 | 65 | public String getCountry() { 66 | return country; 67 | } 68 | 69 | public void setCountry(String country) { 70 | this.country = country; 71 | } 72 | 73 | public String getSlackId() { 74 | return slackId; 75 | } 76 | 77 | public void setSlackId(String slackId) { 78 | this.slackId = slackId; 79 | } 80 | 81 | public String getStatus() { 82 | return status; 83 | } 84 | 85 | public void setStatus(String status) { 86 | this.status = status; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/engagements/Engagement.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models.engagements; 2 | 3 | public class Engagement { 4 | // Whether or not this person had any interaction as a result of the engagement (i.e., joined a zoom, RSVP'd somehow, chatted in an associated slack channel 5 | private boolean impressed; 6 | 7 | // Whether or not this person was rewarded with a prize from this engagement 8 | private boolean rewarded; 9 | 10 | // The reason for the engagement happening ("I saw a poster!", etc) 11 | private String reason; 12 | 13 | public Engagement() {} 14 | 15 | public boolean isImpressed() { 16 | return impressed; 17 | } 18 | 19 | public void setImpressed(boolean impressed) { 20 | this.impressed = impressed; 21 | } 22 | 23 | public boolean isRewarded() { 24 | return rewarded; 25 | } 26 | 27 | public void setRewarded(boolean rewarded) { 28 | this.rewarded = rewarded; 29 | } 30 | public String getReason() { 31 | return reason; 32 | } 33 | 34 | public void setReason(String reason) { 35 | this.reason = reason; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/engagements/OnboardEngagement.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models.engagements; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | import java.util.HashMap; 6 | 7 | public class OnboardEngagement { 8 | private String email; 9 | private String fullName; 10 | private String githubHandle; 11 | private String status; 12 | private String birthDate; 13 | private String city; 14 | private String state; 15 | 16 | // Not publicly instantiable - use factory methods (fromCsv) 17 | private OnboardEngagement() { 18 | } 19 | 20 | // Full Name,Email,Proof of High School Enrollment,GitHub handle,Country,Status,Commented on Github? ,On HCB? ,Birthdate,1st line of shipping address,Zip/Postal code of shipping address,2nd line of shipping address,City (shipping address),State,Referral category,How did you hear about OnBoard?,Created,Is this the first PCB you've made? 21 | public static OnboardEngagement fromCsv(String[] nextLine, HashMap columnIndices) { 22 | OnboardEngagement eng = new OnboardEngagement(); 23 | 24 | eng.email = nextLine[columnIndices.get("Email")]; 25 | eng.fullName = nextLine[columnIndices.get("Full Name")]; 26 | eng.githubHandle = nextLine[columnIndices.get("GitHub handle")]; 27 | eng.status = nextLine[columnIndices.get("Status")]; 28 | eng.birthDate = nextLine[columnIndices.get("Birthdate")]; 29 | eng.city = nextLine[columnIndices.get("City (shipping address)")]; 30 | eng.state = nextLine[columnIndices.get("State")]; 31 | 32 | return eng; 33 | } 34 | 35 | public String getFullName() { 36 | return fullName; 37 | } 38 | 39 | public void setFullName(String fullName) { 40 | this.fullName = fullName; 41 | } 42 | 43 | public String getGithubHandle() { 44 | return githubHandle; 45 | } 46 | 47 | public void setGithubHandle(String githubHandle) { 48 | this.githubHandle = githubHandle; 49 | } 50 | 51 | public String getStatus() { 52 | return status; 53 | } 54 | 55 | public void setStatus(String status) { 56 | this.status = status; 57 | } 58 | 59 | public String getBirthDate() { 60 | return birthDate; 61 | } 62 | 63 | public void setBirthDate(String birthDate) { 64 | this.birthDate = birthDate; 65 | } 66 | 67 | public String getCity() { 68 | return city; 69 | } 70 | 71 | public void setCity(String city) { 72 | this.city = city; 73 | } 74 | 75 | public String getState() { 76 | return state; 77 | } 78 | 79 | public void setState(String state) { 80 | this.state = state; 81 | } 82 | 83 | public String getEmail() { 84 | return email; 85 | } 86 | 87 | public void setEmail(String email) { 88 | this.email = email; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/engagements/SprigEngagement.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models.engagements; 2 | 3 | import java.util.HashMap; 4 | 5 | public class SprigEngagement { 6 | private String name; 7 | private String githubUsername; 8 | private String pullRequest; 9 | private String email; 10 | private String birthday; 11 | private String city; 12 | private String state; 13 | private String country; 14 | private String slackId; 15 | private String status; 16 | 17 | // Not publicly instantiable - use factory methods (fromCsv) 18 | private SprigEngagement() { 19 | } 20 | 21 | // GitHub Username,Submitted AT,Pull Request,Email,Proof of Student,Birthday,Authentication ID,Name,Address line 1,Address line 2,City,State or Province,Zip,Country,Phone (optional),Hack Club Slack ID (optional),Color,In a club?,Sprig Status,Club name,Sprig seeds mailed?,How did you hear about Sprig?,Address Formatted,Status,Notes,Tracking,Carrier,Tracking Base Link,Tracking Emailed,Referral Source,Age (years) 22 | public static SprigEngagement fromCsv(String[] nextLine, HashMap columnIndices) { 23 | SprigEngagement eng = new SprigEngagement(); 24 | 25 | eng.name = nextLine[columnIndices.get("Name")]; 26 | eng.githubUsername = nextLine[columnIndices.get("GitHub Username")]; 27 | eng.pullRequest = nextLine[columnIndices.get("Pull Request")]; 28 | eng.email = nextLine[columnIndices.get("Email")]; 29 | eng.birthday = nextLine[columnIndices.get("Birthday")]; 30 | eng.city = nextLine[columnIndices.get("City")]; 31 | eng.state = nextLine[columnIndices.get("State or Province")]; 32 | eng.country = nextLine[columnIndices.get("Country")]; 33 | eng.slackId = nextLine[columnIndices.get("Hack Club Slack ID (optional)")]; 34 | eng.status = nextLine[columnIndices.get("Sprig Status")]; 35 | 36 | return eng; 37 | } 38 | 39 | public String getGithubUsername() { 40 | return githubUsername; 41 | } 42 | 43 | public void setGithubUsername(String githubUsername) { 44 | this.githubUsername = githubUsername; 45 | } 46 | 47 | public String getPullRequest() { 48 | return pullRequest; 49 | } 50 | 51 | public void setPullRequest(String pullRequest) { 52 | this.pullRequest = pullRequest; 53 | } 54 | 55 | public String getEmail() { 56 | return email; 57 | } 58 | 59 | public void setEmail(String email) { 60 | this.email = email; 61 | } 62 | 63 | public String getBirthday() { 64 | return birthday; 65 | } 66 | 67 | public void setBirthday(String birthday) { 68 | this.birthday = birthday; 69 | } 70 | 71 | public String getCity() { 72 | return city; 73 | } 74 | 75 | public void setCity(String city) { 76 | this.city = city; 77 | } 78 | 79 | public String getState() { 80 | return state; 81 | } 82 | 83 | public void setState(String state) { 84 | this.state = state; 85 | } 86 | 87 | public String getCountry() { 88 | return country; 89 | } 90 | 91 | public void setCountry(String country) { 92 | this.country = country; 93 | } 94 | 95 | public String getSlackId() { 96 | return slackId; 97 | } 98 | 99 | public void setSlackId(String slackId) { 100 | this.slackId = slackId; 101 | } 102 | 103 | public String getStatus() { 104 | return status; 105 | } 106 | 107 | public void setStatus(String status) { 108 | this.status = status; 109 | } 110 | 111 | public String getName() { 112 | return name; 113 | } 114 | 115 | public void setName(String name) { 116 | this.name = name; 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/event/AngelhacksRegistration.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models.event; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | import java.util.HashMap; 6 | 7 | // Name,Email,Phone number,School,Grade,Pronouns,T-shirt size,Duration,Skill Level,Game experience,Goals,Helping,Source,Waivers Done,Added to Postal,Checked in,Checked out 8 | public class AngelhacksRegistration { 9 | private String email; 10 | private String name; 11 | private String school; 12 | private boolean checkedIn; 13 | 14 | // Not publicly instantiable - use factory methods (fromCsv) 15 | private AngelhacksRegistration() { 16 | } 17 | 18 | public static AngelhacksRegistration fromCsv(String[] nextLine, HashMap columnIndices) { 19 | AngelhacksRegistration reg = new AngelhacksRegistration(); 20 | 21 | reg.email = nextLine[columnIndices.get("Email")]; 22 | reg.name = nextLine[columnIndices.get("Name")]; 23 | reg.school = nextLine[columnIndices.get("School")]; 24 | reg.checkedIn = StringUtils.isNotEmpty(nextLine[columnIndices.get("Checked in")]); 25 | 26 | return reg; 27 | } 28 | 29 | public String getEmail() { 30 | return email; 31 | } 32 | 33 | public void setEmail(String email) { 34 | this.email = email; 35 | } 36 | 37 | public String getName() { 38 | return name; 39 | } 40 | 41 | public void setName(String name) { 42 | this.name = name; 43 | } 44 | 45 | public String getSchool() { 46 | return school; 47 | } 48 | 49 | public void setSchool(String school) { 50 | this.school = school; 51 | } 52 | 53 | public boolean isCheckedIn() { 54 | return checkedIn; 55 | } 56 | 57 | public void setCheckedIn(boolean checkedIn) { 58 | this.checkedIn = checkedIn; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/event/AssembleRegistration.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models.event; 2 | 3 | import org.apache.commons.lang3.StringUtils; 4 | 5 | import java.util.HashMap; 6 | 7 | //ID,Email,Log In Path,Full Name,Your Nearest Airport,Birthday,Vaccinated?,"If you're not vaccinated, please explain why:",Do you require a letter for visa applications?,Travel Stipend,Dietary Restrictions,"At the moment, what is your estimated travel cost?",Travel Stipend Cost INT,What would a travel stipend mean to you?,Skill Level,Would you be interested in hosting a workshop session at Assemble?,Workshop Topic,Shirt,Parent Name,Parent Email,Tabs or Spaces,Pineapple on Pizza,Submission Timestamp,Voted For,Team Notes,Stipend,Decision:,Follow Up,Estimated Cost(Hugo),Amount of Votes,Name (For Prefill),Follow Up (For Prefill),Vote *against*,18?,Serious Alum?,Pronouns,Password Code,Send 2 Weeks Out Email,Waiver,Freedom,Off Waitlist,Vaccinated,waiver_type,Send Wed 3 Email,Created at 8 | public class AssembleRegistration { 9 | private String email; 10 | private String fullName; 11 | private String nearestAirport; 12 | private boolean receivedStipend; 13 | 14 | // Not publicly instantiable - use factory methods (fromCsv) 15 | private AssembleRegistration() { 16 | } 17 | 18 | public static AssembleRegistration fromCsv(String[] nextLine, HashMap columnIndices) { 19 | AssembleRegistration reg = new AssembleRegistration(); 20 | 21 | reg.email = nextLine[columnIndices.get("Email")]; 22 | reg.fullName = nextLine[columnIndices.get("Full Name")]; 23 | reg.nearestAirport = nextLine[columnIndices.get("Your Nearest Airport")]; 24 | reg.receivedStipend = StringUtils.isNotEmpty(nextLine[columnIndices.get("Travel Stipend")]); 25 | 26 | return reg; 27 | } 28 | 29 | public String getEmail() { 30 | return email; 31 | } 32 | 33 | public void setEmail(String email) { 34 | this.email = email; 35 | } 36 | 37 | public String getFullName() { 38 | return fullName; 39 | } 40 | 41 | public void setFullName(String fullName) { 42 | this.fullName = fullName; 43 | } 44 | 45 | public String getNearestAirport() { 46 | return nearestAirport; 47 | } 48 | 49 | public void setNearestAirport(String nearestAirport) { 50 | this.nearestAirport = nearestAirport; 51 | } 52 | 53 | public boolean isReceivedStipend() { 54 | return receivedStipend; 55 | } 56 | 57 | public void setReceivedStipend(boolean receivedStipend) { 58 | this.receivedStipend = receivedStipend; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/event/EventRegistration.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models.event; 2 | 3 | public class EventRegistration { 4 | private boolean rsvped = false; 5 | private boolean attended = false; 6 | private boolean stipendRequested = false; 7 | 8 | public EventRegistration() {} 9 | 10 | public boolean isRsvped() { 11 | return rsvped; 12 | } 13 | 14 | public void setRsvped(boolean rsvped) { 15 | this.rsvped = rsvped; 16 | } 17 | 18 | public boolean isAttended() { 19 | return attended; 20 | } 21 | 22 | public void setAttended(boolean attended) { 23 | this.attended = attended; 24 | } 25 | 26 | public boolean isStipendRequested() { 27 | return stipendRequested; 28 | } 29 | 30 | public void setStipendRequested(boolean stipendRequested) { 31 | this.stipendRequested = stipendRequested; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/models/event/OuternetRegistration.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.models.event; 2 | 3 | import com.hackclub.clubs.models.ScrapbookAccount; 4 | import org.apache.commons.lang3.StringUtils; 5 | 6 | import java.util.HashMap; 7 | import java.util.Objects; 8 | 9 | // ID,Name,Club Leader?,Workshop / Lightning Talk Focus,Email,Pronouns,Birthday,T-Shirt Size,Travel,Dietary Restrictions,Parent's Name,Parent's Email,GitHub,Example Project,Curiosity,Guild Interest,Guild Focus,ranking,Workshop / Lightning Talk Interest,Stipend Record,Cool ideas,Shuttle Record,migration,workshop status,Waiver Sent,Created,Stipend Approved,Pod,Checked In?,Notes,Contact's Phone number,Checked out,Accepted Stipends 10 | public class OuternetRegistration { 11 | private String email; 12 | private String name; 13 | private boolean isLeader; 14 | private String github; 15 | private boolean checkedIn; 16 | private boolean receivedStipend; 17 | 18 | // Not publicly instantiable - use factory methods (fromCsv) 19 | private OuternetRegistration() { 20 | } 21 | public static OuternetRegistration fromCsv(String[] nextLine, HashMap columnIndices) { 22 | OuternetRegistration reg = new OuternetRegistration(); 23 | 24 | reg.email = nextLine[columnIndices.get("Email")]; 25 | reg.name = nextLine[columnIndices.get("Name")]; 26 | reg.isLeader = Boolean.parseBoolean(nextLine[columnIndices.get("Club Leader?")]); 27 | reg.github = nextLine[columnIndices.get("GitHub")]; 28 | reg.checkedIn = StringUtils.isNotEmpty(nextLine[columnIndices.get("Checked In?")]); 29 | reg.receivedStipend = StringUtils.isNotEmpty(nextLine[columnIndices.get("Stipend Approved")]); 30 | 31 | return reg; 32 | } 33 | 34 | public String getEmail() { 35 | return email; 36 | } 37 | 38 | public void setEmail(String email) { 39 | this.email = email; 40 | } 41 | 42 | public String getName() { 43 | return name; 44 | } 45 | 46 | public void setName(String name) { 47 | this.name = name; 48 | } 49 | 50 | public boolean isLeader() { 51 | return isLeader; 52 | } 53 | 54 | public void setLeader(boolean leader) { 55 | isLeader = leader; 56 | } 57 | 58 | public String getGithub() { 59 | return github; 60 | } 61 | 62 | public void setGithub(String github) { 63 | this.github = github; 64 | } 65 | 66 | public boolean isCheckedIn() { 67 | return checkedIn; 68 | } 69 | 70 | public void setCheckedIn(boolean checkedIn) { 71 | this.checkedIn = checkedIn; 72 | } 73 | 74 | @Override 75 | public String toString() { 76 | return "OuternetRegistration{" + 77 | "email='" + email + '\'' + 78 | ", name='" + name + '\'' + 79 | ", isLeader=" + isLeader + 80 | ", github='" + github + '\'' + 81 | ", checkedIn=" + checkedIn + 82 | '}'; 83 | } 84 | 85 | public boolean isReceivedStipend() { 86 | return receivedStipend; 87 | } 88 | 89 | public void setReceivedStipend(boolean receivedStipend) { 90 | this.receivedStipend = receivedStipend; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/clubs/slack/SlackUtils.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.clubs.slack; 2 | 3 | import com.hackclub.clubs.models.SlackInfo; 4 | import com.fasterxml.jackson.core.JsonProcessingException; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import com.slack.api.Slack; 7 | import com.slack.api.methods.SlackApiException; 8 | import com.slack.api.methods.request.users.profile.UsersProfileGetRequest; 9 | import com.slack.api.methods.response.users.profile.UsersProfileGetResponse; 10 | import com.slack.api.model.User; 11 | import com.hackclub.common.file.Cache; 12 | import java.util.Map; 13 | import java.util.Optional; 14 | 15 | /** 16 | * Integrates with Slack API 17 | */ 18 | public class SlackUtils { 19 | private final static String githubUrlKey = "Xf0DMHFDQA"; // Note - this is not any sort of secret, just a uid on GH 20 | public static Optional getSlackInfo(String slackUserId, String slackToken) { 21 | Optional slackData = getCachedSlackInfo(slackUserId); 22 | if (slackData.isPresent()) { 23 | return slackData; 24 | } 25 | 26 | Slack slack = Slack.getInstance(); 27 | Optional ret = Optional.empty(); 28 | boolean success = false; 29 | while(!success) { 30 | try { 31 | UsersProfileGetResponse response = slack.methods(slackToken).usersProfileGet(UsersProfileGetRequest.builder() 32 | .user(slackUserId).build()); 33 | 34 | if (response.getProfile() == null) { 35 | ret = Optional.empty(); 36 | success = true; 37 | } else { 38 | Map fields = response.getProfile().getFields(); 39 | 40 | SlackInfo slackInfo = new SlackInfo(); 41 | if (fields.containsKey(githubUrlKey)) { 42 | String githubUrl = fields.get(githubUrlKey).getValue(); 43 | String[] urlParts = githubUrl.split("/"); 44 | String githubUsername = urlParts[urlParts.length - 1]; 45 | 46 | slackInfo.setGithubUrl(githubUrl); 47 | slackInfo.setGithubUsername(githubUsername); 48 | } 49 | ret = Optional.of(slackInfo); 50 | success = true; 51 | } 52 | } catch (SlackApiException e) { 53 | if (e.getResponse().code() == 429) { 54 | handleError(e, false,500); 55 | ret = Optional.empty(); 56 | } else { 57 | handleError(e, true, 0); 58 | ret = Optional.empty(); 59 | } 60 | } 61 | catch (Throwable t) { 62 | handleError(t, true,0); 63 | ret = Optional.empty(); 64 | } 65 | } 66 | 67 | cacheSlackData(slackUserId, ret); 68 | return ret; 69 | } 70 | 71 | private static void handleError(Throwable t, boolean log, long msToWait) { 72 | if (log) { 73 | System.out.println("warning - issue"); 74 | t.printStackTrace(); 75 | } 76 | try { 77 | // We'll put a sleep here to back off a bit, errors here usually relate to rate limits 78 | if (msToWait > 0) { 79 | Thread.sleep(msToWait); 80 | } 81 | } catch (InterruptedException e) { 82 | // Do nothing 83 | } 84 | } 85 | 86 | private static void cacheSlackData(String slackUserId, Optional ret) { 87 | if (slackUserId == null) 88 | return; 89 | 90 | try { 91 | if (ret.isPresent()) { 92 | Cache.save(slackUserId, new ObjectMapper().writeValueAsString(ret.get())); 93 | } 94 | } catch (Throwable t) { 95 | t.printStackTrace(); 96 | } 97 | } 98 | 99 | private static Optional getCachedSlackInfo(String slackUserId) { 100 | if (slackUserId == null) return Optional.empty(); 101 | 102 | Optional slackData = Cache.load(slackUserId); 103 | if (slackData.isPresent()) { 104 | try { 105 | return Optional.of(new ObjectMapper().readValue(slackData.get(), SlackInfo.class)); 106 | } catch (JsonProcessingException e) { 107 | e.printStackTrace(); 108 | } 109 | } 110 | return Optional.empty(); 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/Utils.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.apache.commons.lang3.StringUtils; 6 | import org.apache.commons.text.similarity.LevenshteinDistance; 7 | 8 | import java.io.*; 9 | import java.net.URL; 10 | import java.time.LocalDate; 11 | import java.time.format.DateTimeFormatter; 12 | import java.util.*; 13 | import java.util.concurrent.atomic.AtomicLong; 14 | import java.util.stream.Collectors; 15 | 16 | /** 17 | * A bunch of generic utilities used commonly throughout the project 18 | */ 19 | public class Utils { 20 | public static > Comparator> sortDescendingByValue() { 21 | return (Comparator> & Serializable) 22 | (c1, c2) -> c2.getValue().compareTo(c1.getValue()); 23 | } 24 | 25 | public static > Comparator> sortAscendingByValue() { 26 | return (Comparator> & Serializable) 27 | (c1, c2) -> c1.getValue().compareTo(c2.getValue()); 28 | } 29 | 30 | public static > Map sortByValue(Map map) { 31 | List> list = new ArrayList<>(map.entrySet()); 32 | list.sort(Map.Entry.comparingByValue()); 33 | 34 | Map result = new LinkedHashMap<>(); 35 | for (Map.Entry entry : list) { 36 | result.put(entry.getKey(), entry.getValue()); 37 | } 38 | 39 | return result; 40 | } 41 | 42 | /** 43 | * Reads given resource file as a string. 44 | * 45 | * @param fileName path to the resource file 46 | * @return the file's contents 47 | * @throws IOException if read fails for any reason 48 | */ 49 | public static String getResourceFileAsString(String fileName) throws IOException { 50 | ClassLoader classLoader = ClassLoader.getSystemClassLoader(); 51 | try (InputStream is = classLoader.getResourceAsStream(fileName)) { 52 | if (is == null) return null; 53 | try (InputStreamReader isr = new InputStreamReader(is); 54 | BufferedReader reader = new BufferedReader(isr)) { 55 | return reader.lines().collect(Collectors.joining(System.lineSeparator())); 56 | } 57 | } 58 | } 59 | 60 | public static String getPrettyJson(String rawJson) throws JsonProcessingException { 61 | ObjectMapper mapper = new ObjectMapper(); 62 | return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(mapper.readTree(rawJson)); 63 | } 64 | 65 | public static int levenshtein(String str1, String str2) { 66 | return levenshtein(str1, str2, 5); 67 | } 68 | 69 | public static int levenshtein(String str1, String str2, int threshold) { 70 | return new LevenshteinDistance(threshold).apply(str1, str2); 71 | } 72 | 73 | public static String getLastPathInUrl(String github) { 74 | if (StringUtils.isEmpty(github)) 75 | return null; 76 | 77 | try { 78 | URL url = new URL(github); 79 | String path = url.getPath().replace("/", ""); 80 | return path; 81 | } catch (Throwable t) { 82 | return null; 83 | } 84 | } 85 | 86 | public static double normalizedLevenshtein(String fromStr, String toStr, int maxDelta) { 87 | // Nulls should not match 88 | if (fromStr == null || toStr == null) 89 | return 0; 90 | 91 | double levDist = Utils.levenshtein(fromStr, toStr, maxDelta); 92 | if (levDist == -1) 93 | levDist = maxDelta; 94 | return (maxDelta - levDist) / (float)maxDelta; 95 | } 96 | 97 | public static void doEvery(AtomicLong lastReportTime, long delay, Runnable operation) { 98 | long timeDelta = System.currentTimeMillis() - lastReportTime.get(); 99 | if (timeDelta > delay) { 100 | operation.run(); 101 | lastReportTime.set(System.currentTimeMillis()); 102 | } 103 | } 104 | 105 | public static String safeToLower(String email) { 106 | if (email == null) return null; 107 | return email.toLowerCase(); 108 | } 109 | 110 | public static String sanitizeDate(String birthday) { 111 | try { 112 | DateTimeFormatter formatter = DateTimeFormatter.ofPattern("dd-MM-yyyy"); 113 | LocalDate.parse(birthday, formatter); 114 | return birthday; 115 | } catch (Throwable t) { 116 | try { 117 | DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd"); 118 | LocalDate.parse(birthday, formatter); 119 | return birthday; 120 | } catch (Throwable t2) { 121 | return null; 122 | } 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/agg/Aggregator.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.agg; 2 | 3 | import java.util.Map; 4 | import java.util.concurrent.ConcurrentHashMap; 5 | 6 | /** 7 | * Base class that defines an aggregation operation. Given a list of objects of type EntityType and a bucketing 8 | * function, returns a Map. Often AggregationType will be a numeric type used for occurrence 9 | * counting. 10 | * @param The type of the object you want to aggregate 11 | * @param The type you'll used for counting aggregations - often a basic numeric type like Integer/Double 12 | */ 13 | public abstract class Aggregator { 14 | protected ConcurrentHashMap results = new ConcurrentHashMap<>(); 15 | 16 | public void aggregate(EntityType entity, AggregationType value) { 17 | String key = bucket(entity); 18 | 19 | // Ignore unknown keys 20 | if (key == null) return; 21 | if (!results.containsKey(key)) { 22 | results.put(key, getDefaultValue()); 23 | } 24 | results.put(key, add(results.get(key), value)); 25 | } 26 | 27 | public abstract AggregationType add(AggregationType v1, AggregationType v2); 28 | 29 | public abstract String bucket(EntityType entity); 30 | 31 | public abstract AggregationType getDefaultValue(); 32 | 33 | public Map getResults() { 34 | return results; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/agg/DoubleAggregator.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.agg; 2 | 3 | import java.util.function.Function; 4 | 5 | /** 6 | * A concrete Double-type Aggregator that takes a custom bucketing function 7 | * @param The type of entity we'd like to aggregate 8 | */ 9 | public class DoubleAggregator extends Aggregator { 10 | private Function bucketer; 11 | 12 | /** 13 | * Creates a new DoubleAggregator 14 | * @param bucketer A user-defined bucketing function. 15 | */ 16 | public DoubleAggregator(Function bucketer) { 17 | this.bucketer = bucketer; 18 | } 19 | 20 | public Double add(Double v1, Double v2) { 21 | return v1 + v2; 22 | } 23 | 24 | @Override 25 | public String bucket(EntityType entity) { 26 | return bucketer.apply(entity); 27 | } 28 | 29 | public Double getDefaultValue() { 30 | return 0.0; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/agg/EntityDataExtractor.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.agg; 2 | 3 | /** 4 | * Interface to extract a DataType from a given EntityType 5 | * @param The type of entity you'd like to extract data from 6 | * @param The type of the data you'd like to extract 7 | */ 8 | public interface EntityDataExtractor { 9 | DataType getValue(EntityType entity); 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/agg/EntityProcessor.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.agg; 2 | 3 | import java.util.Map; 4 | import java.util.stream.Stream; 5 | 6 | /** 7 | * Defines an entity processor, which consists of an aggregator (for bucketing/counting data points) and an extractor 8 | * (for extracting/transforming entity data into an aggregatable form) 9 | * @param 10 | * @param 11 | */ 12 | public class EntityProcessor { 13 | private Aggregator aggregator; 14 | private EntityDataExtractor extractor; 15 | 16 | public EntityProcessor(Aggregator aggregator, EntityDataExtractor extractor) { 17 | this.aggregator = aggregator; 18 | this.extractor = extractor; 19 | } 20 | 21 | /** 22 | * Map over all entities, extracting data from each and aggregating it. Note that this is NOT a terminal stream 23 | * operation. 24 | * @param entities The entities you'd like to process 25 | * @return The input Stream 26 | */ 27 | public Stream process(Stream entities) { 28 | return entities.peek(entity -> aggregator.aggregate(entity, extractor.getValue(entity))); 29 | } 30 | 31 | public Map getResults() { 32 | return aggregator.getResults(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/conflation/MatchResult.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.conflation; 2 | 3 | /** 4 | * Immutable class that represents how closely related two entities are 5 | * @param 6 | * @param 7 | */ 8 | public class MatchResult { 9 | private FromType from; 10 | private ToType to; 11 | private double score; 12 | 13 | public MatchResult(FromType from, ToType to, double score) { 14 | this.from = from; 15 | this.to = to; 16 | this.score = score; 17 | } 18 | 19 | public FromType getFrom() { 20 | return from; 21 | } 22 | 23 | public ToType getTo() { 24 | return to; 25 | } 26 | 27 | public double getScore() { 28 | return score; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/conflation/MatchScorer.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.conflation; 2 | 3 | /** 4 | * An interface for defining a matching scorer which is responsible for relating two entities together via a scoring 5 | * metric. 6 | * @param 7 | * @param 8 | */ 9 | public interface MatchScorer { 10 | double score(FromType from, ToType to); 11 | double getThreshold(); 12 | } 13 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/conflation/Matcher.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.conflation; 2 | 3 | import java.util.HashSet; 4 | import java.util.Set; 5 | import java.util.stream.Collectors; 6 | 7 | /** 8 | * A helper class that defines an interface suitable for matching two datasets together. 9 | * @param 10 | * @param 11 | */ 12 | public class Matcher { 13 | private Set> results; 14 | private Set unmatchedFrom = new HashSet<>(); 15 | private Set unmatchedTo = new HashSet<>();; 16 | private int fromSize; 17 | private int toSize; 18 | 19 | /** 20 | * Constructor. Takes in the input datasets, and a scorer that will be used to associate them 21 | * @param fromSet The input 'from' set 22 | * @param toSet The input 'to' set 23 | * @param scorer A scorer that will rank/relate items from each set 24 | */ 25 | public Matcher(String matcherName, Set fromSet, Set toSet, MatchScorer scorer) { 26 | match(fromSet, toSet, scorer); 27 | calculateStatistics(matcherName, fromSet, toSet, results); 28 | } 29 | 30 | private void match(Set fromSet, Set toSet, MatchScorer scorer) { 31 | results = fromSet.parallelStream() 32 | .map(fromItem -> toSet.stream() 33 | .map(toItem -> new MatchResult<>(fromItem, toItem, scorer.score(fromItem, toItem))) 34 | .filter(match -> match.getScore() >= scorer.getThreshold()) 35 | .sorted(this::compareResults) 36 | ) 37 | .flatMap(sortedResults -> sortedResults.findFirst().stream()) 38 | .collect(Collectors.toSet()); 39 | 40 | unmatchedFrom.addAll(fromSet.stream().filter(f -> !resultSetContainsFrom(f, results)).collect(Collectors.toSet())); 41 | unmatchedTo.addAll(toSet.stream().filter(t -> !resultSetContainsTo(t, results)).collect(Collectors.toSet())); 42 | } 43 | 44 | private int compareResults(MatchResult m1, MatchResult m2) { 45 | return Double.compare(m1.getScore(), m2.getScore()); 46 | } 47 | 48 | /** 49 | * Retrieves results from matching 50 | * @return A Set of MatchResult objects 51 | */ 52 | public Set> getResults() { 53 | return results; 54 | } 55 | 56 | public void calculateStatistics(String matcherName, Set fromSet, Set toSet, Set> resultSet) { 57 | double fromSize = fromSet.size(); 58 | double toSize = toSet.size(); 59 | double fromsContainedInResults = fromSet.size() - unmatchedFrom.size(); 60 | double tosContainedInResults = toSet.size() - unmatchedTo.size(); 61 | double totalResults = resultSet.size(); 62 | 63 | System.out.printf("RUNNING MATCHER \"%s\"\r\n", matcherName); 64 | System.out.printf("\tFROM coverage: %.02f%%\r\n", (fromsContainedInResults / fromSize) * 100.0f); 65 | System.out.printf("\t TO coverage: %.02f%%\r\n", (tosContainedInResults / toSize) * 100.0f); 66 | } 67 | private boolean resultSetContainsFrom(FromType f, Set> resultSet) { 68 | return resultSet.stream().anyMatch(r -> r.getFrom() == f); 69 | } 70 | private boolean resultSetContainsTo(ToType t, Set> resultSet) { 71 | return resultSet.stream().anyMatch(r -> r.getTo() == t); 72 | } 73 | public Set getUnmatchedFrom() { 74 | return unmatchedFrom; 75 | } 76 | 77 | public Set getUnmatchedTo() { 78 | return unmatchedTo; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/elasticsearch/ESCommand.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.elasticsearch; 2 | 3 | 4 | import picocli.CommandLine; 5 | 6 | import java.util.concurrent.Callable; 7 | 8 | /** 9 | * Base class for a picocli command that expects to sync data to (or query) an elasticsearch cluster 10 | */ 11 | public abstract class ESCommand implements Callable { 12 | @CommandLine.Option(names = {"--eshost"}, description = "Hostname of elasticsearch cluster to use for indexing") 13 | protected String esHostname = "localhost"; 14 | 15 | @CommandLine.Option(names = {"--esport"}, description = "The port of the elasticsearch cluster") 16 | protected Integer esPort = 9200; 17 | 18 | @CommandLine.Option(names = {"--esusername"}, description = "The username for the elasticsearch cluster") 19 | protected String esUsername = "unknown"; 20 | 21 | @CommandLine.Option(names = {"--espassword"}, description = "The password for the elasticsearch cluster") 22 | protected String esPassword = "not_applicable"; 23 | 24 | @CommandLine.Option(names = {"--esfingerprint"}, description = "The fingerprint for the elasticsearch cluster") 25 | protected String esFingerprint = "please_fill_me_in"; 26 | 27 | @CommandLine.Option(names = {"--esindex"}, description = "The elasticsearch index that you'd like to write data to") 28 | protected String esIndex = "UNKNOWN"; 29 | 30 | @CommandLine.Option(names = {"--google-geocoding-api-key"}, description = "Google geocoding API key") 31 | protected String googleGeocodingApiKey = "please_fill_me_in"; 32 | 33 | @CommandLine.Option(names = {"--github-api-key"}, description = "Github API key") 34 | protected String githubApiKey = "please_fill_me_in"; 35 | 36 | @CommandLine.Option(names = {"--slack-api-key"}, description = "Slack API key") 37 | protected String slackApiKey = "please_fill_me_in"; 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/elasticsearch/ESUtils.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.elasticsearch; 2 | 3 | import co.elastic.clients.elasticsearch.ElasticsearchClient; 4 | import co.elastic.clients.elasticsearch._types.query_dsl.Query; 5 | import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; 6 | import co.elastic.clients.elasticsearch.core.DeleteByQueryRequest; 7 | import co.elastic.clients.elasticsearch.core.DeleteByQueryResponse; 8 | import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; 9 | import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest; 10 | import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse; 11 | import co.elastic.clients.json.jackson.JacksonJsonpMapper; 12 | import co.elastic.clients.transport.TransportUtils; 13 | import co.elastic.clients.transport.rest_client.RestClientTransport; 14 | import org.apache.http.HttpHost; 15 | import org.apache.http.auth.AuthScope; 16 | import org.apache.http.auth.UsernamePasswordCredentials; 17 | import org.apache.http.conn.ssl.SSLConnectionSocketFactory; 18 | import org.apache.http.impl.client.BasicCredentialsProvider; 19 | import org.elasticsearch.client.RestClient; 20 | import org.jetbrains.annotations.NotNull; 21 | 22 | import javax.net.ssl.SSLContext; 23 | import java.io.IOException; 24 | import java.util.HashMap; 25 | 26 | /** 27 | * Convenience/utility methods for interacting with an elasticsearch cluster 28 | */ 29 | public class ESUtils { 30 | public static void clearIndex(ElasticsearchClient esClient, String userIndex) throws IOException { 31 | System.out.printf("Deleting all documents in index %s\n", userIndex); 32 | Query query = QueryBuilders.matchAll().build()._toQuery(); 33 | DeleteByQueryRequest dbyquery = DeleteByQueryRequest 34 | .of(fn -> fn.query(query).index(userIndex)); 35 | 36 | DeleteByQueryResponse response = esClient.deleteByQuery(dbyquery); 37 | System.out.printf("Deleting index took %dms\n", response.took()); 38 | } 39 | 40 | public static void deleteIndex(ElasticsearchClient esClient, String userIndex) throws IOException { 41 | DeleteIndexRequest dir = new DeleteIndexRequest.Builder().index(userIndex).build(); 42 | DeleteIndexResponse response = esClient.indices().delete(dir); 43 | } 44 | 45 | public static void addIndex(ElasticsearchClient esClient, String userIndex) throws IOException { 46 | CreateIndexRequest request = new CreateIndexRequest.Builder() 47 | .index(userIndex) 48 | .build(); 49 | esClient.indices().create(request); 50 | } 51 | 52 | /** 53 | * Create a column name -> column index mapping 54 | * @param columns An array of strings that represent column names 55 | * @return A HashMap that associates column names with a corresponding index 56 | */ 57 | @NotNull 58 | public static HashMap getIndexMapping(String[] columns) { 59 | HashMap columnIndices = new HashMap<>(); 60 | int index = 0; 61 | for(String column : columns) { 62 | columnIndices.put(column, index); 63 | index++; 64 | } 65 | return columnIndices; 66 | } 67 | 68 | public static ElasticsearchClient createElasticsearchClient(String hostname, int port, String username, String password, String fingerprint) throws IOException { 69 | SSLContext sslContext = TransportUtils.sslContextFromCaFingerprint(fingerprint); 70 | BasicCredentialsProvider credsProv = new BasicCredentialsProvider(); 71 | credsProv.setCredentials( 72 | AuthScope.ANY, new UsernamePasswordCredentials(username, password) 73 | ); 74 | 75 | RestClient restClient = RestClient 76 | .builder(new HttpHost(hostname, port, "https")) 77 | .setHttpClientConfigCallback(hc -> hc 78 | .setSSLContext(sslContext) 79 | .setSSLHostnameVerifier(SSLConnectionSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER) 80 | .setDefaultCredentialsProvider(credsProv) 81 | ) 82 | .build(); 83 | 84 | // Create the transport and the API client 85 | return new ElasticsearchClient(new RestClientTransport(restClient, new JacksonJsonpMapper())); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/elasticsearch/InitIndexCommand.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.elasticsearch; 2 | 3 | import co.elastic.clients.elasticsearch.ElasticsearchClient; 4 | import picocli.CommandLine; 5 | import static com.hackclub.common.elasticsearch.ESUtils.addIndex; 6 | import static com.hackclub.common.elasticsearch.ESUtils.deleteIndex; 7 | 8 | /** 9 | * A picocli command that completely deletes and re-adds the associated ES index. 10 | */ 11 | @CommandLine.Command(name = "init") 12 | public class InitIndexCommand extends ESCommand { 13 | @Override 14 | public Integer call() throws Exception { 15 | ElasticsearchClient esClient = ESUtils.createElasticsearchClient(esHostname, esPort, esUsername, esPassword, esFingerprint); 16 | deleteIndex(esClient, esIndex); 17 | addIndex(esClient, esIndex); 18 | 19 | // TODO: I believe we need to add geopoint mappings to the index after recreation for things to work correctly! 20 | return 0; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/file/BlobStore.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.file; 2 | 3 | import org.apache.commons.lang3.NotImplementedException; 4 | 5 | import java.io.File; 6 | import java.net.URI; 7 | 8 | /** 9 | * Filesystem abstraction using URIs. 10 | * Supports: 11 | * Basic filesystem 12 | * S3 (future) 13 | */ 14 | public class BlobStore { 15 | public static File load(URI path) { 16 | String scheme = path.getScheme(); 17 | 18 | switch(scheme) { 19 | case "file": return new File(path.getPath()); 20 | default: throw new NotImplementedException("Not implemented!"); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/file/Cache.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.file; 2 | 3 | import org.apache.commons.codec.digest.DigestUtils; 4 | import org.apache.commons.io.FileUtils; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | import java.io.File; 8 | import java.io.IOException; 9 | import java.nio.charset.StandardCharsets; 10 | import java.nio.file.Files; 11 | import java.nio.file.Paths; 12 | import java.util.Optional; 13 | import java.util.concurrent.atomic.AtomicLong; 14 | 15 | public class Cache { 16 | private static AtomicLong cacheCount = new AtomicLong(0); 17 | public static void save(String key, String data) throws IOException { 18 | FileUtils.writeStringToFile(new File(getCachedPath(key)), data, StandardCharsets.UTF_8); 19 | } 20 | 21 | public static Optional load(String key) { 22 | String path = getCachedPath(key); 23 | try { 24 | return Optional.of(new String(Files.readAllBytes(Paths.get(path)))); 25 | } catch (IOException e) { 26 | // do nothing 27 | } 28 | return Optional.empty(); 29 | } 30 | 31 | @NotNull 32 | private static String getCachedPath(String key) { 33 | String md5Hex = DigestUtils.md5Hex(key).toUpperCase(); 34 | return String.format("/var/data/chronicle/cache/%s", md5Hex); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/com/hackclub/common/geo/Geocoder.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.geo; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.google.maps.GeoApiContext; 5 | import com.google.maps.GeocodingApi; 6 | import com.google.maps.errors.ApiException; 7 | import com.google.maps.model.GeocodingResult; 8 | import com.hackclub.clubs.models.GeoPoint; 9 | import com.hackclub.clubs.models.GithubInfo; 10 | import com.hackclub.common.file.Cache; 11 | 12 | import java.io.IOException; 13 | import java.util.Optional; 14 | 15 | /** 16 | * Integrates with Google's geocoding APIs. Primarily, we want to transform full address strings to lat/lng coords 17 | */ 18 | public class Geocoder { 19 | private static GeoApiContext geoApi = null; 20 | 21 | public static void initialize(String apiKey) { 22 | geoApi = new GeoApiContext.Builder().apiKey(apiKey).build(); 23 | } 24 | 25 | /** 26 | * Convert an address string to a GeoPoint 27 | * @param address A full or partial address 28 | * @return An Optional GeoPoint 29 | * @throws IOException 30 | * @throws InterruptedException 31 | * @throws ApiException 32 | */ 33 | public static Optional geocode(String address) throws IOException, InterruptedException, ApiException { 34 | String geocodingCacheKey = "geocoding_" + address; 35 | 36 | Optional cachedGeo = Cache.load(geocodingCacheKey); 37 | if (cachedGeo.isPresent()) { 38 | return Optional.of(new ObjectMapper().readValue(cachedGeo.get(), GeoPoint.class)); 39 | } else { 40 | GeocodingResult[] results = GeocodingApi.geocode(geoApi, address).await(); 41 | if (results.length == 0) 42 | return Optional.empty(); 43 | 44 | GeoPoint ret = new GeoPoint(results[0].geometry.location.lat, results[0].geometry.location.lng); 45 | System.out.println("Caching geopoint"); 46 | Cache.save(geocodingCacheKey, new ObjectMapper().writeValueAsString(ret)); 47 | 48 | return Optional.of(ret); 49 | } 50 | } 51 | 52 | public static void shutdown() { 53 | geoApi.shutdown(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/resources/es/geomapping.esquery: -------------------------------------------------------------------------------- 1 | PUT search-users/_mapping 2 | { 3 | "dynamic": "true", 4 | "properties": { 5 | "geolocation": { 6 | "type": "geo_point" 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /src/main/resources/es/settings.esquery: -------------------------------------------------------------------------------- 1 | PUT search-users/_settings 2 | { 3 | "index.mapping.total_fields.limit": 10000 4 | } 5 | -------------------------------------------------------------------------------- /src/main/resources/github_user_query.graphql: -------------------------------------------------------------------------------- 1 | { 2 | search(query: "%s", type: USER, first: 1) { 3 | edges { 4 | node { 5 | ... on User { 6 | id 7 | bio 8 | company 9 | topRepositories(last: 100, orderBy: {field: PUSHED_AT, direction: DESC}) { 10 | edges { 11 | node { 12 | name 13 | description 14 | homepageUrl 15 | isFork 16 | isInOrganization 17 | isEmpty 18 | pushedAt 19 | } 20 | } 21 | } 22 | contributionsCollection { 23 | pullRequestContributions(last: 100, orderBy: {direction: DESC}) { 24 | nodes { 25 | pullRequest { 26 | createdAt 27 | repository { 28 | id 29 | owner { 30 | id 31 | login 32 | } 33 | name 34 | primaryLanguage { 35 | name 36 | } 37 | } 38 | } 39 | } 40 | totalCount 41 | } 42 | } 43 | issues(first: 50, orderBy: {field: UPDATED_AT, direction: DESC}) { 44 | edges { 45 | node { 46 | title 47 | body 48 | } 49 | } 50 | totalCount 51 | } 52 | } 53 | } 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /src/test/java/com/hackclub/common/agg/DoubleAggregatorTest.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.agg; 2 | 3 | import org.junit.jupiter.api.Assertions; 4 | import org.junit.jupiter.api.Test; 5 | 6 | import java.util.function.Function; 7 | 8 | class DoubleAggregatorTest { 9 | @Test 10 | void basic() { 11 | // Buckets string values by their length 12 | Function bucketer = str -> Integer.toString(str.length()); 13 | DoubleAggregator characterCounter = new DoubleAggregator<>(bucketer); 14 | characterCounter.aggregate("testing1", 1.0); 15 | characterCounter.aggregate("testing2", 1.0); 16 | Assertions.assertEquals(1, characterCounter.getResults().size()); 17 | 18 | characterCounter.aggregate("test", 1.0); 19 | Assertions.assertEquals(2, characterCounter.getResults().size()); 20 | 21 | Assertions.assertEquals(characterCounter.getResults().get("8"), 2); 22 | } 23 | } -------------------------------------------------------------------------------- /src/test/java/com/hackclub/common/agg/EntityProcessorTest.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.agg; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import java.util.function.Function; 6 | 7 | import static org.junit.jupiter.api.Assertions.*; 8 | 9 | class EntityProcessorTest { 10 | @Test 11 | public void basic() { 12 | Function strLengthBucketer = str -> Integer.toString(str.length()); 13 | Aggregator aggregator = new DoubleAggregator<>(strLengthBucketer); 14 | EntityDataExtractor extractor = entity -> null; 15 | EntityProcessor processor = new EntityProcessor<>(aggregator, extractor); 16 | } 17 | 18 | } -------------------------------------------------------------------------------- /src/test/java/com/hackclub/common/conflation/MatcherTest.java: -------------------------------------------------------------------------------- 1 | package com.hackclub.common.conflation; 2 | 3 | import com.hackclub.common.Utils; 4 | import org.jetbrains.annotations.NotNull; 5 | import org.junit.jupiter.api.Assertions; 6 | import org.junit.jupiter.api.Test; 7 | 8 | import java.util.HashMap; 9 | import java.util.HashSet; 10 | import java.util.Iterator; 11 | import java.util.Set; 12 | 13 | class MatcherTest { 14 | 15 | @Test 16 | void basics() { 17 | Set fromSet = new HashSet<>(); 18 | From from1 = new From("test1", "test1@test.com", 16); 19 | From from2 = new From("test2", "test2@test.com", 17); 20 | fromSet.add(from1); 21 | fromSet.add(from2); 22 | Set toSet = new HashSet<>(); 23 | To to1 = new To("test1", "test1@test.com", 16); 24 | To to2 = new To("test2", "test2@test.com", 17); 25 | toSet.add(to1); 26 | toSet.add(to2); 27 | 28 | HashMap expectedResults = new HashMap<>(); 29 | expectedResults.put(from1, to1); 30 | expectedResults.put(from2, to2); 31 | 32 | Set> results = new Matcher<>("MatcherTest", fromSet, toSet, createBasicScorer(0)).getResults(); 33 | Assertions.assertEquals(results.size(), 2); 34 | for (Iterator> it = results.iterator(); it.hasNext(); ) { 35 | MatchResult match = it.next(); 36 | Assertions.assertEquals(expectedResults.get(match.getFrom()), match.getTo()); 37 | Assertions.assertEquals(1.0, match.getScore()); 38 | } 39 | } 40 | 41 | @Test 42 | void testSlightDifferences() { 43 | Set fromSet = new HashSet<>(); 44 | From from1 = new From("test1", "test1@test.com", 15); 45 | From from2 = new From("test2", "test2@test.com", 17); 46 | fromSet.add(from1); 47 | fromSet.add(from2); 48 | Set toSet = new HashSet<>(); 49 | To to1 = new To("test1", "test1@test.com", 16); 50 | To to2 = new To("test2", "test2@test.com", 18); 51 | toSet.add(to1); 52 | toSet.add(to2); 53 | 54 | HashMap expectedResults = new HashMap<>(); 55 | expectedResults.put(from1, to1); 56 | expectedResults.put(from2, to2); 57 | 58 | Set> results = new Matcher<>("MatcherTestSlightDifferences", fromSet, toSet, createBasicScorer(0)).getResults(); 59 | Assertions.assertEquals(results.size(), 2); 60 | for (Iterator> it = results.iterator(); it.hasNext(); ) { 61 | MatchResult match = it.next(); 62 | Assertions.assertEquals(expectedResults.get(match.getFrom()), match.getTo()); 63 | Assertions.assertNotEquals(1.0, match.getScore()); 64 | } 65 | } 66 | 67 | @Test 68 | void testMassiveDifferences() { 69 | Set fromSet = new HashSet<>(); 70 | From from1 = new From("test1", "test1@wefweftest.com", 15); 71 | From from2 = new From("test2", "test2@test.com", 14); 72 | fromSet.add(from1); 73 | fromSet.add(from2); 74 | Set toSet = new HashSet<>(); 75 | To to1 = new To("test1", "test1@test.com", 20); 76 | To to2 = new To("test2", "test2@twefwefest.com", 18); 77 | toSet.add(to1); 78 | toSet.add(to2); 79 | 80 | HashMap expectedResults = new HashMap<>(); 81 | expectedResults.put(from1, to1); 82 | expectedResults.put(from2, to2); 83 | 84 | Set> results = new Matcher<>("MatcherTestMassiveDifferences", fromSet, toSet, createBasicScorer(0)).getResults(); 85 | Assertions.assertEquals(0, results.size()); 86 | } 87 | 88 | 89 | @NotNull 90 | private static MatchScorer createBasicScorer(float threshold) { 91 | return new MatchScorer<>() { 92 | @Override 93 | public double score(From from, To to) { 94 | int textDiff = Utils.levenshtein(from.name, to.name); 95 | int ageDiff = Math.abs(from.age - to.age); 96 | double ret = 1.0f - (textDiff + ageDiff); 97 | return ret; 98 | } 99 | 100 | @Override 101 | public double getThreshold() { 102 | return threshold; 103 | } 104 | }; 105 | } 106 | 107 | private static class From { 108 | private String name; 109 | private String emailAddress; 110 | private int age; 111 | 112 | public From(String name, String emailAddress, int age) { 113 | this.name = name; 114 | this.emailAddress = emailAddress; 115 | this.age = age; 116 | } 117 | 118 | public String getName() { 119 | return name; 120 | } 121 | 122 | public String getEmailAddress() { 123 | return emailAddress; 124 | } 125 | 126 | public int getAge() { 127 | return age; 128 | } 129 | } 130 | 131 | private static class To { 132 | private String name; 133 | private String emailAddress; 134 | private int age; 135 | 136 | public To(String name, String emailAddress, int age) { 137 | this.name = name; 138 | this.emailAddress = emailAddress; 139 | this.age = age; 140 | } 141 | 142 | public String getName() { 143 | return name; 144 | } 145 | 146 | public String getEmailAddress() { 147 | return emailAddress; 148 | } 149 | 150 | public int getAge() { 151 | return age; 152 | } 153 | } 154 | } --------------------------------------------------------------------------------