├── log └── .gitkeep ├── sql ├── ddl │ └── .gitkeep └── samples │ ├── avg-rows-per-minute.mssql.sql │ ├── events-in-last-minute.mssql.sql │ └── daily-active-users.mssql.sql ├── cfg ├── ledbelly.yml.example ├── sqs.yml.example └── database.yml.example ├── Rakefile ├── src ├── lib │ ├── tasks │ │ ├── reset_logs.rake │ │ ├── create_tables.rake │ │ ├── reduce_logs.rake │ │ └── alter_tables.rake │ └── services │ │ └── maxmind.rb ├── ledbelly_worker.rb ├── schemas │ └── custom.rb ├── actions │ ├── live_stream.rb │ └── import_sql.rb ├── ledbelly_settings.rb ├── ledbelly_support.rb └── events │ ├── ims_caliper.rb │ └── canvas_raw.rb ├── ledbelly.rb ├── .gitignore ├── Gemfile ├── LICENSE ├── SYSTEMD.md ├── Gemfile.lock ├── .rubocop.yml └── README.md /log/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sql/ddl/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /cfg/ledbelly.yml.example: -------------------------------------------------------------------------------- 1 | services: 2 | - maxmind -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | require 'rake/testtask' 3 | Rake.add_rakelib 'src/lib/tasks' -------------------------------------------------------------------------------- /src/lib/tasks/reset_logs.rake: -------------------------------------------------------------------------------- 1 | task :reset_logs do 2 | Dir.glob('./log/*.log') do |logfile| 3 | begin 4 | open(logfile, 'w') { |f| f.puts(nil) } 5 | puts logfile 6 | rescue => e 7 | puts e 8 | end 9 | end 10 | end -------------------------------------------------------------------------------- /ledbelly.rb: -------------------------------------------------------------------------------- 1 | require 'shoryuken' 2 | require 'sequel' 3 | require 'logger' 4 | require_relative 'src/ledbelly_settings' 5 | require_relative 'src/ledbelly_support' 6 | require_relative 'src/ledbelly_worker' 7 | 8 | # load sub services 9 | LED['services']&.each do |service| 10 | require_relative "src/lib/services/#{service}.rb" 11 | end -------------------------------------------------------------------------------- /sql/samples/avg-rows-per-minute.mssql.sql: -------------------------------------------------------------------------------- 1 | -- average rows per minute 2 | SELECT 3 | AVG(a.total_per) 4 | FROM ( 5 | SELECT 6 | DATEADD(minute, DATEDIFF(minute, 0, processed_at), 0) AS day_minute 7 | , COUNT(*) AS total_per 8 | FROM live_stream 9 | GROUP BY 10 | DATEADD(minute, DATEDIFF(minute, 0, processed_at), 0) 11 | ) a -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # risky secrety stuff 2 | database.yml 3 | sqs.yml 4 | .vscode-upload.json 5 | 6 | # local 7 | shoryuken.pid 8 | *.log 9 | sql/ddl/*.sql 10 | 11 | # dev stuff 12 | dev/ 13 | todo.txt 14 | 15 | # OS generated files # 16 | ###################### 17 | .DS_Store 18 | .DS_Store? 19 | ._* 20 | .Spotlight-V100 21 | .Trashes 22 | ehthumbs.db 23 | Thumbs.db 24 | nohup.out -------------------------------------------------------------------------------- /cfg/sqs.yml.example: -------------------------------------------------------------------------------- 1 | # aws 2 | aws: 3 | region: us-west-2 4 | access_key_id: zxcvbnmpoiuytrewq 5 | secret_access_key: asdfghjklpoiuytrewq 6 | 7 | # shoryuken 8 | # https://github.com/phstc/shoryuken/wiki/Shoryuken-options 9 | logfile: ./shoryuken.log # /dev/null #./shoryuken.log 10 | pidfile: ./shoryuken.pid 11 | concurrency: 20 12 | delay: 0 13 | queues: 14 | - canvas-live-events-edu -------------------------------------------------------------------------------- /sql/samples/events-in-last-minute.mssql.sql: -------------------------------------------------------------------------------- 1 | -- events processed within the last 60 seconds 2 | SELECT 3 | event_name 4 | , MAX(processed_at) last_processed_at 5 | , MAX(event_time_local) last_event_time_local 6 | , COUNT(*) messages 7 | FROM live_stream 8 | WHERE processed_at >= DATEADD(minute, -1, GETDATE()) 9 | GROUP BY 10 | event_name 11 | ORDER BY 12 | last_processed_at DESC; -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | # core 3 | gem 'activesupport' 4 | gem 'aws-sdk-sqs' 5 | gem 'sequel' 6 | gem 'shoryuken' 7 | gem 'tiny_tds' 8 | # gem 'mysql2' 9 | # gem 'pg' 10 | # gem 'ruby-oci8' 11 | 12 | # services 13 | # gem 'httparty' 14 | # gem 'maxmind-db' 15 | 16 | # rake and tasks 17 | gem "rake", ">= 12.3.3" 18 | gem 'hashdiff' 19 | gem 'jaro_winkler' 20 | 21 | # code contributing 22 | gem 'rubocop' -------------------------------------------------------------------------------- /cfg/database.yml.example: -------------------------------------------------------------------------------- 1 | # production 2 | adapter: tinytds 3 | host: db.hostname.tld 4 | data: CanvasLMS 5 | user: cda 6 | pass: password 7 | max_connections: 10 8 | 9 | # development 10 | # adapter: tinytds 11 | # host: db.hostname.tld 12 | # data: CanvasLMS 13 | # user: cda 14 | # pass: password 15 | # max_connections: 10 16 | 17 | # adapter: mysql2 18 | # host: db.hostname.tld 19 | # data: CanvasLMS 20 | # user: cda 21 | # pass: password 22 | # max_connections: 10 23 | 24 | # adapter: postgres 25 | # host: db.hostname.tld 26 | # data: CanvasLMS 27 | # user: cda 28 | # pass: password 29 | # max_connections: 10 30 | 31 | # adapter: oracle 32 | # host: db.hostname.tld 33 | # data: CanvasLMS 34 | # user: cda 35 | # pass: password 36 | # max_connections: 10 -------------------------------------------------------------------------------- /sql/samples/daily-active-users.mssql.sql: -------------------------------------------------------------------------------- 1 | ;WITH daily_users AS ( 2 | -- each user by each date 3 | SELECT DISTINCT 4 | CONVERT(date, event_time_local) AS date 5 | , user_id_meta 6 | FROM live_stream 7 | ), daily_active_users AS ( 8 | -- count the number of users for each date 9 | SELECT 10 | date 11 | , COUNT(user_id_meta) AS dau 12 | FROM daily_users 13 | GROUP BY 14 | date 15 | ), daily_avg_users AS ( 16 | -- 7 day rolling average users per day 17 | SELECT 18 | date 19 | , AVG(dau) OVER (ORDER BY date ASC ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS avg_dau 20 | FROM daily_active_users 21 | ) 22 | 23 | SELECT 24 | dau.date 25 | , DATEPART(dw, dau.date) week_day 26 | , dau.dau 27 | , avg_dau.avg_dau 28 | FROM daily_active_users dau 29 | JOIN daily_avg_users avg_dau ON avg_dau.date = dau.date 30 | WHERE dau.date >= GETDATE()-30 31 | ORDER BY dau.date -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Clark County School District, created by Robert Carroll 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /SYSTEMD.md: -------------------------------------------------------------------------------- 1 | ### Service Manager 2 | systemd and systemctl allow you to add LEDbelly to the systems service manager. This enables LEDbelly to be reloaded if it crashes, disconnects or your system reboots, helping to keep your Live Events working in real-time. 3 | 4 | 5 | 1) Edit `sudo nano /etc/systemd/system/ledbelly.service` 6 | 7 | with the following, adjusting your installation path `/canvas/live-events/` 8 | ``` 9 | [Unit] 10 | Description=LEDbelly - Live Events Daemon 11 | Requires=network.target 12 | After=syslog.target network-online.target 13 | 14 | [Service] 15 | Type=forking 16 | WorkingDirectory=/canvas/live-events 17 | ExecStart=/usr/bin/bash -lc 'bundle exec shoryuken -r ./ledbelly -C cfg/sqs.yml -L /dev/null -d' 18 | PIDFile=/canvas/live-events/log/shoryuken.pid 19 | Restart=on-failure 20 | RestartSec=1800 21 | KillMode=process 22 | 23 | [Install] 24 | WantedBy=multi-user.target 25 | ``` 26 | 27 | 2) Reload and Enable 28 | 29 | `sudo systemctl daemon-reload; sudo systemctl enable ledbelly` 30 | 31 | 3) Start the service 32 | 33 | `sudo systemctl start ledbelly` 34 | 35 | 4) Check the status 36 | 37 | `sudo systemctl status ledbelly.service` 38 | 39 | 5) Stop and Restart 40 | 41 | `sudo systemctl stop ledbelly` 42 | 43 | `sudo systemctl restart ledbelly` -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | activesupport (5.2.4.3) 5 | concurrent-ruby (~> 1.0, >= 1.0.2) 6 | i18n (>= 0.7, < 2) 7 | minitest (~> 5.1) 8 | tzinfo (~> 1.1) 9 | ast (2.4.0) 10 | aws-eventstream (1.0.1) 11 | aws-partitions (1.141.0) 12 | aws-sdk-core (3.46.2) 13 | aws-eventstream (~> 1.0) 14 | aws-partitions (~> 1.0) 15 | aws-sigv4 (~> 1.0) 16 | jmespath (~> 1.0) 17 | aws-sdk-sqs (1.10.0) 18 | aws-sdk-core (~> 3, >= 3.39.0) 19 | aws-sigv4 (~> 1.0) 20 | aws-sigv4 (1.0.3) 21 | concurrent-ruby (1.1.4) 22 | hashdiff (1.0.0) 23 | i18n (1.5.3) 24 | concurrent-ruby (~> 1.0) 25 | jaro_winkler (1.5.4) 26 | jmespath (1.4.0) 27 | minitest (5.11.3) 28 | parallel (1.19.1) 29 | parser (2.7.0.4) 30 | ast (~> 2.4.0) 31 | rainbow (3.0.0) 32 | rake (13.0.1) 33 | rexml (3.2.4) 34 | rubocop (0.80.1) 35 | jaro_winkler (~> 1.5.1) 36 | parallel (~> 1.10) 37 | parser (>= 2.7.0.1) 38 | rainbow (>= 2.2.2, < 4.0) 39 | rexml 40 | ruby-progressbar (~> 1.7) 41 | unicode-display_width (>= 1.4.0, < 1.7) 42 | ruby-progressbar (1.10.1) 43 | sequel (5.17.0) 44 | shoryuken (4.0.3) 45 | aws-sdk-core (>= 2) 46 | concurrent-ruby 47 | thor 48 | thor (0.20.3) 49 | thread_safe (0.3.6) 50 | tiny_tds (2.1.2) 51 | tzinfo (1.2.5) 52 | thread_safe (~> 0.1) 53 | unicode-display_width (1.6.1) 54 | 55 | PLATFORMS 56 | ruby 57 | 58 | DEPENDENCIES 59 | activesupport 60 | aws-sdk-sqs 61 | hashdiff 62 | jaro_winkler 63 | rake (>= 12.3.3) 64 | rubocop 65 | sequel 66 | shoryuken 67 | tiny_tds 68 | 69 | BUNDLED WITH 70 | 2.0.2 71 | -------------------------------------------------------------------------------- /src/ledbelly_worker.rb: -------------------------------------------------------------------------------- 1 | require_relative 'events/canvas_raw' 2 | require_relative 'events/ims_caliper' 3 | require_relative 'actions/import_sql' 4 | require_relative 'actions/live_stream' 5 | 6 | class LiveEvents 7 | include Shoryuken::Worker 8 | include CanvasRawEvents 9 | include IMSCaliperEvents 10 | include SQLInsertEvent 11 | include LiveStream 12 | 13 | shoryuken_options queue: SQS_CFG['queues'][0], auto_delete: true 14 | 15 | # terminal output, if terminal/interactive 16 | puts 'LEDbelly loaded, consuming events...' if $stdout.isatty 17 | 18 | # https://github.com/phstc/shoryuken/wiki/Worker-options#body_parser 19 | shoryuken_options body_parser: :json 20 | shoryuken_options body_parser: ->(sqs_msg){ REXML::Document.new(sqs_msg.body) } 21 | shoryuken_options body_parser: JSON 22 | 23 | # https://github.com/phstc/shoryuken/blob/master/lib/shoryuken/worker/inline_executor.rb#L8 24 | def perform(sqs_msg, _body) 25 | 26 | # event attributes 27 | event_name = sqs_msg.message_attributes.dig('event_name', 'string_value') 28 | event_time = sqs_msg.message_attributes.dig('event_time', 'string_value') 29 | event_data = JSON.parse(sqs_msg.body) 30 | 31 | begin 32 | # handle caliper 33 | if event_data['dataVersion'] == 'http://purl.imsglobal.org/ctx/caliper/v1p1' 34 | 35 | # pass to parser 36 | event_parsed = _caliper(event_name, event_data) 37 | if event_parsed.is_a?(Hash) 38 | # import to db 39 | import(event_name, event_time, event_data, event_parsed) 40 | end 41 | 42 | # handle canvas raw 43 | else 44 | 45 | # parse event data 46 | event_parsed = _canvas(event_data) 47 | if event_parsed.is_a?(Hash) 48 | # import to db 49 | import(event_name, event_time, event_data, event_parsed) 50 | # extras 51 | live_stream(event_name, event_time, event_data) 52 | end 53 | 54 | end 55 | rescue => e 56 | warn "#{event_name} #{default_timezone(event_time)}\n#{e}" 57 | warn e.backtrace 58 | raise 59 | end 60 | end 61 | 62 | end -------------------------------------------------------------------------------- /src/lib/tasks/create_tables.rake: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | 3 | task :create_tables do 4 | 5 | #DB_CFG = YAML.load_file('./cfg/database.yml') 6 | adapter = DB_CFG['adapter'] 7 | # adapter = 'postgres' 8 | # the name of the schema/database name... might be 'dbo' for tinytds or whatever you named the db 9 | dbname = DB_CFG['data'] 10 | 11 | primary_keys = { 12 | mysql2: 'BIGINT NOT NULL AUTO_INCREMENT', 13 | oracle: 'NUMBER GENERATED ALWAYS AS IDENTITY', 14 | postgres: 'SERIAL PRIMARY KEY', 15 | tinytds: 'BIGINT IDENTITY(1,1) PRIMARY KEY' 16 | } 17 | 18 | Dir.glob('./src/schemas/*.rb') do |schema_hash| 19 | require schema_hash 20 | format = File.basename(schema_hash, ".rb") 21 | ddlout = [] 22 | 23 | # tables 24 | $schema.each do |table, columns| 25 | # create table 26 | tbobj = [ 'mysql2', 'tinytds' ].include?(adapter) ? "#{dbname}.#{table}" : table 27 | if adapter == 'tinytds' 28 | ddlout << "IF OBJECT_ID('#{dbname}.#{table}', 'U') IS NOT NULL DROP TABLE #{dbname}.#{table};\n" 29 | ddlout << "IF NOT EXISTS (SELECT * FROM sysobjects WHERE name = '#{table}' AND xtype = 'U')\n" 30 | else 31 | ddlout << "DROP TABLE IF EXISTS #{tbobj};\n" 32 | end 33 | ddlout << "CREATE TABLE #{tbobj} (\n" 34 | 35 | # columns 36 | colout = [] 37 | pk_col = '' 38 | columns.each do |column, params| 39 | # primary key 40 | if params[:primary_key] == true 41 | colout << "\t#{column} #{primary_keys[adapter.to_sym]}" 42 | pk_col = column 43 | # others 44 | else 45 | 46 | # handle type swapping 47 | case params[:type] 48 | when 'string' 49 | if params[:size] == 'MAX' && adapter != 'tinytds' 50 | mysql_utf8 = adapter == 'mysql2' ? ' CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci' : nil 51 | colout << "\t#{column} TEXT#{mysql_utf8}" 52 | else 53 | coltype = [ 'tinytds' ].include?(adapter) && params[:mbstr] == true ? 'NVARCHAR' : 'VARCHAR' 54 | colout << "\t#{column} #{coltype.upcase}(#{params[:size]})" 55 | end 56 | when 'datetime' 57 | dtype = [ 'mysql2', 'tinytds' ].include?(adapter) ? 'DATETIME' : 'TIMESTAMP' 58 | colout << "\t#{column} #{dtype}" 59 | 60 | # no specific swapping 61 | else 62 | coltype = adapter != 'oracle' ? params[:type] : 'NUMBER' 63 | # size is set 64 | colsize = !params[:size].nil? ? "\t#{column} #{coltype.upcase}(#{params[:size]})" : "\t#{column} #{coltype.upcase}" 65 | colout << colsize 66 | end 67 | end 68 | end 69 | if adapter == 'mysql2' 70 | colout << "\tPRIMARY KEY (`#{pk_col}`)" 71 | end 72 | ddlout << colout.join(",\n") 73 | close_table = adapter == 'mysql2' ? ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE utf8mb4_unicode_ci;" : ");" 74 | ddlout << close_table 75 | end 76 | 77 | # puts ddlout 78 | open("./sql/ddl/#{format}-create-#{adapter}-#{Time.now.strftime('%Y%m%d%H%M%S')}.sql", 'w') { |f| f.puts(ddlout) } 79 | end 80 | end -------------------------------------------------------------------------------- /src/schemas/custom.rb: -------------------------------------------------------------------------------- 1 | $schema = { 2 | 3 | live_stream: { 4 | # created 5 | live_stream_id: { type: 'bigint', primary_key: true }, 6 | processed_at: { type: 'datetime' }, 7 | event_time_local: { type: 'datetime' }, 8 | # stream data 9 | event_name: { type: 'string', size: 64 }, 10 | event_time: { type: 'datetime' }, 11 | real_user_id: { type: 'bigint' }, 12 | user_id_body: { type: 'bigint' }, 13 | user_id_meta: { type: 'bigint' }, 14 | user_login_meta: { type: 'string', size: 64 }, 15 | user_sis_id_meta: { type: 'string', size: 32 }, 16 | user_account_id: { type: 'bigint' }, 17 | user_agent: { type: 'string', size: 512 }, 18 | context_id_body: { type: 'bigint' }, 19 | context_id_meta: { type: 'bigint' }, 20 | context_role_body: { type: 'string', size: 24 }, 21 | context_role_meta: { type: 'string', size: 24 }, 22 | context_type_body: { type: 'string', size: 24 }, 23 | context_type_meta: { type: 'string', size: 24 }, 24 | context_sis_source_id: { type: 'string', size: 32 }, 25 | # request 26 | client_ip: { type: 'string', size: 39 }, 27 | time_zone: { type: 'string', size: 255 }, 28 | request_id: { type: 'string', size: 36 }, 29 | session_id: { type: 'string', size: 32 }, 30 | url_meta: { type: 'string', size: 'MAX' }, 31 | referrer: { type: 'string', size: 'MAX' }, 32 | http_method: { type: 'string', size: 7 }, 33 | developer_key_id: { type: 'bigint' }, 34 | # assets 35 | asset_id: { type: 'bigint' }, 36 | asset_type: { type: 'string', size: 24 }, 37 | asset_subtype: { type: 'string', size: 24 }, 38 | asset_category: { type: 'string', size: 24 }, 39 | asset_level: { type: 'string', size: 24 }, 40 | asset_role: { type: 'string', size: 24 }, 41 | # submissions 42 | submission_id: { type: 'bigint' }, 43 | assignment_id: { type: 'bigint' }, 44 | quiz_id: { type: 'bigint' }, 45 | submitted_at: { type: 'datetime' }, 46 | # system meta 47 | hostname: { type: 'string', size: 64 }, 48 | producer: { type: 'string', size: 12 }, 49 | root_account_id: { type: 'bigint' }, 50 | root_account_lti_guid: { type: 'string', size: 100 }, 51 | root_account_uuid: { type: 'string', size: 40 }, 52 | job_id: { type: 'bigint' }, 53 | job_tag: { type: 'string', size: 100 }, 54 | # generic 55 | workflow_state: { type: 'string', size: 256 }, 56 | }, 57 | 58 | led_geo_maxmind: { 59 | id: { type: 'bigint', primary_key: true }, 60 | ip: { type: 'string', size: 39 }, 61 | city: { type: 'string', size: 255 }, 62 | region_name: { type: 'string', size: 255 }, 63 | region_code: { type: 'string', size: 255 }, 64 | country_code: { type: 'string', size: 6 }, 65 | country_name: { type: 'string', size: 84 }, 66 | continent_code: { type: 'string', size: 12 }, 67 | in_eu: { type: 'string', size: 5 }, 68 | postal: { type: 'string', size: 12 }, 69 | latitude: { type: 'real', size: 4 }, 70 | longitude: { type: 'real', size: 4 }, 71 | timezone: { type: 'string', size: 50 }, 72 | utc_offset: { type: 'string', size: 6 }, 73 | country_calling_code: { type: 'string', size: 6 }, 74 | currency: { type: 'string', size: 6 }, 75 | languages: { type: 'string', size: 100 }, 76 | asn: { type: 'string', size: 32 }, 77 | org: { type: 'string', size: 255 }, 78 | } 79 | } -------------------------------------------------------------------------------- /src/actions/live_stream.rb: -------------------------------------------------------------------------------- 1 | module LiveStream 2 | 3 | def live_stream(event_name, event_time, event_data) 4 | 5 | meta = event_data['metadata'] 6 | body = event_data['body'] 7 | 8 | data = { 9 | # stream data 10 | event_name: meta['event_name']&.to_s, 11 | event_time: meta['event_time'].nil? ? nil : default_timezone(meta['event_time']), 12 | real_user_id: meta['real_user_id']&.to_i, 13 | user_id_body: body['user_id']&.to_i, 14 | user_id_meta: meta['user_id']&.to_i, 15 | user_login_meta: meta['user_login']&.to_s, 16 | user_sis_id_meta: meta['user_sis_id']&.to_s, 17 | user_account_id: meta['user_account_id']&.to_i, 18 | user_agent: meta['user_agent']&.to_s, 19 | context_id_body: body['context_id']&.to_i, 20 | context_id_meta: meta['context_id']&.to_i, 21 | context_role_body: body['context_role']&.to_s, 22 | context_role_meta: meta['context_role']&.to_s, 23 | context_type_body: body['context_type']&.to_s, 24 | context_type_meta: meta['context_type']&.to_s, 25 | context_sis_source_id: meta['context_sis_source_id']&.to_s, 26 | # request 27 | client_ip: meta['client_ip']&.to_s, 28 | time_zone: meta['time_zone']&.to_s, 29 | request_id: meta['request_id']&.to_s, 30 | session_id: meta['session_id']&.to_s, 31 | url_meta: meta['url']&.to_s, 32 | referrer: meta['referrer']&.to_s, 33 | http_method: meta['http_method']&.to_s, 34 | developer_key_id: meta['developer_key_id']&.to_i, 35 | # assets 36 | asset_id: body['asset_id']&.to_i, 37 | asset_type: body['asset_type']&.to_s, 38 | asset_subtype: body['asset_subtype']&.to_s, 39 | asset_category: body['category']&.to_s, 40 | asset_role: body['role']&.to_s, 41 | asset_level: body['level']&.to_s, 42 | # submissions 43 | submission_id: body['submission_id']&.to_i, 44 | assignment_id: body['assignment_id']&.to_i, 45 | quiz_id: body['quiz_id']&.to_i, 46 | submitted_at: body['submitted_at'].nil? ? nil : default_timezone(body['submitted_at']), 47 | # system meta 48 | hostname: meta['hostname']&.to_s, 49 | job_id: meta['job_id']&.to_i, 50 | job_tag: meta['job_tag']&.to_s, 51 | producer: meta['producer']&.to_s, 52 | root_account_id: meta['root_account_id']&.to_i, 53 | root_account_lti_guid: meta['root_account_lti_guid']&.to_s, 54 | root_account_uuid: meta['root_account_uuid']&.to_s, 55 | # generic 56 | workflow_state: body['workflow_state']&.to_s, 57 | }.compact 58 | 59 | # passively truncate strings to DDL length, keeps data insertion, logs warning for manual update 60 | limit_to_ddl('live_stream', data) 61 | 62 | processed = Time.new 63 | created = { 64 | processed_at: processed.strftime('%Y-%m-%d %H:%M:%S.%L').to_s, 65 | event_time_local: Time.parse(event_time).utc.localtime.strftime(TIME_FORMAT).to_s, 66 | } 67 | data = data.merge(created) 68 | 69 | begin 70 | DB[:live_stream].insert(data) 71 | rescue => e 72 | handle_db_errors(e, event_name, event_data, data) 73 | end 74 | end 75 | end -------------------------------------------------------------------------------- /src/ledbelly_settings.rb: -------------------------------------------------------------------------------- 1 | # LEDbelly config 2 | LED = YAML.load_file('./cfg/ledbelly.yml') 3 | 4 | # shoryuken config and settings 5 | SQS_CFG = YAML.load_file('./cfg/sqs.yml') 6 | Shoryuken.sqs_client = Aws::SQS::Client.new( 7 | region: SQS_CFG['aws']['region'], 8 | credentials: Aws::Credentials.new( 9 | SQS_CFG['aws']['access_key_id'], 10 | SQS_CFG['aws']['secret_access_key'] 11 | ) 12 | ) 13 | # https://github.com/phstc/shoryuken/wiki/Long-Polling 14 | Shoryuken.sqs_client_receive_message_opts = { 15 | wait_time_seconds: 20 16 | } 17 | 18 | # database config and settings 19 | DB_CFG = YAML.load_file('./cfg/database.yml') 20 | # db connection parameters 21 | DB = Sequel.connect( 22 | adapter: DB_CFG['adapter'], 23 | host: DB_CFG['host'], 24 | database: DB_CFG['data'], 25 | user: DB_CFG['user'], 26 | password: DB_CFG['pass'], 27 | max_connections: DB_CFG['max_connections'], 28 | encoding: 'utf8', # encoding works for tinytds, mysql2, and postgres adapters (whereas charset is mysql specific) 29 | # tinytds = 'utf8', mysql = 'utf8mb4', oracle = 'AL32UTF8', postgres = 'UTF8' 30 | timeout: 600, 31 | pool_timeout: 45, 32 | # this will log every single transactions statement, results in very large files 33 | # logger: Logger.new('log/database.log', 'daily') 34 | ) 35 | # The connection_validator extension modifies a database's 36 | # connection pool to validate that connections checked out 37 | # from the pool are still valid, before yielding them for use 38 | DB.extension :connection_validator 39 | # DB.pool.connection_validation_timeout = DB_CFG['timeout'] 40 | 41 | # Sequel error extension 42 | DB.extension :error_sql 43 | # convert table and column names to downcase 44 | DB.extension :identifier_mangling 45 | DB.identifier_input_method = :downcase 46 | # https://github.com/jeremyevans/sequel/blob/master/doc/opening_databases.rdoc#tinytds 47 | if DB_CFG['adapter'] == 'tinytds' 48 | DB.run("SET ANSI_WARNINGS ON;") 49 | end 50 | 51 | # collects everythig in lib/schemas and compiles it into 1 large hash 52 | ddl_stack = {} 53 | Dir["./src/schemas/*.rb"].sort.each do |schema| 54 | require schema 55 | ddl_stack = ddl_stack.merge!($schema) 56 | end 57 | # then stored as a constant 58 | EVENT_DDL = ddl_stack.freeze 59 | 60 | # format for all time strings 61 | TIME_FORMAT = '%Y-%m-%d %H:%M:%S'.freeze 62 | 63 | # broad warnings for available adapters 64 | WARN_ERRORS = [ 65 | # mysql 66 | # oracle 67 | 'table or view does not exist', # missing table 68 | 'invalid identifier', # missing column 69 | # postgres 70 | 'UndefinedColumn', 71 | # 'relation "afd" does not exist' # relation "" does not exist 72 | # 'column "sdf" does not exist' # column "" does not exist 73 | # tinytds 74 | 'Invalid object name', # missing table 75 | 'Invalid column name', # missing column 76 | 'String or binary data would be truncated', # value larger than column definition 77 | ].freeze 78 | 79 | # broad errors worthy of disconnecting, to preserve messages in the queue 80 | DISCONNECT_ERRORS = [ 81 | # mysql 82 | 'MySQL server has gone away', 83 | 'Lost connection to MySQL server during query', 84 | # oracle 85 | # postgres 86 | # tinytds 87 | 'Adaptive Server connection timed out', 88 | 'Cannot continue the execution because the session is in the kill state', 89 | 'Login failed for user', 90 | 'Read from the server failed', 91 | 'Server name not found in configuration files', 92 | 'The transaction log for database', 93 | 'Unable to access availability database', 94 | 'Unable to connect: Adaptive Server is unavailable or does not exist', 95 | 'Write to the server failed', 96 | 'Cannot open user default database. Login failed.' 97 | ].freeze 98 | 99 | # broad errors worthy of disconnecting, to preserve messages in the queue 100 | IDLE_ERRORS = [ 101 | # tinytds 102 | # 'is being recovered. Waiting until recovery is finished', 103 | # 'because the database replica is not in the PRIMARY or SECONDARY role', 104 | # 'is participating in an availability group and is currently not accessible for queries' 105 | ].freeze 106 | -------------------------------------------------------------------------------- /src/lib/tasks/reduce_logs.rake: -------------------------------------------------------------------------------- 1 | # reduces log files to a somewhat intelligent guess of what needs to be updated 2 | # prevents reviewing thousands of log file lines, trying to keep schema's updated 3 | 4 | task :reduce_logs do 5 | 6 | def open_log(log) 7 | if File.exist?(log) 8 | File.read(log) 9 | end 10 | end 11 | 12 | # reduces log/ddl-warnings.log, indentifying unique updates and tries to provide sample datatype from schema 13 | def ddl_warnings 14 | updates = [] 15 | open_log("log/ddl-warnings.log")&.each_line do |line| 16 | if match = line.match(/((?:live_|ims_)[a-z_]+).([a-z_]+)\s--\s(.*)/i) 17 | event_name, column, value = match.captures 18 | sample = warnings_type_test(column) 19 | sample = sample.size == 1 ? sample.first.strip : 'too many choices' 20 | updates << "#{event_name} | #{column} | #{sample}" 21 | end 22 | end 23 | puts "### updates for schema column length" 24 | updates.uniq.sort.each { |line| puts line } 25 | end 26 | 27 | def warnings_type_test(check) 28 | schema = `git show HEAD:src/schemas/canvas.rb` 29 | size = schema.each_line.select { |line| line =~ /#{check}/ } 30 | size.uniq 31 | end 32 | 33 | # reduces log/ddl-undefined.log, identifying unique columns that need to be added to schema 34 | def undefined_columns 35 | updates = {} 36 | open_log("log/ddl-undefined.log")&.each_line do |line| 37 | if match = line.match(/sample: { event_name: ([a-z_.]+), undefined: \[(.*)\]/i) 38 | event_name_str, undefined = match.captures 39 | event_name = event_name_str.to_sym 40 | updates[event_name] ||= {} 41 | #sample = undefined.delete('"').delete(' ').split(',') 42 | sample = undefined[1..-1].split('", "') 43 | sample.each do |s| 44 | column_str, value = s.strip.split(':::') 45 | column = column_str.to_sym 46 | key = updates.dig(event_name, column) 47 | 48 | # check if current value is larger than stored value, keep largest/length 49 | updates[event_name][column] = if !!key && key[:value]&.size.to_i > value&.size.to_i 50 | { value: key[:value], type: key[:value]&.class, size: key[:size] } 51 | else 52 | { value: value, type: undefined_type_test?(value), size: value&.size.to_i } 53 | end 54 | end 55 | end 56 | end 57 | puts "\n\n### updates for events, columns not defined" 58 | updates.each do |event_name, columns| 59 | puts "#{event_name}" 60 | columns.each do |column, params| 61 | puts " #{column}: { type: '#{params[:type]}', size: #{params[:size]} } # #{params[:value]}" 62 | end 63 | end 64 | end 65 | 66 | def undefined_type_test? string 67 | 'Integer' if Integer(string) rescue 'Float' if Float(string) rescue 'String' 68 | end 69 | 70 | # reduces log/sql-truncations.log, identifing what event columns need to be updated with longer lengths 71 | def sql_truncations 72 | updates = {} 73 | open_log("log/sql-truncations.log")&.each_line do |line| 74 | if match = line.match(/([a-z_]+).([a-z_]+) {\ssupplied: (\d+), expecting: (\d+) }/i) 75 | event_name_str, column_str, supplied, expected = match.captures 76 | column = column_str.to_sym 77 | event_name = event_name_str.to_sym 78 | updates[event_name] ||= {} 79 | key = updates.dig(event_name, column) 80 | 81 | # check if current value is larger than stored value, keep largest/length 82 | updates[event_name][column] = if !!key && key[:supplied]&.to_i > supplied&.to_i 83 | { supplied: key[:supplied], expected: key[:expected] } 84 | else 85 | { supplied: supplied, expected: expected } 86 | end 87 | end 88 | end 89 | puts "\n\n### sql data truncated on insert, update column length" 90 | updates.each do |event_name, columns| 91 | puts "#{event_name}" 92 | columns.each do |column, params| 93 | puts " #{column}: { supplied: '#{params[:supplied]}', expecting: '#{params[:expected]}' }" 94 | end 95 | end 96 | end 97 | 98 | ddl_warnings 99 | undefined_columns 100 | sql_truncations 101 | end -------------------------------------------------------------------------------- /src/actions/import_sql.rb: -------------------------------------------------------------------------------- 1 | module SQLInsertEvent 2 | 3 | def import(event_name, event_time, event_data, import_data) 4 | 5 | event_source = event_data.dig('dataVersion').nil? ? 'live' : 'ims' 6 | event_table = "#{event_source}_#{event_name}".gsub(/[^\w\s]/, '_') 7 | 8 | # passively truncate strings to DDL length, keeps data insertion, logs warning for manual update 9 | limit_to_ddl(event_table, import_data) 10 | 11 | processed = Time.new 12 | created = { 13 | processed_at: processed.strftime('%Y-%m-%d %H:%M:%S.%L').to_s, 14 | event_time_local: Time.parse(event_time).utc.localtime.strftime(TIME_FORMAT).to_s, 15 | } 16 | data = import_data.merge(created) 17 | 18 | begin 19 | # insert the data 20 | DB[event_table.to_sym].insert(data) 21 | 22 | # terminal output, if terminal/interactive 23 | printf("\r%s: %s\e[0J", created[:event_time_local], event_table) if $stdout.isatty 24 | rescue => e 25 | handle_db_errors(e, event_name, event_data, import_data) 26 | end 27 | end 28 | 29 | def limit_to_ddl(event_table, import_data) 30 | # loop through each string value, compare length to defined (DDL) length 31 | # if defined as multibyte string, check values bytesize against mb/length (defined 2 x actual byte length) 32 | # https://api.rubyonrails.org/classes/ActiveSupport/Multibyte/Chars.html#method-i-limit 33 | import_data.each do |k,v| 34 | next if v.nil? 35 | 36 | # log warnings if the key is not defined in the schema 37 | unless EVENT_DDL[event_table.to_sym].key?(k) 38 | log = "\nunexpected key, not found in schema : #{event_table}.#{k} -- #{v}" 39 | open('log/ddl-warnings.log', 'a') { |f| f << "#{Time.now} #{log}\n" } 40 | # remove the key from the hash 41 | import_data.delete(k) 42 | next 43 | end 44 | 45 | next unless EVENT_DDL[event_table.to_sym][k][:type] == 'string' && EVENT_DDL[event_table.to_sym][k][:size] != 'MAX' 46 | 47 | v = if EVENT_DDL[event_table.to_sym][k].key?(:mbstr) 48 | # multi-byte strings 49 | v.mb_chars.limit(EVENT_DDL[event_table.to_sym][k][:size] * 2).to_s 50 | else 51 | begin 52 | # regular string 53 | v.mb_chars.limit(EVENT_DDL[event_table.to_sym][k][:size]).to_s 54 | rescue => e 55 | puts EVENT_DDL[event_table.to_sym][k] 56 | puts e 57 | puts "########{event_table}" 58 | end 59 | end 60 | 61 | next unless import_data[k].mb_chars.length > v.mb_chars.length 62 | 63 | # collect warning 64 | log = "#{event_table}.#{k} { supplied: #{import_data[k].mb_chars.length}, expecting: #{EVENT_DDL[event_table.to_sym][k][:size]} }" 65 | # overwrite/update inserted value 66 | import_data[k] = v 67 | # log warning 68 | open('log/sql-truncations.log', 'a') { |f| f << "#{Time.now} #{log}\n" } 69 | end 70 | end 71 | 72 | def handle_db_errors(exp, event_name, event_data, import_data) 73 | 74 | # create a log entry 75 | err = %W[ 76 | ---#{event_name}---\n 77 | ------ time 78 | at: #{import_data['processed_at']} event_time: #{event_time}\n 79 | #{exp.message}\n 80 | ------\n 81 | #{exp.sql}\n 82 | ------ import data\n 83 | #{import_data}\n 84 | ------ event data\n 85 | #{event_data}\n 86 | ].join 87 | # puts err 88 | 89 | # store in log file 90 | open('log/sql-errors.log', 'a') { |f| f << "#{err}\n\n" } 91 | # drop the failed SQL statement into a file 92 | # we can use this file to import the records later 93 | open('log/sql-recovery.log', 'a') { |f| f << "#{exp.sql};\n" } 94 | open('log/sql-recovery.json', 'a') { |f| f << "#{event_data}\n" } 95 | 96 | if exp.message.match? Regexp.union(WARN_ERRORS) 97 | warn "#{exp.message} (#{event_name})" 98 | end 99 | 100 | if exp.message.match? Regexp.union(DISCONNECT_ERRORS) 101 | # disconnect the db 102 | DB.disconnect 103 | 104 | # terminal output, if terminal/interactive 105 | warn exp.message if $stdout.isatty 106 | 107 | # kill shoryuken/ledbelly 108 | shoryuken_pid = File.read('log/shoryuken.pid').to_i 109 | Process.kill('TERM', shoryuken_pid) 110 | end 111 | end 112 | 113 | end -------------------------------------------------------------------------------- /src/lib/services/maxmind.rb: -------------------------------------------------------------------------------- 1 | require 'maxmind/db' 2 | require 'active_support/time_with_zone' 3 | 4 | class MaxMindGeo 5 | 6 | def start 7 | puts 'w/MaxMindDB' if $stdout.isatty 8 | max_ip_lookup 9 | end 10 | 11 | def ready 12 | test = DB[:led_geo_maxmind].select(:ip).limit(1) 13 | if test.count == 1 14 | true 15 | end 16 | rescue => e 17 | if e.message.match? Regexp.union(WARN_ERRORS) 18 | warn e 19 | false 20 | # test = DB.create_table(:led_geo_maxmind) do 21 | # # created 22 | # primary_key :id, type: :Bignum 23 | # String :ip, size: 39 24 | # String :city, size: 255 25 | # String :region_name, size: 255 26 | # String :region_code, size: 255 27 | # String :country_code, size: 6 28 | # String :country_name, size: 84 29 | # String :continent_code, size: 12 30 | # String :in_eu, size: 5 31 | # String :postal, size: 12 32 | # Float :latitude 33 | # Float :longitude 34 | # String :timezone, size: 50 35 | # String :utc_offset, size: 6 36 | # String :country_calling_code, size: 6 37 | # String :currency, size: 6 38 | # String :languages, size: 32 39 | # String :asn, size: 32 40 | # String :org, size: 255 41 | # end 42 | # p test 43 | end 44 | end 45 | 46 | def select_ips 47 | DB[:live_stream].distinct.select(:client_ip). 48 | left_outer_join(:led_geo_maxmind, :ip => :client_ip). 49 | where{client_ip !~ nil}.where{ip =~ nil}. 50 | limit(20) 51 | rescue => e 52 | p e 53 | end 54 | 55 | def add_ip(max) 56 | DB[:led_geo_maxmind].insert(max) 57 | # { 58 | # ip: max[:ip], 59 | # city: max[:city], 60 | # region_name: max[:region_name], 61 | # region_code: max[:region_code], 62 | # country_code: max[:country_code], 63 | # country_name: max[:country_name], 64 | # continent_code: max[:continent_code], 65 | # in_eu: max[:in_eu], 66 | # postal: max[:postal], 67 | # latitude: max[:latitude], 68 | # longitude: max[:longitude], 69 | # timezone: max[:timezone], 70 | # utc_offset: max[:utc_offset], 71 | # country_calling_code: max[:country_calling_code], 72 | # currency: max[:currency], 73 | # languages: max[:languages], 74 | # asn: max[:asn], 75 | # org: max[:org], 76 | # }) 77 | rescue => e 78 | p e 79 | end 80 | 81 | def utc_offset(tmzn) 82 | ActiveSupport::TimeZone.seconds_to_utc_offset(ActiveSupport::TimeZone[tmzn].utc_offset) 83 | end 84 | 85 | def max_ip_lookup 86 | Thread.new do 87 | if ready == false 88 | abort 89 | end 90 | 91 | # https://dev.maxmind.com/geoip/geoip2/geolite2/ 92 | max_db = MaxMind::DB.new('src/lib/services/GeoLite2-City.mmdb', mode: MaxMind::DB::MODE_MEMORY) 93 | select_ips&.map(:client_ip)&.each do |client_ip| 94 | max = max_db.get(client_ip) 95 | 96 | if max.nil? 97 | puts "#{client_ip} was not found in the database" 98 | else 99 | ip_data = { 100 | ip: client_ip, 101 | city: max.key?('city') ? max['city']['names']['en']&.to_s : nil, 102 | region_name: max.key?('subdivisions') ? max['subdivisions'][0]['names']['en']&.to_s : nil, 103 | region_code: max.key?('subdivisions') ? max['subdivisions'][0]['iso_code']&.to_s : nil, 104 | country_code: max['country']['iso_code']&.to_s, 105 | country_name: max['country']['names']['en']&.to_s, 106 | continent_code: max['continent']['code']&.to_s, 107 | in_eu: max['continent']['code'] == 'EU' ? 1 : 0, 108 | postal: max.key?('postal') ? max['postal']['code']&.to_s : nil, 109 | latitude: max['location']['latitude'], 110 | longitude: max['location']['longitude'], 111 | timezone: max['location']['time_zone']&.to_s, 112 | utc_offset: max['location'].key?('time_zone') ? utc_offset(max['location']['time_zone']) : nil, 113 | country_calling_code: nil, 114 | currency: nil, 115 | languages: nil, 116 | asn: nil, 117 | org: nil, 118 | } 119 | 120 | if $stdout.isatty 121 | time = DateTime.now.strftime('%Y-%m-%d %H:%M:%S.%L').to_s 122 | out = [ 123 | ip_data[:country_name], 124 | ip_data[:region_name], 125 | ip_data[:city], 126 | ].compact.join('.').downcase 127 | printf("\r%s: %s\e[0J", time, "[geo] #{out}") 128 | end 129 | 130 | add_ip(ip_data) 131 | end 132 | end 133 | max_db.close 134 | sleep 20 135 | max_ip_lookup 136 | end 137 | end 138 | end 139 | maxip = MaxMindGeo.new 140 | maxip.start -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | TargetRubyVersion: 2.5 3 | Exclude: 4 | - 'dev/**/*' 5 | - 'src/lib/tasks/**/*' 6 | 7 | # our style changes: disabling style rules we aren't interested in 8 | Layout/AlignParameters: 9 | Enabled: false 10 | Layout/ElseAlignment: 11 | Enabled: false 12 | Layout/EmptyLines: 13 | Enabled: false 14 | Layout/EmptyLinesAroundAccessModifier: 15 | Enabled: false 16 | Layout/EmptyLinesAroundArguments: 17 | Enabled: false 18 | Layout/EmptyLinesAroundBlockBody: 19 | Enabled: false 20 | Layout/EmptyLinesAroundClassBody: 21 | Enabled: false 22 | Layout/EmptyLinesAroundMethodBody: 23 | Enabled: false 24 | Layout/EmptyLinesAroundModuleBody: 25 | Enabled: false 26 | Layout/IndentFirstArrayElement: 27 | Enabled: false 28 | Layout/IndentFirstHashElement: 29 | Enabled: false 30 | Layout/IndentationConsistency: 31 | Enabled: false 32 | Layout/IndentationWidth: 33 | Enabled: false 34 | Layout/IndentHeredoc: 35 | Enabled: false 36 | Layout/MultilineOperationIndentation: 37 | Enabled: false 38 | Layout/SpaceAfterColon: 39 | Enabled: false 40 | Layout/SpaceAfterComma: 41 | Enabled: false 42 | Layout/SpaceAroundEqualsInParameterDefault: 43 | Enabled: false 44 | Layout/SpaceAroundOperators: 45 | Enabled: false 46 | Layout/SpaceBeforeBlockBraces: 47 | Enabled: false 48 | Layout/SpaceBeforeFirstArg: 49 | Enabled: false 50 | Layout/SpaceInLambdaLiteral: 51 | Enabled: false 52 | Layout/SpaceInsideArrayLiteralBrackets: 53 | Enabled: false 54 | Layout/SpaceInsideBlockBraces: 55 | Enabled: false 56 | Layout/SpaceInsideHashLiteralBraces: 57 | Enabled: false 58 | Layout/SpaceInsideReferenceBrackets: 59 | Enabled: false 60 | Layout/TrailingBlankLines: 61 | Enabled: false 62 | Layout/TrailingWhitespace: 63 | Enabled: false 64 | 65 | Style/FormatStringToken: 66 | Enabled: false 67 | Style/StringLiterals: 68 | Enabled: false 69 | Style/SignalException: 70 | Enabled: false 71 | Style/NumericLiterals: 72 | Enabled: false 73 | Style/BracesAroundHashParameters: 74 | Enabled: false 75 | Style/PercentLiteralDelimiters: 76 | Enabled: false 77 | Style/Documentation: 78 | Enabled: false 79 | Style/ClassAndModuleChildren: 80 | Enabled: false 81 | Style/RegexpLiteral: 82 | Enabled: false 83 | Style/GuardClause: 84 | Enabled: false 85 | Style/RedundantSelf: 86 | Enabled: false 87 | Style/IfUnlessModifier: 88 | Enabled: false 89 | Style/WordArray: 90 | Enabled: false 91 | Style/PercentQLiterals: 92 | Enabled: false 93 | Style/DoubleNegation: 94 | Enabled: false 95 | Style/TrailingCommaInArguments: 96 | Enabled: false 97 | Style/TrailingCommaInArrayLiteral: 98 | Enabled: false 99 | Style/TrailingCommaInHashLiteral: 100 | Enabled: false 101 | Style/MethodCallWithoutArgsParentheses: 102 | Enabled: false 103 | Style/MethodCallWithArgsParentheses: 104 | Enabled: false 105 | Layout/DotPosition: 106 | Enabled: true 107 | EnforcedStyle: trailing 108 | Layout/AlignHash: 109 | Enabled: false 110 | Style/Lambda: 111 | Enabled: false 112 | Style/WhileUntilModifier: 113 | Enabled: false 114 | Style/ParallelAssignment: 115 | Enabled: false 116 | Style/ZeroLengthPredicate: 117 | Enabled: false 118 | Style/NumericPredicate: 119 | Enabled: false 120 | Naming/VariableNumber: 121 | Enabled: false 122 | Style/Dir: 123 | Enabled: false 124 | Style/ReturnNil: 125 | Enabled: false 126 | Style/StderrPuts: 127 | Enabled: false 128 | Style/DateTime: 129 | Enabled: false 130 | Style/SymbolArray: 131 | Enabled: false 132 | # We may want to enable this when we start working toward Ruby 3 133 | Style/FrozenStringLiteralComment: 134 | Enabled: false 135 | Style/AsciiComments: 136 | Enabled: false 137 | Style/BlockDelimiters: 138 | Enabled: true 139 | Exclude: 140 | - spec/**/*_spec.rb 141 | - spec/shared_examples/**/*.rb 142 | 143 | # this isn't good for us because of how we pin dependencies 144 | Bundler/OrderedGems: 145 | Enabled: false 146 | Gemspec/OrderedDependencies: 147 | Enabled: false 148 | Gemspec/RequiredRubyVersion: 149 | Enabled: false 150 | 151 | # Lint changes 152 | Lint/AmbiguousRegexpLiteral: 153 | Severity: convention 154 | Lint/AmbiguousBlockAssociation: 155 | Exclude: 156 | - spec/**/* 157 | Lint/UselessAssignment: 158 | Severity: convention 159 | Lint/Debugger: 160 | Severity: error 161 | Lint/PercentStringArray: 162 | Enabled: false 163 | 164 | # Performance changes 165 | Performance/Detect: 166 | Severity: warning 167 | Performance/TimesMap: 168 | Exclude: 169 | - spec/**/* 170 | 171 | # these need better configuration than the default: 172 | Style/AndOr: 173 | EnforcedStyle: conditionals 174 | Style/RescueModifier: 175 | Severity: warning 176 | Layout/MultilineMethodCallIndentation: 177 | EnforcedStyle: indented 178 | # Layout/IndentArray: 179 | # EnforcedStyle: consistent 180 | Layout/EndAlignment: 181 | EnforcedStyleAlignWith: variable 182 | Severity: convention 183 | 184 | # these are invalid pre-Ruby 2.4 185 | Performance/RegexpMatch: 186 | Enabled: false 187 | 188 | # Things we may want to tighten down later 189 | Metrics/AbcSize: 190 | Enabled: false 191 | Metrics/LineLength: 192 | Max: 200 193 | Metrics/MethodLength: 194 | Max: 1000 195 | Metrics/ClassLength: 196 | Enabled: false 197 | Metrics/ModuleLength: 198 | Enabled: false 199 | Metrics/BlockLength: 200 | Max: 100 201 | Exclude: 202 | Metrics/CyclomaticComplexity: 203 | Enabled: false 204 | Metrics/PerceivedComplexity: 205 | Enabled: false 206 | Style/HashSyntax: 207 | Enabled: false 208 | 209 | Style/RescueStandardError: 210 | Enabled: false 211 | -------------------------------------------------------------------------------- /src/lib/tasks/alter_tables.rake: -------------------------------------------------------------------------------- 1 | require 'hashdiff' 2 | require 'jaro_winkler' 3 | 4 | task :alter_tables do 5 | 6 | #DB_CFG = YAML.load_file('./cfg/database.yml') 7 | 8 | def sql_to_hash(log) 9 | out = {} 10 | current_event = '' 11 | File.read(log)&.each_line do |line| 12 | if event_match = line.match(/CREATE TABLE\s+(?:[^.]+\.)?((?:live|ims)_\w+)\s+\(/i) 13 | event_table_name_str = event_match.captures.first 14 | event_table_name = event_table_name_str.to_sym 15 | out[event_table_name] ||= {} 16 | current_event = event_table_name 17 | elsif column_name_match = line.match(/^\s{1}([a-z_]+)\s(.*)/i) 18 | event_column_name_str, event_column_name_type_str = column_name_match.captures 19 | out[current_event][event_column_name_str.to_sym] = event_column_name_type_str.chomp(',') 20 | end 21 | end 22 | out 23 | end 24 | 25 | def alter_tables_sql(diff, old_ddl, new_ddl) 26 | changes = {create: [], add: [], modify: [], rename: [], drop: [], double_check: []} 27 | renamed = [] 28 | diff.each_with_index do |change, i| 29 | table_name, column_name = change[1].split('.') 30 | datatype = change.last 31 | 32 | case change.first 33 | # modify 34 | when '~' 35 | if ['tinytds', 'postgres'].include?(DB_CFG['adapter']) 36 | changes[:modify] << "ALTER TABLE #{table_name} ALTER COLUMN #{column_name} #{datatype};" 37 | elsif ['mysql2', 'oracle'].include?(DB_CFG['adapter']) 38 | changes[:modify] << "ALTER TABLE #{table_name} MODIFY COLUMN #{column_name} #{datatype};" 39 | end 40 | # remove 41 | when '-' 42 | # first, check if column was renamed 43 | next if diff[i+1].nil? 44 | next_change = diff[i+1] 45 | next_table_name, next_column_name = next_change[1].split('.') 46 | next_datatype = next_change.last 47 | 48 | # might be a column rename if... 49 | rename_if = { 50 | table_and_datatype_length: [table_name, datatype] == [next_table_name, next_datatype], 51 | table_and_datatype: [table_name, datatype.gsub(/(\W|\d)/, "")] == [next_table_name, next_datatype.gsub(/(\W|\d)/, "")], 52 | same_table_key_index: column_key_compare([table_name, column_name], [next_table_name, next_column_name], old_ddl, new_ddl), 53 | jw_distance: rename_weight(change, next_change) 54 | }.select { |k,v| v == true } 55 | # at least 2 of the 4 56 | if rename_if.size >= 2 57 | if ['tinytds'].include?(DB_CFG['adapter']) 58 | changes[:rename] << "EXEC sp_rename '#{table_name}.#{column_name}', '#{next_column_name}', 'COLUMN';" 59 | elsif ['mysql2'].include?(DB_CFG['adapter']) 60 | changes[:rename] << "ALTER TABLE #{table_name} CHANGE COLUMN `#{column_name}` `#{next_column_name}` #{next_datatype};" 61 | elsif ['oracle', 'postgres'].include?(DB_CFG['adapter']) 62 | changes[:rename] << "ALTER TABLE #{table_name} RENAME COLUMN #{column_name} TO #{next_column_name};" 63 | end 64 | renamed << "#{table_name}:::#{column_name}:::#{next_column_name}" 65 | changes[:double_check] << change 66 | # drop column 67 | else 68 | changes[:drop] << "ALTER TABLE #{table_name} DROP COLUMN #{column_name};" 69 | end 70 | # add 71 | when '+' 72 | # addition is a new event table 73 | if change.last.class == Hash 74 | changes[:create] << "-- ADD #{table_name} FROM CREATE TABLES" 75 | else 76 | next if diff[i-1].nil? 77 | # ensure this one wasn't setup as a rename 78 | prev_change = diff[i-1] 79 | prev_table_name, prev_column_name = prev_change[1].split('.') 80 | prev_datatype = prev_change.last 81 | # add if new, not if renamed 82 | if !renamed.any? { |r| r.include?("#{table_name}:::#{prev_column_name}:::#{column_name}") } 83 | changes[:add] << "ALTER TABLE #{table_name} ADD #{column_name} #{datatype};" 84 | else 85 | changes[:double_check] << change 86 | end 87 | end 88 | end 89 | end 90 | changes 91 | end 92 | 93 | def column_key_compare(change, next_change, old_ddl, new_ddl) 94 | # get the hash key index of the current column, in the old schema 95 | column_idx = old_ddl.dig(change[0].to_sym).find_index { |k,_| k== change[1].to_sym } 96 | # get the has key index of the next column, in the new schema 97 | next_column_idx = new_ddl.dig(next_change[0].to_sym).find_index { |k,_| k== next_change[1].to_sym } 98 | column_idx == next_column_idx 99 | end 100 | 101 | def rename_weight(change, next_change) 102 | (JaroWinkler.distance "#{change[1]} #{change.last}", "#{next_change[1]} #{next_change.last}", ignore_case: true) >= 0.91 103 | end 104 | 105 | def compare_sql(format) 106 | sql_out = [] 107 | sql_files = Dir["sql/ddl/*#{format}*"].sort_by{ |f| File.birthtime(f) }[0...2] 108 | if sql_files.size < 2 109 | warn 'must have 2 to compare! run `rake create_tables`' 110 | exit 111 | end 112 | old_ddl = sql_to_hash(sql_files.first) 113 | new_ddl = sql_to_hash(sql_files.last) 114 | changes = Hashdiff.diff(old_ddl, new_ddl) 115 | sql_out << "--------------------" 116 | sql_out << "-- #{format} schema" 117 | sql_out << "-- comparing: " + sql_files.to_s 118 | sql_out << "--------------------" 119 | alter_tables_sql(changes, old_ddl, new_ddl).each do |type, changed| 120 | sql_out << "-- #{type}" 121 | changed.each { |change| sql_out << " #{change}" } 122 | sql_out << "" 123 | end 124 | sql_out.join("\n") 125 | end 126 | 127 | schemas = Dir["sql/ddl/*sql"].map{ |s| File.basename(s).split(/[._\-]/)[0] }.uniq.sort 128 | schemas.each { |f| print compare_sql(f) + "\n" if ['canvas','caliper','custom'].include? f } 129 | end -------------------------------------------------------------------------------- /src/ledbelly_support.rb: -------------------------------------------------------------------------------- 1 | def default_timezone(string) 2 | Time.parse(string).utc.strftime(TIME_FORMAT).to_s unless string.nil? 3 | end 4 | 5 | # flattens the nested/recursive data structure of events to underscore_notation 6 | def _flatten(data, recursive_key = '') 7 | data.each_with_object({}) do |(k, v), ret| 8 | key = recursive_key + k.to_s 9 | key = key.gsub(/[^a-zA-Z]/, '_') 10 | begin 11 | if v.is_a? Hash 12 | ret.merge! _flatten(v, key + '_') 13 | elsif v.is_a? Array 14 | v.each do |x| 15 | if x.is_a? String 16 | ret[key] = v.join(',') 17 | else 18 | ret.merge! _flatten(x, key + '_') 19 | end 20 | end 21 | else 22 | ret[key] = v 23 | end 24 | rescue 25 | pp [v, v.class, v.size, v.length, v.empty?] 26 | end 27 | end 28 | end 29 | 30 | # reduces underscore notation, removing repeated and verbose strings 31 | def _squish(hash) 32 | hash = _flatten(hash) 33 | hash.each_with_object({}) do |(k, v), ret| 34 | k = k.gsub(/extensions|com|instructure|canvas/, '').gsub(/_+/, '_').gsub(/^_/, '').downcase 35 | ret[k] = v 36 | end 37 | end 38 | 39 | # send message to another queue for caching 40 | def collect_unknown(event_name, event_data) 41 | puts "unexpected event: #{event_name}\nstoring event data in #{SQS_CFG['queues'][0]}-moo queue" 42 | message = { 43 | message_body: event_data.to_json, 44 | message_attributes: { 45 | event_name: { 46 | string_value: event_name.to_s, 47 | data_type: "String", 48 | }, 49 | event_time: { 50 | string_value: (event_data.dig('metadata', 'event_time') || event_data.dig('data', 0, 'eventTime')).to_s, 51 | data_type: "String", 52 | }, 53 | } 54 | } 55 | Shoryuken::Client.queues("#{SQS_CFG['queues'][0]}-moo").send_message(message) 56 | # LiveEvents.perform_async(event_data, queue: "#{SQS_CFG['queues'][0]}-moo") 57 | rescue => e 58 | pp ['moo queue failed, saving payload to file', e, event_name] 59 | # write event and payload to file 60 | open('log/payload-cache.js', 'a') do |f| 61 | f << "\n//#{event_name}\n" 62 | f << event_data.to_json 63 | end 64 | end 65 | 66 | # counts the fields sent with caliper event to what is expected, log if something was missed 67 | def caliper_count(event_name, sent, expected) 68 | 69 | # original hash, cloned, compact (no nil), strings, keys 70 | copy_sent = sent.clone.compact.stringify_keys.keys 71 | copy_expected = expected.clone.stringify_keys.keys 72 | 73 | # what's missing, get the difference 74 | missing = copy_sent - copy_expected # | copy_expected - copy_sent 75 | missing = missing.reject { |k| ['id'].include? k } 76 | if missing.size.positive? 77 | sample = missing.map { |k| "#{k}:::#{sent.fetch(k)}"} 78 | err = <<~ERRLOG 79 | event_name: ims_#{event_name} 80 | count: { sent: #{sent.keys.count}, defined: #{copy_expected.count} } 81 | summary: { event_name: ims_#{event_name}, undefined: #{missing.to_s.gsub('"', '')} } 82 | sample: { event_name: ims_#{event_name}, undefined: #{sample} } 83 | message: #{sent.to_json} 84 | 85 | ERRLOG 86 | # store in log file, print if interactive 87 | open('log/ddl-undefined.log', 'a') { |f| f << err } 88 | puts err if $stdout.isatty 89 | end 90 | end 91 | 92 | # counts the meta data fields sent with canvas event to what is expected, log if something was missed 93 | def missing_meta(sent, expected) 94 | 95 | # original hash, cloned, compact (no nil), strings, keys 96 | sent_meta = sent['metadata'].clone.compact.stringify_keys.keys 97 | collected_meta = expected.clone.stringify_keys.keys 98 | 99 | # normalize the key names, since we've added _meta to some 100 | normal_meta = collected_meta.map{ |k| k.gsub(/_meta/, '')} 101 | 102 | # what's missing, get the difference 103 | missing = sent_meta - normal_meta | normal_meta - sent_meta 104 | 105 | # store in log file 106 | if missing.size.positive? 107 | sample = missing.map { |k| "#{k}:::#{sent['metadata'].fetch(k)}" } 108 | err = <<~ERRLOG 109 | event_name: live_#{sent['metadata']['event_name']} 110 | count: { sent: #{sent['metadata'].keys.count}, defined: #{normal_meta.count} } 111 | summary: { event_name: live_#{sent['metadata']['event_name']}, undefined: #{missing.to_s.gsub('"', '')} } 112 | sample: { event_name: live_#{sent['metadata']['event_name']}, undefined: #{sample} } 113 | message: #{sent.to_json} 114 | 115 | ERRLOG 116 | # store in log file, print if interactive 117 | open('log/ddl-undefined.log', 'a') { |f| f << err } 118 | puts err if $stdout.isatty 119 | end 120 | end 121 | 122 | # counts the body fields sent with canvas event to what is expected, log if something was missed 123 | def missing_body(event_data, bodydata) 124 | 125 | ed = event_data.clone 126 | bd = bodydata.clone 127 | flag = false 128 | 129 | # flag if event body has more fields than we're expecting 130 | missing = ed['body'].stringify_keys.keys - bd.stringify_keys.keys 131 | flag = true if missing.size.positive? 132 | 133 | # compare body fields to metadata fields, keep keys where the values are different 134 | flag = true if missing.reject { |k| bd[k] == ed['metadata'][k] }.size.positive? 135 | 136 | # compare data in deeply nested events 137 | if flag == true && missing.any? {|k| ed['body'][k].is_a? Hash} 138 | 139 | # for each body key, store a similar value with the missing prefix removed 140 | body_keys = _flatten(ed['body']).keys 141 | compare = body_keys.map { |k| [k, k.sub(Regexp.union(missing), '').sub(/^_/, '')] } || [] 142 | 143 | # for each comparison key, check it within the expected body keys 144 | # for each comparison set, if 1 item in the compare array is a match for an expected key, store the expected key to a new array 145 | # store missing keys to a different array 146 | found = [] 147 | compare.clone.each do |c| 148 | c.each do |k| 149 | next unless bodydata.stringify_keys.keys.any? k 150 | 151 | found << k 152 | # don't compare it again 153 | compare.delete(c) 154 | end 155 | end 156 | 157 | # if the compare is empty now, continue, nothing was missed 158 | if compare.empty? 159 | flag = false 160 | # something was missed, identify it 161 | else 162 | 163 | missing_nested = [] 164 | compare.each do |c| 165 | c.each do |k| 166 | # if the key is not found in the expected bodydata, collect it 167 | missing_nested << k if bodydata.stringify_keys.keys.none? k 168 | end 169 | end 170 | 171 | # get the difference in keys sent from expected 172 | # check each missing key and see if we're storing it's parent 173 | # remove any keys where we store the parent element and it's content 174 | key_diff = body_keys - bd.stringify_keys.keys 175 | missing_still = key_diff.clone 176 | key_diff.each do |k| 177 | bd.stringify_keys.keys.each do |dk| 178 | missing_still.delete(k) if key_diff.any?(/^#{dk}/) 179 | end 180 | end 181 | 182 | if missing_still.count == 0 183 | flag = false 184 | else 185 | missing = missing_still 186 | deep_sample = missing.map { |k| "#{k}:::#{_flatten(ed['body']).fetch(k)}" } 187 | # puts "missing still" 188 | # puts missing_still.count 189 | 190 | # o = { 191 | # expected: bd.stringify_keys.keys, 192 | # sent: body_keys, 193 | # compare: check, 194 | # missing_nested: missing_nested, 195 | # missing: missing, 196 | # missing_still: missing_still 197 | # } 198 | # pp o 199 | # open('log/ddl-debug-missing.log', 'a') { |f| f << o } 200 | 201 | end 202 | end 203 | end 204 | 205 | if flag == true 206 | sample = deep_sample || missing&.map { |k| "#{k}:::#{ed['body'].fetch(k)}" } 207 | err = <<~ERRLOG 208 | event_name: live_#{ed['metadata']['event_name']} 209 | count: { sent: #{ed['body'].keys.count}, defined: #{bd.keys.count} } 210 | summary: { event_name: #{ed['metadata']['event_name']}, undefined: #{missing.to_s.gsub('"', '')} } 211 | sample: { event_name: #{ed['metadata']['event_name']}, undefined: #{sample} } 212 | message: #{ed.to_json} 213 | 214 | ERRLOG 215 | # store in log file, print if interactive 216 | open('log/ddl-undefined.log', 'a') { |f| f << err } 217 | puts err if $stdout.isatty 218 | end 219 | end -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LEDbelly 2 | 3 | LEDbelly, or Live Events Daemon is powerful and lightweight middleware for consuming CanvasLMS Live Events, from SQS to SQL. 4 | 5 | - provides a fast multi-threaded message processor via [Shoryuken](https://github.com/phstc/shoryuken) 6 | - provides support and compatibility for popular databases used by the Canvas Community via [Sequel](https://github.com/jeremyevans/sequel) 7 | - currently supports MSSQL, MySQL, PostgreSQL, and Oracle[*](https://github.com/ccsd/ledbelly#known-issues) 8 | - supports both [Canvas Raw](https://github.com/instructure/canvas-lms/blob/master/doc/api/live_events.md) and [IMS Caliper](https://github.com/instructure/canvas-lms/blob/master/doc/api/caliper_live_events.md) formats 9 | 10 | Creating an SQS Queue and receiving events is easy and relatively low cost via [AWS](https://aws.amazon.com/sqs/pricing/), but consuming and using events is a bit more complicated. Live Events are published in real-time as users or the system performs actions within Canvas. This real-time data is extremely useful for building applications and services. Event messages are published to Amazon Simple Queue Service (SQS) in JSON. Some of the data is nested in the Canvas format, and all events are nested in the Caliper format. This creates a problem for pushing events to SQL since data isn't structured for this purpose. There are many options for software and AWS services like Glue can do this easily and efficiently, for a price. However, with a little configuration and a dedicated server, LEDbelly will help you add Live Events into your existing Canvas Data SQL pipeline. 11 | 12 | LEDbelly aims to make using Live Events Services easy and attainable to any school or institution that cannot afford more robust options and licensing costs. The products that can do this often have costs or overhead out of reach for small budgets and teams. Therefore, LED has been designed to be easily integrated and maintainable for your purposes. Additional features have been implemented to aid in the continual maintenance as Canvas periodically adds new events and new columns to existing events, and more improvements are coming! 13 | 14 | I welcome any community contributions and collaboration on this project. I will maintain and update as necessary. 15 | 16 | ## Features 17 | 18 | __Shoryuken__, provides an easy to use multi-threaded SQS processor. Handling events from SQS is [pretty easy](https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/sqs-examples.html), but the number of messages that can come through can go from trickle to firehose instantly when there's a lot of activity in Canvas. Without multi-threading, messages will backup and your real-time data eventually becomes 'when it arrives', likely to catch up in the middle of the night when things slow down. 19 | 20 | __Sequel__, provides the support for multiple databases without extra code management. It also provides DB connection pooling to handle the numerous threads inserting rows. 21 | 22 | _These two gems were chosen for their active support and contributions by multiple contributors. This allows _us_ to focus on Live Events, and not multi-threaded SQS processing or supporting multiple database adapters and engines._ 23 | 24 | __Live Events Worker__, is the Shoryuken process that polls messages an passes them to the parser, then handles importing data to SQL. 25 | 26 | __Parsers__, are provided for each event format in `src/events`. These files are the product of patiently collecting auto-generated JSON to SQL tables and columns. Then auto generating the code to simplified `CASE` statements for each event type to produce `key value` pairs of _known_ event information. Each event is passed to the parser, and the known fields are defined and datatype are set. 27 | 28 | _Most maintenance happens here, when Canvas adds new columns or events, you'll need to update the code... Hopefully we'll do this collectively and share what we see. It's entirely possible to never see certain events based on how your users use Canvas. So that first school to see new data could write the update and submit a PR so others can benefit._ 29 | 30 | __Import__ happens after each event is parsed. A `Hash` with non-null fields is passed to the import method. Each string field is passed through its schema definition and trimmed to the defined length, ensuring the row is inserted. 31 | 32 | __Schemas__, are configured in `lib/schemas`, and help prepare data before import, and simplify the maintenance cycle. Each schema file for Canvas or Caliper (or custom tables) maintains the event type table definition for each column. This helps LEDbelly know when new fields are added and existing fields are out-of-bounds. The schemas are also used to auto generate the DDL for your preferred database using `rake create_tables` task. 33 | 34 | [__Automation Tasks__](https://github.com/ccsd/ledbelly/wiki/Tasks) 35 | 36 | [__Logging__](https://github.com/ccsd/ledbelly/wiki/Logging) 37 | 38 | __Live Stream__, while LEDbelly will process both Canvas and Caliper formats into their specific event tables, sometimes it's easier to deal with things collectively. Live Stream available in `src/extentions` passes common fields for all Canvas events into a `live_stream` table. This is useful for tracking active users and types of activity without overly cumbersome views and joins. I __recommend__ using this feature, it's available by default, but you can remove it. It's also packaged to give an example of providing your own extensions in case you want to add or manipulate some stream without modifying the defaults. A couple of quick SQL queries in `sql/samples` are provided. 39 | 40 | 41 | ## Getting Started 42 | [Installation & Setup](https://github.com/ccsd/ledbelly/wiki/Getting-Started) 43 | 44 | 45 | ## License 46 | LEDbelly, is distributed under the MIT License 47 | 48 | >Copyright (c) 2019 Clark County School District, created by Robert Carroll 49 | > 50 | >Permission is hereby granted, free of charge, to any person obtaining a copy 51 | of this software and associated documentation files (the "Software"), to deal 52 | in the Software without restriction, including without limitation the rights 53 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 54 | copies of the Software, and to permit persons to whom the Software is 55 | furnished to do so, subject to the following conditions: 56 | > 57 | >The above copyright notice and this permission notice shall be included in all 58 | copies or substantial portions of the Software. 59 | > 60 | >THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 61 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 62 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 63 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 64 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 65 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 66 | SOFTWARE. 67 | 68 | 69 | ## Support & Contributions 70 | 71 | [I'm available on the CanvasLMS Community](https://community.canvaslms.com/people/carroll-ccsd) 72 | 73 | This project it is intended for community use and contributions. My hope is for there to be community collaboration and governance for future development and changes. Canvas cloud hosted instances are updated regularly and changes to Live Events often occur with these updates. Users of LEDbelly, would benefit from working together to continuously maintain this repo. 74 | 75 | I am not a Rubyist, and this is my first multi file Ruby project, if you can _school me_ and help improve the code please do! 76 | 77 | New code should pass the [RuboCop](https://github.com/rubocop-hq/rubocop) rules supplied. However, one(1) existing Cop fails, for introducing globals, if you know how to work around this issue please share. :D 78 | 79 | > For now, simply fork the repo, create a feature branch, commit, and push your changes, and then create a pull request. 80 | > 81 | > Review and consider [Issues](https://github.com/ccsd/ledbelly/issues) 82 | 83 | ## Resources 84 | #### Community 85 | - [LEDbelly discussion](https://community.canvaslms.com/message/157736-ledbelly-live-events-daemon-for-canvas-lms-sqs-to-sql) 86 | - [Canvas Live Events | Awesome CanvasLMS](https://community.canvaslms.com/docs/DOC-17354-awesome-canvaslms#CanvasLiveEvents) 87 | 88 | #### Canvas Source Files 89 | 90 | Watch commits to this file, they usually indicate when you'll see new events or columns. 91 | - [canvas/live_events.rb](https://github.com/instructure/canvas-lms/blob/master/lib/canvas/live_events.rb) 92 | - [canvas/live_events_spec.rb](https://github.com/instructure/canvas-lms/blob/master/spec/lib/canvas/live_events_spec.rb) 93 | - Changes to Live Events are also posted to Canvas Release Notes 94 | 95 | 96 | ## Known Issues 97 | 98 | 1) Much of the Caliper data where you'd want to have integer ID's contains strings with ID's. I am not currently using Caliper events, but I receive them for development purposes. I have refrained from making decisions for others here, but would expect the desirable choice would be RegEx matching and storing the ID. 99 | ``` 100 | actor_id: 'urn:instructure:canvas:user:100000000000001' 101 | membership_id: 'urn:instructure:canvas:course:100000000000012:Instructor:100000000000001' 102 | membership_member_id: 'urn:instructure:canvas:user:100000000000001' 103 | membership_member_type: 'Person' 104 | membership_organization_id: 'urn:instructure:canvas:course:100000000000012' 105 | membership_organization_type: 'CourseOffering' 106 | ``` 107 | 108 | 2) Oracle's 30 character limit poses problems for Live Events as many of the fields (especially from IMS Caliper) are formatted using underscore_notation. Some guidance from Canvas Community Oracle users would be much appreciated here. While Sequel will successfully handle SQS to SQL for Oracle manually shortening each field that is >30, it would be nice if we could create some uniform shortening for LEDbelly that keeps it simple for everyone. 109 | 110 | 3) A second Oracle issue exists in the `rake create_tables` task, in order to generate DDL's for each database I wrote a custom task that parses the `lib/schemas/*` from `Hash` to DDL statements. While my dev environment is currently at 12c, the compatibility is at 11, making it hard for me to test a DDL with the simple `CREATE TABLE with IDENTITY` for the PK. 111 | 112 | 4) `interaction_data` for `quizzes.item_created` and `quizzes.item_updated` needs to be improved, but I am not currently using these columns. 113 | 114 | ## Credits 115 | Many thanks to Pablo Cantero and Jeremy Evans for their open source contributions. LEDbelly would be much harder to maintain without these Gems. 116 | 117 | The name _LEDbelly_ was chosen one morning while listening to [Huddie Ledbetter aka Lead Belly](https://en.wikipedia.org/wiki/Lead_Belly). It seemed apropos for a _Live Events Daemon and Consumer_ 118 | 119 | [The CCSD Canvas Team](http://obl.ccsd.net) 120 | 121 | ![Elvis Panda](https://s3-us-west-2.amazonaws.com/ccsd-canvas/branding/images/ccsd-elvis-panda-sm.png "Elvis Panda") -------------------------------------------------------------------------------- /src/events/ims_caliper.rb: -------------------------------------------------------------------------------- 1 | module IMSCaliperEvents 2 | 3 | # handle ims caliper 4 | def _imsdata(event_name, data) 5 | 6 | common = { 7 | uuid: data['id']&.to_s, 8 | action: data['action']&.to_s, 9 | actor_entity_id: data['actor_entity_id']&.to_i, 10 | actor_id: data['actor_id']&.to_s, 11 | actor_real_user_id: data['actor_real_user_id']&.to_i, 12 | actor_root_account_id: data['actor_root_account_id']&.to_i, 13 | actor_root_account_lti_guid: data['actor_root_account_lti_guid']&.to_s, 14 | actor_root_account_uuid: data['actor_root_account_uuid']&.to_s, 15 | actor_type: data['actor_type']&.to_s, 16 | actor_user_login: data['actor_user_login']&.to_s, 17 | actor_user_sis_id: data['actor_user_sis_id']&.to_s, 18 | client_ip: data['client_ip']&.to_s, 19 | context: data['context']&.to_s, 20 | edapp_id: data['edapp_id']&.to_s, 21 | edapp_type: data['edapp_type']&.to_s, 22 | eventtime: data['eventtime'].nil? ? nil : default_timezone(data['eventtime']), 23 | group_context_type: data['group_context_type']&.to_s, 24 | group_entity_id: data['group_entity_id']&.to_i, 25 | group_id: data['group_id']&.to_s, 26 | group_type: data['group_type']&.to_s, 27 | hostname: data['hostname']&.to_s, 28 | job_id: data['job_id']&.to_i, 29 | job_tag: data['job_tag']&.to_s, 30 | membership_id: data['membership_id']&.to_s, 31 | membership_member_id: data['membership_member_id']&.to_s, 32 | membership_member_type: data['membership_member_type']&.to_s, 33 | membership_organization_id: data['membership_organization_id']&.to_s, 34 | membership_organization_type: data['membership_organization_type']&.to_s, 35 | membership_roles: data['membership_roles']&.to_s, 36 | membership_type: data['membership_type']&.to_s, 37 | object_id: data['object_id']&.to_s, 38 | object_entity_id: data['object_entity_id']&.to_i, 39 | object_name: data['object_name']&.to_s, 40 | object_type: data['object_type']&.to_s, 41 | referrer: data['referrer']&.to_s, 42 | request_url: data['request_url']&.to_s, 43 | request_id: data['request_id']&.to_s, 44 | session_id: data['session_id']&.to_s, 45 | session_type: data['session_type']&.to_s, 46 | type: data['type']&.to_s, 47 | user_agent: data['user_agent']&.to_s, 48 | version: data['version']&.to_s, 49 | }.compact 50 | 51 | case event_name 52 | 53 | when 'asset_accessed' 54 | specific = { 55 | object_asset_name: data['object_asset_name']&.to_s, 56 | object_asset_subtype: data['object_asset_subtype']&.to_s, 57 | object_asset_type: data['object_asset_type']&.to_s, 58 | object_context_account_id: data['object_context_account_id']&.to_i, 59 | object_developer_key_id: data['object_developer_key_id']&.to_i, 60 | object_display_name: data['object_display_name']&.to_s, 61 | object_domain: data['object_domain']&.to_s, 62 | object_filename: data['object_filename']&.to_s, 63 | object_http_method: data['object_http_method']&.to_s, 64 | object_url: data['object_url']&.to_s, 65 | } 66 | 67 | when 'assignment_created' 68 | specific = { 69 | object_datecreated: data['object_datecreated'].nil? ? nil : default_timezone(data['object_datecreated']), 70 | object_maxscore: data['object_maxscore']&.to_f, 71 | object_maxscore_numberstr: data['object_maxscore_numberstr']&.to_f, 72 | object_description: data['object_description']&.to_s, 73 | object_lock_at: data['object_lock_at'].nil? ? nil : default_timezone(data['object_lock_at']), 74 | object_datetoshow: data['object_datetoshow'].nil? ? nil : default_timezone(data['object_datetoshow']), 75 | object_datetosubmit: data['object_datetosubmit'].nil? ? nil : default_timezone(data['object_datetosubmit']), 76 | } 77 | 78 | when 'assignment_updated' 79 | specific = { 80 | object_description: data['object_description']&.to_s, 81 | object_datemodified: data['object_datemodified'].nil? ? nil : default_timezone(data['object_datemodified']), 82 | object_workflow_state: data['object_workflow_state']&.to_s, 83 | object_datetosubmit: data['object_datetosubmit'].nil? ? nil : default_timezone(data['object_datetosubmit']), 84 | object_maxscore: data['object_maxscore']&.to_f, 85 | object_maxscore_numberstr: data['object_maxscore_numberstr']&.to_f, 86 | object_lock_at: data['object_lock_at'].nil? ? nil : default_timezone(data['object_lock_at']), 87 | object_datetoshow: data['object_datetoshow'].nil? ? nil : default_timezone(data['object_datetoshow']), 88 | } 89 | 90 | when 'assignment_override_created' 91 | specific = { 92 | object_assignment_id: data['object_assignment_id']&.to_i, 93 | object_all_day_date: data['object_all_day_date'].nil? ? nil : default_timezone(data['object_all_day_date']), 94 | object_all_day: data['object_all_day']&.to_s, 95 | object_course_section_id: data['object_course_section_id']&.to_s, 96 | object_datetoshow: data['object_datetoshow'].nil? ? nil : default_timezone(data['object_datetoshow']), 97 | object_datetosubmit: data['object_datetosubmit'].nil? ? nil : default_timezone(data['object_datetosubmit']), 98 | object_group_id: data['object_group_id']&.to_i, 99 | object_lock_at: data['object_lock_at'].nil? ? nil : default_timezone(data['object_lock_at']), 100 | object_workflow_state: data['object_workflow_state']&.to_s, 101 | membership_organization_suborganizationof_id: data['membership_organization_suborganizationof_id']&.to_s, 102 | membership_organization_suborganizationof_type: data['membership_organization_suborganizationof_type']&.to_s, 103 | } 104 | 105 | when 'assignment_override_updated' 106 | specific = { 107 | object_assignment_id: data['object_assignment_id']&.to_i, 108 | object_all_day_date: data['object_all_day_date'].nil? ? nil : default_timezone(data['object_all_day_date']), 109 | object_all_day: data['object_all_day']&.to_s, 110 | object_course_section_id: data['object_course_section_id']&.to_s, 111 | object_datetoshow: data['object_datetoshow'].nil? ? nil : default_timezone(data['object_datetoshow']), 112 | object_datetosubmit: data['object_datetosubmit'].nil? ? nil : default_timezone(data['object_datetosubmit']), 113 | object_group_id: data['object_group_id']&.to_i, 114 | object_lock_at: data['object_lock_at'].nil? ? nil : default_timezone(data['object_lock_at']), 115 | object_workflow_state: data['object_workflow_state']&.to_s, 116 | membership_organization_suborganizationof_id: data['membership_organization_suborganizationof_id']&.to_s, 117 | membership_organization_suborganizationof_type: data['membership_organization_suborganizationof_type']&.to_s, 118 | } 119 | 120 | when 'attachment_created' 121 | specific = { 122 | object_datecreated: data['object_datecreated'].nil? ? nil : default_timezone(data['object_datecreated']), 123 | object_context_id: data['object_context_id']&.to_i, 124 | object_context_type: data['object_context_type']&.to_s, 125 | object_filename: data['object_filename']&.to_s, 126 | object_folder_id: data['object_folder_id']&.to_i, 127 | object_mediatype: data['object_mediatype']&.to_s, 128 | } 129 | 130 | when 'attachment_deleted' 131 | specific = { 132 | object_datemodified: data['object_datemodified'].nil? ? nil : default_timezone(data['object_datemodified']), 133 | object_context_id: data['object_context_id']&.to_i, 134 | object_context_type: data['object_context_type']&.to_s, 135 | object_filename: data['object_filename']&.to_s, 136 | object_folder_id: data['object_folder_id']&.to_i, 137 | object_mediatype: data['object_mediatype']&.to_s, 138 | } 139 | 140 | when 'attachment_updated' 141 | specific = { 142 | object_datemodified: data['object_datemodified'].nil? ? nil : default_timezone(data['object_datemodified']), 143 | object_context_id: data['object_context_id']&.to_i, 144 | object_context_type: data['object_context_type']&.to_s, 145 | object_filename: data['object_filename']&.to_s, 146 | object_folder_id: data['object_folder_id']&.to_i, 147 | object_mediatype: data['object_mediatype']&.to_s, 148 | } 149 | 150 | when 'course_created' 151 | specific = {} 152 | 153 | when 'discussion_entry_created' 154 | specific = { 155 | object_ispartof_id: data['object_ispartof_id']&.to_s, 156 | object_ispartof_type: data['object_ispartof_type']&.to_s, 157 | object_body: data['object_body']&.to_s, 158 | } 159 | 160 | when 'discussion_topic_created' 161 | specific = { 162 | object_is_announcement: data['object_is_announcement']&.to_s, 163 | } 164 | 165 | when 'enrollment_created' 166 | specific = { 167 | object_datecreated: data['object_datecreated'].nil? ? nil : default_timezone(data['object_datecreated']), 168 | object_course_id: data['object_course_id']&.to_s, 169 | object_course_section_id: data['object_course_section_id']&.to_s, 170 | object_limit_privileges_to_course_section: data['object_limit_privileges_to_course_section']&.to_s, 171 | object_user_id: data['object_user_id']&.to_s, 172 | object_user_name: data['object_user_name']&.to_s, 173 | object_workflow_state: data['object_workflow_state']&.to_s, 174 | membership_organization_suborganizationof_id: data['membership_organization_suborganizationof_id']&.to_s, 175 | membership_organization_suborganizationof_type: data['membership_organization_suborganizationof_type']&.to_s, 176 | } 177 | 178 | when 'enrollment_state_created' 179 | specific = { 180 | object_access_is_current: data['object_access_is_current']&.to_s, 181 | object_restricted_access: data['object_restricted_access']&.to_s, 182 | object_state: data['object_state']&.to_s, 183 | object_state_is_current: data['object_state_is_current']&.to_s, 184 | object_state_valid_until: data['object_state_valid_until']&.to_s, 185 | object_startedattime: data['object_startedattime'].nil? ? nil : default_timezone(data['object_startedattime']), 186 | } 187 | 188 | when 'enrollment_state_updated' 189 | specific = { 190 | object_access_is_current: data['object_access_is_current']&.to_s, 191 | object_restricted_access: data['object_restricted_access']&.to_s, 192 | object_state: data['object_state']&.to_s, 193 | object_state_is_current: data['object_state_is_current']&.to_s, 194 | object_state_valid_until: data['object_state_valid_until']&.to_s, 195 | object_startedattime: data['object_startedattime'].nil? ? nil : default_timezone(data['object_startedattime']), 196 | } 197 | 198 | when 'enrollment_updated' 199 | specific = { 200 | object_datecreated: data['object_datecreated'].nil? ? nil : default_timezone(data['object_datecreated']), 201 | object_datemodified: data['object_datemodified'].nil? ? nil : default_timezone(data['object_datemodified']), 202 | object_course_id: data['object_course_id']&.to_s, 203 | object_course_section_id: data['object_course_section_id']&.to_s, 204 | object_limit_privileges_to_course_section: data['object_limit_privileges_to_course_section']&.to_s, 205 | object_user_id: data['object_user_id']&.to_s, 206 | object_user_name: data['object_user_name']&.to_s, 207 | object_workflow_state: data['object_workflow_state']&.to_s, 208 | membership_organization_suborganizationof_id: data['membership_organization_suborganizationof_id']&.to_s, 209 | membership_organization_suborganizationof_type: data['membership_organization_suborganizationof_type']&.to_s, 210 | } 211 | 212 | when 'grade_change' 213 | specific = { 214 | object_grade: data['object_grade']&.to_s, 215 | object_assignee_id: data['object_assignee_id']&.to_s, 216 | object_assignee_type: data['object_assignee_type']&.to_s, 217 | object_assignee_sis_id: data['object_assignee_sis_id']&.to_s, 218 | object_assignable_id: data['object_assignable_id']&.to_s, 219 | object_assignable_type: data['object_assignable_type']&.to_s, 220 | generated_id: data['generated_id']&.to_s, 221 | generated_type: data['generated_type']&.to_s, 222 | generated_grade: data['generated_grade']&.to_s, 223 | generated_entity_id: data['generated_entity_id']&.to_s, 224 | generated_attempt_id: data['generated_attempt_id']&.to_s, 225 | generated_attempt_type: data['generated_attempt_type']&.to_s, 226 | generated_attempt_grade: data['generated_attempt_grade']&.to_s, 227 | generated_attempt_assignee_id: data['generated_attempt_assignee_id']&.to_s, 228 | generated_attempt_assignee_type: data['generated_attempt_assignee_type']&.to_s, 229 | generated_attempt_assignee_sis_id: data['generated_attempt_assignee_sis_id']&.to_s, 230 | generated_attempt_assignable_id: data['generated_attempt_assignable_id']&.to_s, 231 | generated_attempt_assignable_type: data['generated_attempt_assignable_type']&.to_s, 232 | generated_maxscore_numberstr: data['generated_maxscore_numberstr']&.to_f, 233 | generated_scoregiven_numberstr: data['generated_scoregiven_numberstr']&.to_f, 234 | generated_scoredby: data['generated_scoredby']&.to_s, 235 | generated_scoregiven: data['generated_scoregiven']&.to_f, 236 | generated_maxscore: data['generated_maxscore']&.to_f, 237 | } 238 | 239 | when 'group_created' 240 | specific = { 241 | object_ispartof_id: data['object_ispartof_id']&.to_s, 242 | object_ispartof_name: data['object_ispartof_name']&.to_s, 243 | object_ispartof_type: data['object_ispartof_type']&.to_s, 244 | } 245 | 246 | when 'group_category_created' 247 | specific = {} 248 | 249 | when 'group_membership_created' 250 | specific = { 251 | object_member_id: data['object_member_id']&.to_s, 252 | object_member_type: data['object_member_type']&.to_s, 253 | object_organization_entity_id: data['object_organization_entity_id']&.to_s, 254 | object_organization_id: data['object_organization_id']&.to_s, 255 | object_organization_ispartof_id: data['object_organization_ispartof_id']&.to_s, 256 | object_organization_ispartof_name: data['object_organization_ispartof_name']&.to_s, 257 | object_organization_ispartof_type: data['object_organization_ispartof_type']&.to_s, 258 | object_organization_name: data['object_organization_name']&.to_s, 259 | object_organization_type: data['object_organization_type']&.to_s, 260 | object_roles: data['object_roles']&.to_s, 261 | } 262 | 263 | when 'logged_in' 264 | specific = { 265 | object_redirect_url: data['object_redirect_url']&.to_s, 266 | } 267 | 268 | when 'logged_out' 269 | specific = {} 270 | 271 | when 'quiz_submitted' 272 | specific = { 273 | object_assignee_id: data['object_assignee_id']&.to_s, 274 | object_assignee_type: data['object_assignee_type']&.to_s, 275 | object_assignable_id: data['object_assignable_id']&.to_s, 276 | object_assignable_type: data['object_assignable_type']&.to_s, 277 | } 278 | 279 | when 'submission_created' 280 | specific = { 281 | object_datecreated: data['object_datecreated'].nil? ? nil : default_timezone(data['object_datecreated']), 282 | object_submission_type: data['object_submission_type']&.to_s, 283 | object_assignee_id: data['object_assignee_id']&.to_s, 284 | object_assignee_type: data['object_assignee_type']&.to_s, 285 | object_assignable_id: data['object_assignable_id']&.to_s, 286 | object_assignable_type: data['object_assignable_type']&.to_s, 287 | object_count: data['object_count']&.to_s, 288 | object_body: data['object_body']&.to_s, 289 | object_url: data['object_url']&.to_s, 290 | } 291 | 292 | when 'submission_updated' 293 | specific = { 294 | object_datemodified: data['object_datemodified'].nil? ? nil : default_timezone(data['object_datemodified']), 295 | object_assignee_id: data['object_assignee_id']&.to_s, 296 | object_assignee_type: data['object_assignee_type']&.to_s, 297 | object_assignable_id: data['object_assignable_id']&.to_s, 298 | object_assignable_type: data['object_assignable_type']&.to_s, 299 | object_submission_type: data['object_submission_type']&.to_s, 300 | object_count: data['object_count']&.to_s, 301 | object_url: data['object_url']&.to_s, 302 | object_body: data['object_body']&.to_s, 303 | } 304 | 305 | when 'syllabus_updated' 306 | specific = { 307 | object_creators_id: data['object_creators_id']&.to_s, 308 | object_creators_type: data['object_creators_type']&.to_s, 309 | object_body: data['object_body']&.to_s, 310 | } 311 | 312 | when 'user_account_association_created' 313 | specific = { 314 | object_datecreated: data['object_datecreated'].nil? ? nil : default_timezone(data['object_datecreated']), 315 | object_datemodified: data['object_datemodified'].nil? ? nil : default_timezone(data['object_datemodified']), 316 | object_is_admin: data['object_is_admin']&.to_s, 317 | object_user_id: data['object_user_id']&.to_s, 318 | } 319 | 320 | when 'wiki_page_created' 321 | specific = { 322 | object_body: data['object_body']&.to_s, 323 | } 324 | 325 | when 'wiki_page_deleted' 326 | specific = {} 327 | 328 | when 'wiki_page_updated' 329 | specific = { 330 | object_body: data['object_body']&.to_s, 331 | } 332 | 333 | # catch and save events, we don't have configured or we aren't expecting 334 | else 335 | collect_unknown(event_name, event_data) 336 | # return if the message cannot be prepped for import 337 | return 338 | end 339 | 340 | # return parsed event data 341 | # merge the common fields with the event specific data 342 | specific.merge(common) 343 | end 344 | 345 | def _caliper(event_name, event_data) 346 | data = _squish(event_data['data'][0]) 347 | imsdata = _imsdata(event_name, data) 348 | return if !imsdata.is_a? Hash 349 | 350 | # check if we missed any new data 351 | caliper_count(event_name, data, imsdata) 352 | 353 | # return event data - parsed, flattened, ready for sql 354 | imsdata 355 | end 356 | end -------------------------------------------------------------------------------- /src/events/canvas_raw.rb: -------------------------------------------------------------------------------- 1 | module CanvasRawEvents 2 | 3 | # collect and return all possible message data fields 4 | def _metadata(event_data) 5 | meta = event_data['metadata'] 6 | { 7 | client_ip: meta['client_ip']&.to_s, 8 | context_account_id: meta['context_account_id']&.to_i, 9 | context_id_meta: meta['context_id']&.to_i, 10 | context_role_meta: meta['context_role']&.to_s, 11 | context_sis_source_id: meta['context_sis_source_id']&.to_s, 12 | context_type_meta: meta['context_type']&.to_s, 13 | event_name: meta['event_name']&.to_s, 14 | event_time: meta['event_time'].nil? ? nil : default_timezone(meta['event_time']), 15 | developer_key_id: meta['developer_key_id']&.to_i, 16 | hostname: meta['hostname']&.to_s, 17 | http_method: meta['http_method']&.to_s, 18 | job_id: meta['job_id']&.to_i, 19 | job_tag: meta['job_tag']&.to_s, 20 | producer: meta['producer']&.to_s, 21 | real_user_id: meta['real_user_id']&.to_i, 22 | request_id: meta['request_id']&.to_s, 23 | root_account_id_meta: meta['root_account_id']&.to_i, 24 | root_account_lti_guid: meta['root_account_lti_guid']&.to_s, 25 | root_account_uuid: meta['root_account_uuid']&.to_s, 26 | session_id: meta['session_id']&.to_s, 27 | time_zone: meta['time_zone']&.to_s, 28 | url_meta: meta['url']&.to_s, 29 | referrer: meta['referrer']&.to_s, 30 | user_account_id: meta['user_account_id']&.to_i, 31 | user_agent: meta['user_agent']&.to_s, 32 | user_id_meta: meta['user_id']&.to_i, 33 | user_login_meta: meta['user_login']&.to_s, 34 | user_sis_id_meta: meta['user_sis_id']&.to_s, 35 | }.compact 36 | end 37 | 38 | def _bodydata(event_data) 39 | 40 | event_name = event_data.dig("metadata", "event_name") 41 | body = event_data['body'] 42 | 43 | case event_name 44 | 45 | when 'account_notification_created' 46 | 47 | bodydata = { 48 | account_notification_id: body['account_notification_id']&.to_i, 49 | subject: body['subject']&.to_s, 50 | message: body['message']&.to_s, 51 | icon: body['icon']&.to_s, 52 | start_at: body['start_at'].nil? ? nil : default_timezone(body['start_at']), 53 | end_at: body['end_at'].nil? ? nil : default_timezone(body['end_at']), 54 | } 55 | 56 | when 'asset_accessed' 57 | 58 | bodydata = { 59 | asset_id: body['asset_id']&.to_i, 60 | asset_name: body['asset_name']&.to_s, 61 | asset_type: body['asset_type']&.to_s, 62 | asset_subtype: body['asset_subtype']&.to_s, 63 | category: body['category']&.to_s, 64 | role: body['role']&.to_s, 65 | level: body['level']&.to_s, 66 | filename: body['filename']&.to_s, 67 | display_name: body['display_name']&.to_s, 68 | domain: body['domain']&.to_s, 69 | url: body['url']&.to_s, 70 | enrollment_id: body['enrollment_id']&.to_i, 71 | section_id: body['section_id']&.to_i, 72 | } 73 | 74 | when 'assignment_created' 75 | 76 | bodydata = { 77 | assignment_id: body['assignment_id']&.to_i, 78 | context_id: body['context_id']&.to_i, 79 | context_type: body['context_type']&.to_s, 80 | context_uuid: body['context_uuid']&.to_s, 81 | assignment_group_id: body['assignment_group_id']&.to_i, 82 | workflow_state: body['workflow_state']&.to_s, 83 | title: body['title']&.to_s, 84 | description: body['description']&.to_s, 85 | due_at: body['due_at'].nil? ? nil : default_timezone(body['due_at']), 86 | unlock_at: body['unlock_at'].nil? ? nil : default_timezone(body['unlock_at']), 87 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 88 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 89 | points_possible: body['points_possible']&.to_f, 90 | lti_assignment_id: body['lti_assignment_id']&.to_s, 91 | lti_resource_link_id: body['lti_resource_link_id']&.to_s, 92 | lti_resource_link_id_duplicated_from: body['lti_resource_link_id_duplicated_from']&.to_s, 93 | submission_types: body['submission_types']&.to_s, 94 | } 95 | 96 | when 'assignment_updated' 97 | 98 | bodydata = { 99 | assignment_id: body['assignment_id']&.to_i, 100 | context_id: body['context_id']&.to_i, 101 | context_type: body['context_type']&.to_s, 102 | context_uuid: body['context_uuid']&.to_s, 103 | assignment_group_id: body['assignment_group_id']&.to_i, 104 | workflow_state: body['workflow_state']&.to_s, 105 | title: body['title']&.to_s, 106 | description: body['description']&.to_s, 107 | due_at: body['due_at'].nil? ? nil : default_timezone(body['due_at']), 108 | unlock_at: body['unlock_at'].nil? ? nil : default_timezone(body['unlock_at']), 109 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 110 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 111 | points_possible: body['points_possible']&.to_f, 112 | lti_assignment_id: body['lti_assignment_id']&.to_s, 113 | lti_resource_link_id: body['lti_resource_link_id']&.to_s, 114 | lti_resource_link_id_duplicated_from: body['lti_resource_link_id_duplicated_from']&.to_s, 115 | submission_types: body['submission_types']&.to_s, 116 | } 117 | 118 | when 'assignment_group_created' 119 | 120 | bodydata = { 121 | assignment_group_id: body['assignment_group_id']&.to_i, 122 | context_id: body['context_id']&.to_i, 123 | context_type: body['context_type']&.to_s, 124 | name: body['name']&.to_s, 125 | position: body['position']&.to_i, 126 | group_weight: body['group_weight']&.to_f, 127 | sis_source_id: body['sis_source_id']&.to_s, 128 | integration_data: body['integration_data']&.to_s, 129 | rules: body['rules']&.to_s, 130 | workflow_state: body['workflow_state']&.to_s, 131 | } 132 | 133 | when 'assignment_group_updated' 134 | 135 | bodydata = { 136 | assignment_group_id: body['assignment_group_id']&.to_i, 137 | context_id: body['context_id']&.to_i, 138 | context_type: body['context_type']&.to_s, 139 | name: body['name']&.to_s, 140 | position: body['position']&.to_i, 141 | group_weight: body['group_weight']&.to_f, 142 | sis_source_id: body['sis_source_id']&.to_s, 143 | integration_data: body['integration_data']&.to_s, 144 | rules: body['rules']&.to_s, 145 | workflow_state: body['workflow_state']&.to_s, 146 | } 147 | 148 | when 'assignment_override_created' 149 | 150 | bodydata = { 151 | assignment_override_id: body['assignment_override_id']&.to_i, 152 | assignment_id: body['assignment_id']&.to_i, 153 | due_at: body['due_at'].nil? ? nil : default_timezone(body['due_at']), 154 | all_day: body['all_day']&.to_s, 155 | all_day_date: body['all_day_date'].nil? ? nil : default_timezone(body['all_day_date']), 156 | unlock_at: body['unlock_at'].nil? ? nil : default_timezone(body['unlock_at']), 157 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 158 | type: body['type']&.to_s, 159 | workflow_state: body['workflow_state']&.to_s, 160 | course_section_id: body['course_section_id']&.to_s, 161 | group_id: body['group_id']&.to_i, 162 | } 163 | 164 | when 'assignment_override_updated' 165 | 166 | bodydata = { 167 | assignment_override_id: body['assignment_override_id']&.to_i, 168 | assignment_id: body['assignment_id']&.to_i, 169 | due_at: body['due_at'].nil? ? nil : default_timezone(body['due_at']), 170 | all_day: body['all_day']&.to_s, 171 | all_day_date: body['all_day_date'].nil? ? nil : default_timezone(body['all_day_date']), 172 | unlock_at: body['unlock_at'].nil? ? nil : default_timezone(body['unlock_at']), 173 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 174 | type: body['type']&.to_s, 175 | workflow_state: body['workflow_state']&.to_s, 176 | course_section_id: body['course_section_id']&.to_s, 177 | group_id: body['group_id']&.to_i, 178 | } 179 | 180 | when 'attachment_created' 181 | 182 | bodydata = { 183 | attachment_id: body['attachment_id']&.to_i, 184 | user_id: body['user_id']&.to_i, 185 | display_name: body['display_name']&.to_s, 186 | filename: body['filename']&.to_s, 187 | folder_id: body['folder_id']&.to_i, 188 | context_type: body['context_type']&.to_s, 189 | context_id: body['context_id']&.to_i, 190 | content_type: body['content_type']&.to_s, 191 | unlock_at: body['unlock_at'].nil? ? nil : default_timezone(body['unlock_at']), 192 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 193 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 194 | } 195 | 196 | when 'attachment_deleted' 197 | 198 | bodydata = { 199 | attachment_id: body['attachment_id']&.to_i, 200 | user_id: body['user_id']&.to_i, 201 | display_name: body['display_name']&.to_s, 202 | filename: body['filename']&.to_s, 203 | folder_id: body['folder_id']&.to_i, 204 | context_type: body['context_type']&.to_s, 205 | context_id: body['context_id']&.to_i, 206 | content_type: body['content_type']&.to_s, 207 | unlock_at: body['unlock_at'].nil? ? nil : default_timezone(body['unlock_at']), 208 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 209 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 210 | } 211 | 212 | when 'attachment_updated' 213 | 214 | bodydata = { 215 | attachment_id: body['attachment_id']&.to_i, 216 | user_id: body['user_id']&.to_i, 217 | display_name: body['display_name']&.to_s, 218 | old_display_name: body['old_display_name']&.to_s, 219 | folder_id: body['folder_id']&.to_i, 220 | filename: body['filename']&.to_s, 221 | context_type: body['context_type']&.to_s, 222 | context_id: body['context_id']&.to_i, 223 | content_type: body['content_type']&.to_s, 224 | unlock_at: body['unlock_at'].nil? ? nil : default_timezone(body['unlock_at']), 225 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 226 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 227 | } 228 | 229 | when 'content_migration_completed' 230 | 231 | bodydata = { 232 | content_migration_id: body['content_migration_id']&.to_i, 233 | context_id: body['context_id']&.to_i, 234 | context_type: body['context_type']&.to_s, 235 | lti_context_id: body['lti_context_id']&.to_s, 236 | context_uuid: body['context_uuid']&.to_s, 237 | import_quizzes_next: body['import_quizzes_next']&.to_s, 238 | } 239 | 240 | when 'conversation_created' 241 | 242 | bodydata = { 243 | conversation_id: body['conversation_id']&.to_i, 244 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 245 | } 246 | 247 | when 'conversation_message_created' 248 | 249 | bodydata = { 250 | author_id: body['author_id']&.to_i, 251 | conversation_id: body['conversation_id']&.to_i, 252 | message_id: body['message_id']&.to_i, 253 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 254 | } 255 | 256 | when 'course_completed' 257 | 258 | bodydata = { 259 | # body progress 260 | requirement_count: body['progress']['requirement_count'].nil? ? nil : body['progress']['requirement_count'].to_i, 261 | requirement_completed_count: body['progress']['requirement_completed_count'].nil? ? nil : body['progress']['requirement_completed_count'].to_i, 262 | next_requirement_url: body['progress']['next_requirement_url'].nil? ? nil : body['progress']['next_requirement_url'].to_s, 263 | completed_at: body['progress']['completed_at'].nil? ? nil : default_timezone(body['progress']['completed_at']), 264 | error_message: body['progress'].key?('error') ? body['progress']['error']['message'].to_s : nil, 265 | # body user 266 | user_id: body['user']['id'].nil? ? nil : body['user']['id'].to_i, 267 | user_name: body['user']['name'].nil? ? nil : body['user']['name'].to_s, 268 | user_email: body['user']['email'].nil? ? nil : body['user']['email'].to_s, 269 | # body course 270 | course_id: body['course']['id'].nil? ? nil : body['course']['id'].to_i, 271 | course_name: body['course']['name'].nil? ? nil : body['course']['name'].to_s, 272 | account_id: body['course']['account_id'].nil? ? nil : body['course']['account_id'].to_i, 273 | sis_source_id: body['course']['sis_source_id'].nil? ? nil : body['course']['sis_source_id'].to_s, 274 | } 275 | 276 | when 'course_created' 277 | 278 | bodydata = { 279 | course_id: body['course_id']&.to_i, 280 | uuid: body['uuid']&.to_s, 281 | account_id: body['account_id']&.to_i, 282 | name: body['name']&.to_s, 283 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 284 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 285 | workflow_state: body['workflow_state']&.to_s, 286 | } 287 | 288 | when 'course_grade_change' 289 | 290 | bodydata = { 291 | user_id: body['user_id']&.to_i, 292 | course_id: body['course_id']&.to_i, 293 | workflow_state: body['workflow_state']&.to_s, 294 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 295 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 296 | current_score: body['current_score']&.to_f, 297 | old_current_score: body['old_current_score']&.to_f, 298 | final_score: body['final_score']&.to_f, 299 | old_final_score: body['old_final_score']&.to_f, 300 | unposted_current_score: body['unposted_current_score']&.to_f, 301 | old_unposted_current_score: body['old_unposted_current_score']&.to_f, 302 | unposted_final_score: body['unposted_final_score']&.to_f, 303 | old_unposted_final_score: body['old_unposted_final_score']&.to_f, 304 | } 305 | 306 | when 'course_progress' 307 | 308 | bodydata = { 309 | # body progress 310 | error_message: body['progress']['error'].nil? ? nil : body['progress']['error']['message'].to_s, 311 | requirement_count: body['progress']['requirement_count'].nil? ? nil : body['progress']['requirement_count'].to_i, 312 | requirement_completed_count: body['progress']['requirement_completed_count'].nil? ? nil : body['progress']['requirement_completed_count'].to_i, 313 | next_requirement_url: body['progress']['next_requirement_url'].nil? ? nil : body['progress']['next_requirement_url'].to_s, 314 | completed_at: body['progress']['completed_at'].nil? ? nil : default_timezone(body['progress']['completed_at']), 315 | # body user 316 | user_id: body['user']['id'].nil? ? nil : body['user']['id'].to_i, 317 | user_name: body['user']['name'].nil? ? nil : body['user']['name'].to_s, 318 | user_email: body['user']['email'].nil? ? nil : body['user']['email'].to_s, 319 | # body course 320 | course_id: body['course']['id'].nil? ? nil : body['course']['id'].to_i, 321 | course_name: body['course']['name'].nil? ? nil : body['course']['name'].to_s, 322 | account_id: body['course']['account_id'].nil? ? nil : body['course']['account_id'].to_i, 323 | sis_source_id: body['course']['sis_source_id'].nil? ? nil : body['course']['sis_source_id'].to_s, 324 | } 325 | 326 | when 'course_section_created' 327 | 328 | bodydata = { 329 | course_section_id: body['course_section_id']&.to_i, 330 | sis_source_id: body['sis_source_id']&.to_s, 331 | sis_batch_id: body['sis_batch_id']&.to_s, 332 | course_id: body['course_id']&.to_i, 333 | root_account_id: body['root_account_id']&.to_i, 334 | enrollment_term_id: body['enrollment_term_id']&.to_s, 335 | name: body['name']&.to_s, 336 | default_section: body['default_section']&.to_s, 337 | accepting_enrollments: body['accepting_enrollments']&.to_s, 338 | can_manually_enroll: body['can_manually_enroll']&.to_s, 339 | start_at: body['start_at'].nil? ? nil : default_timezone(body['start_at']), 340 | end_at: body['end_at'].nil? ? nil : default_timezone(body['end_at']), 341 | workflow_state: body['workflow_state']&.to_s, 342 | restrict_enrollments_to_section_dates: body['restrict_enrollments_to_section_dates']&.to_s, 343 | nonxlist_course_id: body['nonxlist_course_id']&.to_s, 344 | stuck_sis_fields: body['stuck_sis_fields'].length == 0 ? nil : body['stuck_sis_fields'].join(','), 345 | integration_id: body['integration_id']&.to_s, 346 | } 347 | 348 | when 'course_section_updated' 349 | 350 | bodydata = { 351 | course_section_id: body['course_section_id']&.to_i, 352 | sis_source_id: body['sis_source_id']&.to_s, 353 | sis_batch_id: body['sis_batch_id']&.to_s, 354 | course_id: body['course_id']&.to_i, 355 | root_account_id: body['root_account_id']&.to_i, 356 | enrollment_term_id: body['enrollment_term_id']&.to_s, 357 | name: body['name']&.to_s, 358 | default_section: body['default_section']&.to_s, 359 | accepting_enrollments: body['accepting_enrollments']&.to_s, 360 | can_manually_enroll: body['can_manually_enroll']&.to_s, 361 | start_at: body['start_at'].nil? ? nil : default_timezone(body['start_at']), 362 | end_at: body['end_at'].nil? ? nil : default_timezone(body['end_at']), 363 | workflow_state: body['workflow_state']&.to_s, 364 | restrict_enrollments_to_section_dates: body['restrict_enrollments_to_section_dates']&.to_s, 365 | nonxlist_course_id: body['nonxlist_course_id']&.to_s, 366 | stuck_sis_fields: body['stuck_sis_fields'].length == 0 ? nil : body['stuck_sis_fields'].join(','), 367 | integration_id: body['integration_id']&.to_s, 368 | } 369 | 370 | when 'course_updated' 371 | 372 | bodydata = { 373 | course_id: body['course_id']&.to_i, 374 | account_id: body['account_id']&.to_i, 375 | uuid: body['uuid']&.to_s, 376 | name: body['name']&.to_s, 377 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 378 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 379 | workflow_state: body['workflow_state']&.to_s, 380 | } 381 | 382 | when 'discussion_entry_created' 383 | 384 | bodydata = { 385 | user_id: body['user_id']&.to_i, 386 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 387 | discussion_entry_id: body['discussion_entry_id']&.to_i, 388 | parent_discussion_entry_id: body['parent_discussion_entry_id']&.to_i, 389 | parent_discussion_entry_author_id: body['parent_discussion_entry_author_id']&.to_i, 390 | discussion_topic_id: body['discussion_topic_id']&.to_i, 391 | text: body['text']&.to_s, 392 | } 393 | 394 | when 'discussion_entry_submitted' 395 | 396 | bodydata = { 397 | user_id: body['user_id']&.to_i, 398 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 399 | discussion_entry_id: body['discussion_entry_id']&.to_i, 400 | discussion_topic_id: body['discussion_topic_id']&.to_i, 401 | text: body['text']&.to_s, 402 | parent_discussion_entry_id: body['parent_discussion_entry_id']&.to_i, 403 | assignment_id: body['assignment_id']&.to_i, 404 | submission_id: body['submission_id']&.to_i, 405 | } 406 | 407 | when 'discussion_topic_created' 408 | 409 | bodydata = { 410 | discussion_topic_id: body['discussion_topic_id']&.to_i, 411 | is_announcement: body['is_announcement']&.to_s, 412 | title: body['title']&.to_s, 413 | body: body['body']&.to_s, 414 | assignment_id: body['assignment_id']&.to_i, 415 | context_id: body['context_id']&.to_i, 416 | context_type: body['context_type']&.to_s, 417 | workflow_state: body['workflow_state']&.to_s, 418 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 419 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 420 | } 421 | 422 | when 'discussion_topic_updated' 423 | 424 | bodydata = { 425 | discussion_topic_id: body['discussion_topic_id']&.to_i, 426 | is_announcement: body['is_announcement']&.to_s, 427 | title: body['title']&.to_s, 428 | body: body['body']&.to_s, 429 | assignment_id: body['body']&.to_i, 430 | context_id: body['context_id']&.to_i, 431 | context_type: body['context_type']&.to_s, 432 | workflow_state: body['workflow_state']&.to_s, 433 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 434 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 435 | } 436 | 437 | when 'enrollment_created' 438 | 439 | bodydata = { 440 | enrollment_id: body['enrollment_id']&.to_i, 441 | course_id: body['course_id']&.to_i, 442 | user_id: body['user_id']&.to_i, 443 | user_name: body['user_name']&.to_s, 444 | type: body['type']&.to_s, 445 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 446 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 447 | limit_privileges_to_course_section: body['limit_privileges_to_course_section']&.to_s, 448 | course_section_id: body['course_section_id']&.to_i, 449 | associated_user_id: body['associated_user_id']&.to_i, 450 | workflow_state: body['workflow_state']&.to_s, 451 | } 452 | 453 | when 'enrollment_state_created' 454 | 455 | bodydata = { 456 | enrollment_id: body['enrollment_id']&.to_i, 457 | state: body['state']&.to_s, 458 | state_started_at: body['state_started_at'].nil? ? nil : default_timezone(body['state_started_at']), 459 | state_is_current: body['state_is_current']&.to_s, 460 | state_valid_until: body['state_valid_until'].nil? ? nil : default_timezone(body['state_valid_until']), 461 | restricted_access: body['restricted_access']&.to_s, 462 | access_is_current: body['access_is_current']&.to_s, 463 | state_invalidated_at: body['state_invalidated_at'].nil? ? nil : default_timezone(body['state_invalidated_at']), 464 | state_recalculated_at: body['state_recalculated_at'].nil? ? nil : default_timezone(body['state_recalculated_at']), 465 | access_invalidated_at: body['access_invalidated_at'].nil? ? nil : default_timezone(body['access_invalidated_at']), 466 | access_recalculated_at: body['access_recalculated_at'].nil? ? nil : default_timezone(body['access_recalculated_at']), 467 | } 468 | 469 | when 'enrollment_state_updated' 470 | 471 | bodydata = { 472 | enrollment_id: body['enrollment_id']&.to_i, 473 | state: body['state']&.to_s, 474 | state_started_at: body['state_started_at'].nil? ? nil : default_timezone(body['state_started_at']), 475 | state_is_current: body['state_is_current']&.to_s, 476 | state_valid_until: body['state_valid_until'].nil? ? nil : default_timezone(body['state_valid_until']), 477 | restricted_access: body['restricted_access']&.to_s, 478 | access_is_current: body['access_is_current']&.to_s, 479 | state_invalidated_at: body['state_invalidated_at'].nil? ? nil : default_timezone(body['state_invalidated_at']), 480 | state_recalculated_at: body['state_recalculated_at'].nil? ? nil : default_timezone(body['state_recalculated_at']), 481 | access_invalidated_at: body['access_invalidated_at'].nil? ? nil : default_timezone(body['access_invalidated_at']), 482 | access_recalculated_at: body['access_recalculated_at'].nil? ? nil : default_timezone(body['access_recalculated_at']), 483 | } 484 | 485 | when 'enrollment_updated' 486 | 487 | bodydata = { 488 | enrollment_id: body['enrollment_id']&.to_i, 489 | course_id: body['course_id']&.to_i, 490 | user_id: body['user_id']&.to_i, 491 | user_name: body['user_name']&.to_s, 492 | type: body['type']&.to_s, 493 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 494 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 495 | limit_privileges_to_course_section: body['limit_privileges_to_course_section']&.to_s, 496 | course_section_id: body['course_section_id']&.to_i, 497 | associated_user_id: body['associated_user_id']&.to_i, 498 | workflow_state: body['workflow_state']&.to_s, 499 | } 500 | 501 | when 'grade_change' 502 | 503 | bodydata = { 504 | submission_id: body['submission_id']&.to_i, 505 | assignment_id: body['assignment_id']&.to_i, 506 | assignment_name: body['assignment_name']&.to_s, 507 | grade: body['grade']&.to_s, 508 | old_grade: body['old_grade']&.to_s, 509 | score: body['score']&.to_f, 510 | old_score: body['old_score']&.to_f, 511 | points_possible: body['points_possible']&.to_f, 512 | old_points_possible: body['old_points_possible']&.to_f, 513 | grader_id: body['grader_id']&.to_i, 514 | student_id: body['student_id']&.to_i, 515 | student_sis_id: body['student_sis_id']&.to_s, 516 | user_id: body['user_id']&.to_i, 517 | grading_complete: body['grading_complete']&.to_s, 518 | muted: body['muted']&.to_s 519 | } 520 | 521 | when 'group_category_created' 522 | 523 | bodydata = { 524 | group_category_id: body['group_category_id']&.to_i, 525 | group_category_name: body['group_category_name']&.to_s, 526 | context_id: body['context_id']&.to_i, 527 | context_type: body['context_type']&.to_s, 528 | group_limit: body['group_limit']&.to_i, 529 | } 530 | 531 | when 'group_category_updated' 532 | 533 | bodydata = { 534 | group_category_id: body['group_category_id']&.to_i, 535 | group_category_name: body['group_category_name']&.to_s, 536 | context_id: body['context_id']&.to_i, 537 | context_type: body['context_type']&.to_s, 538 | group_limit: body['group_limit']&.to_i, 539 | } 540 | 541 | when 'group_created' 542 | 543 | bodydata = { 544 | group_category_id: body['group_category_id']&.to_i, 545 | group_category_name: body['group_category_name']&.to_s, 546 | group_id: body['group_id']&.to_i, 547 | group_name: body['group_name']&.to_s, 548 | uuid: body['uuid']&.to_s, 549 | context_type: body['context_type']&.to_s, 550 | context_id: body['context_id']&.to_i, 551 | account_id: body['account_id']&.to_i, 552 | workflow_state: body['workflow_state']&.to_s, 553 | max_membership: body['max_membership']&.to_i, 554 | } 555 | 556 | when 'group_membership_created' 557 | 558 | bodydata = { 559 | group_membership_id: body['group_membership_id']&.to_i, 560 | user_id: body['user_id']&.to_i, 561 | group_id: body['group_id']&.to_i, 562 | group_name: body['group_name']&.to_s, 563 | group_category_id: body['group_category_id']&.to_i, 564 | group_category_name: body['group_category_name']&.to_s, 565 | workflow_state: body['workflow_state']&.to_s, 566 | } 567 | 568 | when 'group_membership_updated' 569 | 570 | bodydata = { 571 | group_membership_id: body['group_membership_id']&.to_i, 572 | user_id: body['user_id']&.to_i, 573 | group_id: body['group_id']&.to_i, 574 | group_name: body['group_name']&.to_s, 575 | group_category_id: body['group_category_id']&.to_i, 576 | group_category_name: body['group_category_name']&.to_s, 577 | workflow_state: body['workflow_state']&.to_s, 578 | } 579 | 580 | when 'group_updated' 581 | 582 | bodydata = { 583 | group_category_id: body['group_category_id']&.to_i, 584 | group_category_name: body['group_category_name']&.to_s, 585 | group_id: body['group_id']&.to_i, 586 | group_name: body['group_name']&.to_s, 587 | uuid: body['uuid']&.to_s, 588 | context_type: body['context_type']&.to_s, 589 | context_id: body['context_id']&.to_i, 590 | account_id: body['account_id']&.to_i, 591 | workflow_state: body['workflow_state']&.to_s, 592 | max_membership: body['max_membership']&.to_i, 593 | } 594 | 595 | when 'learning_outcome_created' 596 | 597 | bodydata = { 598 | learning_outcome_id: body['learning_outcome_id']&.to_i, 599 | context_id: body['context_id']&.to_i, 600 | context_type: body['context_type']&.to_s, 601 | display_name: body['display_name']&.to_s, 602 | short_description: body['short_description']&.to_s, 603 | description: body['description']&.to_s, 604 | vendor_guid: body['vendor_guid']&.to_s, 605 | calculation_method: body['calculation_method']&.to_s, 606 | calculation_int: body['calculation_int']&.to_s, 607 | rubric_criterion_description: body['rubric_criterion']['description']&.to_s, 608 | rubric_criterion_ratings: body['rubric_criterion']['ratings']&.to_json.to_s, 609 | rubric_criterion_mastery_points: body['rubric_criterion']['mastery_points']&.to_f, 610 | rubric_criterion_points_possible: body['rubric_criterion']['points_possible']&.to_f, 611 | title: body['title']&.to_s, 612 | workflow_state: body['workflow_state']&.to_s, 613 | } 614 | 615 | when 'learning_outcome_group_created' 616 | 617 | bodydata = { 618 | learning_outcome_group_id: body['learning_outcome_group_id']&.to_i, 619 | context_id: body['context_id']&.to_i, 620 | context_type: body['context_type']&.to_s, 621 | title: body['title']&.to_s, 622 | description: body['description']&.to_s, 623 | vendor_guid: body['vendor_guid']&.to_s, 624 | parent_outcome_group_id: body['parent_outcome_group_id']&.to_i, 625 | workflow_state: body['workflow_state']&.to_s, 626 | } 627 | 628 | when 'learning_outcome_group_updated' 629 | 630 | bodydata = { 631 | learning_outcome_group_id: body['learning_outcome_group_id']&.to_i, 632 | context_id: body['context_id']&.to_i, 633 | context_type: body['context_type']&.to_s, 634 | title: body['title']&.to_s, 635 | description: body['description']&.to_s, 636 | vendor_guid: body['vendor_guid']&.to_s, 637 | parent_outcome_group_id: body['parent_outcome_group_id']&.to_i, 638 | workflow_state: body['workflow_state']&.to_s, 639 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 640 | } 641 | 642 | when 'learning_outcome_link_created' 643 | 644 | bodydata = { 645 | learning_outcome_link_id: body['learning_outcome_link_id']&.to_i, 646 | learning_outcome_id: body['learning_outcome_id']&.to_i, 647 | learning_outcome_group_id: body['learning_outcome_group_id']&.to_i, 648 | context_id: body['context_id']&.to_i, 649 | context_type: body['context_type']&.to_s, 650 | workflow_state: body['workflow_state']&.to_s, 651 | } 652 | 653 | when 'learning_outcome_link_updated' 654 | 655 | bodydata = { 656 | learning_outcome_link_id: body['learning_outcome_link_id']&.to_i, 657 | learning_outcome_id: body['learning_outcome_id']&.to_i, 658 | learning_outcome_group_id: body['learning_outcome_group_id']&.to_i, 659 | context_id: body['context_id']&.to_i, 660 | context_type: body['context_type']&.to_s, 661 | workflow_state: body['workflow_state']&.to_s, 662 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 663 | } 664 | 665 | when 'learning_outcome_result_created' 666 | 667 | bodydata = { 668 | learning_outcome_id: body['learning_outcome_id']&.to_i, 669 | mastery: body['mastery']&.to_s, 670 | score: body['score']&.to_f, 671 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 672 | attempt: body['attempt']&.to_i, 673 | possible: body['possible']&.to_f, 674 | original_score: body['original_score']&.to_f, 675 | original_possible: body['original_possible']&.to_f, 676 | original_mastery: body['original_mastery']&.to_s, 677 | assessed_at: body['assessed_at'].nil? ? nil : default_timezone(body['updated_at']), 678 | title: body['title']&.to_s, 679 | percent: body['percent']&.to_f, 680 | } 681 | 682 | when 'learning_outcome_result_updated' 683 | 684 | bodydata = { 685 | learning_outcome_id: body['learning_outcome_id']&.to_i, 686 | mastery: body['mastery']&.to_s, 687 | score: body['score']&.to_f, 688 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 689 | attempt: body['attempt']&.to_i, 690 | possible: body['possible']&.to_f, 691 | original_score: body['original_score']&.to_f, 692 | original_possible: body['original_possible']&.to_f, 693 | original_mastery: body['original_mastery']&.to_s, 694 | assessed_at: body['assessed_at'].nil? ? nil : default_timezone(body['updated_at']), 695 | title: body['title']&.to_s, 696 | percent: body['percent']&.to_f, 697 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 698 | } 699 | 700 | when 'learning_outcome_updated' 701 | 702 | bodydata = { 703 | learning_outcome_id: body['learning_outcome_id']&.to_i, 704 | context_id: body['context_id']&.to_i, 705 | context_type: body['context_type']&.to_s, 706 | display_name: body['display_name']&.to_s, 707 | short_description: body['short_description']&.to_s, 708 | description: body['description']&.to_s, 709 | vendor_guid: body['vendor_guid']&.to_s, 710 | calculation_method: body['calculation_method']&.to_s, 711 | calculation_int: body['calculation_int']&.to_i, 712 | rubric_criterion_description: body['rubric_criterion']['description']&.to_s, 713 | rubric_criterion_ratings: body['rubric_criterion']['ratings']&.to_json.to_s, 714 | rubric_criterion_mastery_points: body['rubric_criterion']['mastery_points']&.to_f, 715 | rubric_criterion_points_possible: body['rubric_criterion']['points_possible']&.to_f, 716 | title: body['title']&.to_s, 717 | workflow_state: body['workflow_state']&.to_s, 718 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 719 | } 720 | 721 | when 'logged_in' 722 | 723 | bodydata = { 724 | redirect_url: body['redirect_url']&.to_s, 725 | } 726 | 727 | when 'logged_out' 728 | 729 | bodydata = {} 730 | 731 | when 'module_created' 732 | 733 | bodydata = { 734 | module_id: body['module_id']&.to_i, 735 | context_id: body['context_id']&.to_i, 736 | context_type: body['context_type']&.to_s, 737 | name: body['name']&.to_s, 738 | position: body['position']&.to_i, 739 | workflow_state: body['workflow_state']&.to_s, 740 | } 741 | 742 | when 'module_item_created' 743 | 744 | bodydata = { 745 | module_item_id: body['module_item_id']&.to_i, 746 | module_id: body['module_id']&.to_i, 747 | context_id: body['context_id']&.to_i, 748 | context_type: body['context_type']&.to_s, 749 | position: body['position']&.to_i, 750 | workflow_state: body['workflow_state']&.to_s, 751 | } 752 | 753 | when 'module_item_updated' 754 | 755 | bodydata = { 756 | module_item_id: body['module_item_id']&.to_i, 757 | module_id: body['module_id']&.to_i, 758 | context_id: body['context_id']&.to_i, 759 | context_type: body['context_type']&.to_s, 760 | position: body['position']&.to_i, 761 | workflow_state: body['workflow_state']&.to_s, 762 | } 763 | 764 | when 'module_updated' 765 | 766 | bodydata = { 767 | module_id: body['module_id']&.to_i, 768 | context_id: body['context_id']&.to_i, 769 | context_type: body['context_type']&.to_s, 770 | name: body['name']&.to_s, 771 | position: body['position']&.to_i, 772 | workflow_state: body['workflow_state']&.to_s, 773 | } 774 | 775 | when 'plagiarism_resubmit' 776 | 777 | bodydata = { 778 | submission_id: body['submission_id']&.to_i, 779 | assignment_id: body['assignment_id']&.to_i, 780 | user_id: body['user_id']&.to_i, 781 | submitted_at: body['submitted_at'].nil? ? nil : default_timezone(body['submitted_at']), 782 | lti_user_id: body['lti_user_id']&.to_s, 783 | graded_at: body['graded_at'].nil? ? nil : default_timezone(body['graded_at']), 784 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 785 | score: body['score']&.to_s, 786 | grade: body['grade']&.to_s, 787 | submission_type: body['submission_type']&.to_s, 788 | body: body['body']&.to_s, 789 | url: body['url']&.to_s, 790 | attempt: body['attempt']&.to_i, 791 | lti_assignment_id: body['lti_assignment_id']&.to_s, 792 | group_id: body['group_id']&.to_i, 793 | late: body['late']&.to_s, 794 | missing: body['missing']&.to_s, 795 | } 796 | 797 | when 'quiz_export_complete' 798 | 799 | bodydata = { 800 | assignment_resource_link_id: body['assignment']['resource_link_id'].nil? ? nil : body['assignment']['resource_link_id'].to_s, 801 | assignment_id: body['assignment']['assignment_id'].nil? ? nil : body['assignment']['assignment_id'].to_i, 802 | assignment_title: body['assignment']['title'].nil? ? nil : body['assignment']['title'].to_s, 803 | assignment_context_title: body['assignment']['context_title'].nil? ? nil : body['assignment']['context_title'].to_s, 804 | assignment_course_uuid: body['assignment']['course_uuid'].nil? ? nil : body['assignment']['course_uuid'].to_s, 805 | qti_export_url: body['qti_export']['url'].nil? ? nil : body['qti_export']['url'].to_s, 806 | } 807 | 808 | when 'quiz_submitted' 809 | 810 | bodydata = { 811 | submission_id: body['submission_id']&.to_i, 812 | quiz_id: body['quiz_id']&.to_i, 813 | } 814 | 815 | when 'quizzes.item_created' 816 | 817 | if body.key?('properties') 818 | properties = body['properties'] 819 | properties_rich_content_editor = properties.key?('rich_content_editor') ? properties['rich_content_editor'].to_json.to_s : nil 820 | properties_show_word_count = properties.key?('show_word_count') ? properties['show_word_count'].to_json.to_s : nil 821 | properties_shuffle_rules = properties.key?('shuffle_rules') ? properties['shuffle_rules'].to_json.to_s : nil 822 | properties_spell_check = properties.key?('spell_check') ? properties['spell_check'].to_json.to_s : nil 823 | properties_word_limit_max = properties.key?('word_limit_max') ? properties['word_limit_max'].to_json.to_s : nil 824 | properties_word_limit_min = properties.key?('word_limit_min') ? properties['word_limit_min'].to_json.to_s : nil 825 | properties_word_limit = properties.key?('word_limit') ? properties['word_limit'].to_json.to_s : nil 826 | properties_display_answers_paragraph = properties.key?('display_answers_paragraph') ? properties['display_answers_paragraph'].to_json.to_s : nil 827 | properties_include_labels = properties.key?('include_labels') ? properties['include_labels'].to_json.to_s : nil 828 | properties_top_label = properties.key?('top_label') ? properties['top_label'].to_json.to_s : nil 829 | properties_bottom_label = properties.key?('bottom_label') ? properties['bottom_label'].to_json.to_s : nil 830 | end 831 | 832 | if body.key?('interaction_data') 833 | interaction_data = body['interaction_data'] 834 | # "interaction_data": { 835 | # "choices": [{ 836 | # "id": "9ee5b221-7a55-44fe-bda3-deb656463c61", 837 | # "position": 1, 838 | # "item_body": "

Color

" 839 | # }, { 840 | # "id": "9b87a79a-fc75-45bb-b578-09c120453d9a", 841 | # "position": 3, 842 | # "item_body": "

Value

" 843 | # }, { 844 | # "id": "dfdf22dc-d03c-457f-96ee-4c5a691373de", 845 | # "position": 4, 846 | # "item_body": "

Line

" 847 | # }] 848 | # }, 849 | interaction_data_choices = interaction_data.key?('choices') ? interaction_data['choices'].to_json.to_s : nil 850 | # "interaction_data": { 851 | # "true_choice": "True", 852 | # "false_choice": "False" 853 | # }, 854 | interaction_data_true_choice = interaction_data.key?('true_choice') ? interaction_data['true_choice'].to_json.delete('"').to_s : nil 855 | interaction_data_false_choice = interaction_data.key?('false_choice') ? interaction_data['false_choice'].to_json.delete('"').to_s : nil 856 | # "interaction_data": { 857 | # "essay": null, 858 | # "rce": true, 859 | # "spell_check": false, 860 | # "word_count": false, 861 | # "word_limit_enabled": false, 862 | # "word_limit_max": null, 863 | # "word_limit_min": null, 864 | # "file_upload": false 865 | # }, 866 | interaction_data_essay = interaction_data.key?('essay') ? interaction_data['essay'].to_json.to_s : nil 867 | interaction_data_rce = interaction_data.key?('rce') ? interaction_data['rce'].to_json.to_s : nil 868 | interaction_data_spell_check = interaction_data.key?('spell_check') ? interaction_data['spell_check'].to_json.to_s : nil 869 | interaction_data_word_count = interaction_data.key?('word_count') ? interaction_data['word_count'].to_json.to_s : nil 870 | interaction_data_word_limit_enabled = interaction_data.key?('word_limit_enabled') ? interaction_data['word_limit_enabled'].to_json.to_s : nil 871 | interaction_data_word_limit_max = interaction_data.key?('word_limit_max') ? interaction_data['word_limit_max'].to_json.to_s : nil 872 | interaction_data_word_limit_min = interaction_data.key?('word_limit_min') ? interaction_data['word_limit_min'].to_json.to_s : nil 873 | interaction_data_file_upload = interaction_data.key?('file_upload') ? interaction_data['file_upload'].to_json.to_s : nil 874 | end 875 | 876 | if body.key?('scoring_data') 877 | scoring_data = body['scoring_data'].nil? ? nil : body['scoring_data'].to_json.to_s 878 | end 879 | 880 | bodydata = { 881 | # body 882 | id: body['id']&.to_i, 883 | title: body['title']&.to_s, 884 | label: body['label']&.to_s, 885 | item_body: body['item_body']&.to_s, 886 | properties_rich_content_editor: properties_rich_content_editor, 887 | properties_show_word_count: properties_show_word_count, 888 | properties_shuffle_rules: properties_shuffle_rules, 889 | properties_spell_check: properties_spell_check, 890 | properties_word_limit_max: properties_word_limit_max, 891 | properties_word_limit_min: properties_word_limit_min, 892 | properties_word_limit: properties_word_limit, 893 | properties_display_answers_paragraph: properties_display_answers_paragraph, 894 | properties_include_labels: properties_include_labels, 895 | properties_top_label: properties_top_label, 896 | properties_bottom_label: properties_bottom_label, 897 | interaction_data_choices: interaction_data_choices, 898 | interaction_data_true_choice: interaction_data_true_choice, 899 | interaction_data_false_choice: interaction_data_false_choice, 900 | interaction_data_essay: interaction_data_essay, 901 | interaction_data_rce: interaction_data_rce, 902 | interaction_data_spell_check: interaction_data_spell_check, 903 | interaction_data_word_count: interaction_data_word_count, 904 | interaction_data_word_limit_enabled: interaction_data_word_limit_enabled, 905 | interaction_data_word_limit_max: interaction_data_word_limit_max, 906 | interaction_data_word_limit_min: interaction_data_word_limit_min, 907 | interaction_data_file_upload: interaction_data_file_upload, 908 | user_response_type: body['user_response_type']&.to_s, 909 | outcome_alignment_set_guid: body['outcome_alignment_set_guid']&.to_s, 910 | scoring_data: scoring_data, 911 | scoring_algorithm: body['scoring_algorithm']&.to_s, 912 | } 913 | 914 | when 'quizzes.item_updated' 915 | 916 | if body.key?('properties') 917 | properties = body['properties'] 918 | properties_rich_content_editor = properties.key?('rich_content_editor') ? properties['rich_content_editor'].to_json.to_s : nil 919 | properties_show_word_count = properties.key?('show_word_count') ? properties['show_word_count'].to_json.to_s : nil 920 | properties_shuffle_rules = properties.key?('shuffle_rules') ? properties['shuffle_rules'].to_json.to_s : nil 921 | properties_spell_check = properties.key?('spell_check') ? properties['spell_check'].to_json.to_s : nil 922 | properties_word_limit_max = properties.key?('word_limit_max') ? properties['word_limit_max'].to_json.to_s : nil 923 | properties_word_limit_min = properties.key?('word_limit_min') ? properties['word_limit_min'].to_json.to_s : nil 924 | properties_word_limit = properties.key?('word_limit') ? properties['word_limit'].to_json.to_s : nil 925 | properties_display_answers_paragraph = properties.key?('display_answers_paragraph') ? properties['display_answers_paragraph'].to_json.to_s : nil 926 | properties_include_labels = properties.key?('include_labels') ? properties['include_labels'].to_json.to_s : nil 927 | properties_top_label = properties.key?('top_label') ? properties['top_label'].to_json.to_s : nil 928 | properties_bottom_label = properties.key?('bottom_label') ? properties['bottom_label'].to_json.to_s : nil 929 | end 930 | 931 | if body.key?('interaction_data') 932 | interaction_data = body['interaction_data'] 933 | interaction_data_choices = interaction_data.key?('choices') ? interaction_data['choices'].to_json.to_s : nil 934 | interaction_data_true_choice = interaction_data.key?('true_choice') ? interaction_data['true_choice'].to_json.delete('"').to_s : nil 935 | interaction_data_false_choice = interaction_data.key?('false_choice') ? interaction_data['false_choice'].to_json.delete('"').to_s : nil 936 | interaction_data_essay = interaction_data.key?('essay') ? interaction_data['essay'].to_json.to_s : nil 937 | interaction_data_rce = interaction_data.key?('rce') ? interaction_data['rce'].to_json.to_s : nil 938 | interaction_data_spell_check = interaction_data.key?('spell_check') ? interaction_data['spell_check'].to_json.to_s : nil 939 | interaction_data_word_count = interaction_data.key?('word_count') ? interaction_data['word_count'].to_json.to_s : nil 940 | interaction_data_word_limit_enabled = interaction_data.key?('word_limit_enabled') ? interaction_data['word_limit_enabled'].to_json.to_s : nil 941 | interaction_data_word_limit_max = interaction_data.key?('word_limit_max') ? interaction_data['word_limit_max'].to_json.to_s : nil 942 | interaction_data_word_limit_min = interaction_data.key?('word_limit_min') ? interaction_data['word_limit_min'].to_json.to_s : nil 943 | interaction_data_file_upload = interaction_data.key?('file_upload') ? interaction_data['file_upload'].to_json.to_s : nil 944 | end 945 | 946 | if body.key?('scoring_data') 947 | scoring_data = body['scoring_data'].nil? ? nil : body['scoring_data'].to_json.to_s 948 | end 949 | 950 | bodydata = { 951 | # body 952 | id: body['id']&.to_i, 953 | title: body['title']&.to_s, 954 | label: body['label']&.to_s, 955 | item_body: body['item_body']&.to_s, 956 | properties_rich_content_editor: properties_rich_content_editor, 957 | properties_show_word_count: properties_show_word_count, 958 | properties_shuffle_rules: properties_shuffle_rules, 959 | properties_spell_check: properties_spell_check, 960 | properties_word_limit_max: properties_word_limit_max, 961 | properties_word_limit_min: properties_word_limit_min, 962 | properties_word_limit: properties_word_limit, 963 | properties_display_answers_paragraph: properties_display_answers_paragraph, 964 | properties_include_labels: properties_include_labels, 965 | properties_top_label: properties_top_label, 966 | properties_bottom_label: properties_bottom_label, 967 | interaction_data_choices: interaction_data_choices, 968 | interaction_data_true_choice: interaction_data_true_choice, 969 | interaction_data_false_choice: interaction_data_false_choice, 970 | interaction_data_essay: interaction_data_essay, 971 | interaction_data_rce: interaction_data_rce, 972 | interaction_data_spell_check: interaction_data_spell_check, 973 | interaction_data_word_count: interaction_data_word_count, 974 | interaction_data_word_limit_enabled: interaction_data_word_limit_enabled, 975 | interaction_data_word_limit_max: interaction_data_word_limit_max, 976 | interaction_data_word_limit_min: interaction_data_word_limit_min, 977 | interaction_data_file_upload: interaction_data_file_upload, 978 | user_response_type: body['user_response_type']&.to_s, 979 | outcome_alignment_set_guid: body['outcome_alignment_set_guid']&.to_s, 980 | scoring_data: scoring_data, 981 | scoring_algorithm: body['scoring_algorithm']&.to_s, 982 | } 983 | 984 | when 'quizzes-lti.grade_changed' 985 | 986 | bodydata = { 987 | user_uuid: body['user_uuid']&.to_s, 988 | quiz_id: body['quiz_id']&.to_i, 989 | score_to_keep: body['score_to_keep']&.to_s, 990 | } 991 | 992 | when 'quizzes_next_quiz_duplicated' 993 | 994 | bodydata = { 995 | new_assignment_id: body['new_assignment_id']&.to_i, 996 | original_course_uuid: body['original_course_uuid']&.to_s, 997 | original_resource_link_id: body['original_resource_link_id']&.to_s, 998 | new_course_uuid: body['new_course_uuid']&.to_s, 999 | new_course_resource_link_id: body['new_course_resource_link_id']&.to_s, 1000 | new_course_id: body['new_course_id']&.to_s, 1001 | new_resource_link_id: body['new_resource_link_id']&.to_s, 1002 | } 1003 | 1004 | when 'quizzes.qti_import_completed' 1005 | 1006 | bodydata = { 1007 | qti_type: body['qti_type']&.to_s, 1008 | quiz_id: body['quiz_id']&.to_i, 1009 | success: body['success']&.to_s, 1010 | } 1011 | 1012 | when 'quizzes.quiz_clone_job_created' 1013 | 1014 | bodydata = { 1015 | id: body['id']&.to_s, 1016 | original_quiz_id: body['original_quiz_id']&.to_s, 1017 | status: body['status']&.to_s, 1018 | } 1019 | 1020 | when 'quizzes.quiz_clone_job_updated' 1021 | 1022 | bodydata = { 1023 | id: body['id']&.to_s, 1024 | original_quiz_id: body['original_quiz_id']&.to_s, 1025 | status: body['status']&.to_s, 1026 | cloned_quiz_id: body['cloned_quiz_id']&.to_s, 1027 | } 1028 | 1029 | when 'quizzes.quiz_created' 1030 | 1031 | bodydata = { 1032 | id: body['id']&.to_i, 1033 | title: body['title']&.to_s, 1034 | instructions: body['instructions']&.to_s, 1035 | context_id: body['context_id']&.to_i, 1036 | owner: body['owner']&.to_s, 1037 | has_time_limit: body['has_time_limit']&.to_s, 1038 | due_at: body['due_at'].nil? ? nil : default_timezone(body['due_at']), 1039 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 1040 | session_time_limit_in_seconds: body['session_time_limit_in_seconds']&.to_i, 1041 | shuffle_questions: body['shuffle_questions']&.to_s, 1042 | shuffle_answers: body['shuffle_answers']&.to_s, 1043 | status: body['status']&.to_s, 1044 | outcome_alignment_set_guid: body['outcome_alignment_set_guid']&.to_s, 1045 | } 1046 | 1047 | when 'quizzes.quiz_graded' 1048 | 1049 | bodydata = { 1050 | quiz_session_id: body['quiz_session_id']&.to_i, 1051 | quiz_session_result_id: body['quiz_session_result_id']&.to_i, 1052 | grader_id: body['grader_id']&.to_i, 1053 | grading_method: body['grading_method']&.to_s, 1054 | status: body['status']&.to_s, 1055 | score: body['score']&.to_f, 1056 | fudge_points: body['fudge_points']&.to_s, 1057 | points_possible: body['points_possible']&.to_f, 1058 | percentage: body['percentage']&.to_f, 1059 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1060 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 1061 | } 1062 | 1063 | # when 'quizzes.quiz_session_graded' 1064 | 1065 | when 'quizzes.quiz_session_submitted' 1066 | 1067 | bodydata = { 1068 | accepted_student_access_code_at: body['accepted_student_access_code_at'].nil? ? nil : default_timezone(body['accepted_student_access_code_at']), 1069 | allow_backtracking: body['allow_backtracking']&.to_s, 1070 | attempt: body['attempt']&.to_i, 1071 | authoritative_result_id: body['authoritative_result_id']&.to_i, 1072 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1073 | end_at: body['end_at'].nil? ? nil : default_timezone(body['end_at']), 1074 | grade_passback_guid: body['grade_passback_guid']&.to_s, 1075 | graded_url: body['graded_url']&.to_s, 1076 | id: body['id']&.to_i, 1077 | invalidated_student_access_code_at: body['invalidated_student_access_code_at'].nil? ? nil : default_timezone(body['invalidated_student_access_code_at']), 1078 | one_at_a_time_type: body['one_at_a_time_type']&.to_s, 1079 | passback_url: body['passback_url']&.to_s, 1080 | points_possible: body['points_possible']&.to_f, 1081 | quiz_id: body['quiz_id']&.to_i, 1082 | session_items_count: body['session_items_count']&.to_i, 1083 | start_at: body['start_at'].nil? ? nil : default_timezone(body['start_at']), 1084 | status: body['status']&.to_s, 1085 | submitted_at: body['submitted_at'].nil? ? nil : default_timezone(body['submitted_at']), 1086 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 1087 | exclude_from_stats: body['exclude_from_stats']&.to_s, 1088 | } 1089 | 1090 | when 'quizzes.quiz_session_ungraded' 1091 | 1092 | bodydata = { 1093 | accepted_student_access_code_at: body['accepted_student_access_code_at'].nil? ? nil : default_timezone(body['accepted_student_access_code_at']), 1094 | allow_backtracking: body['allow_backtracking']&.to_s, 1095 | attempt: body['attempt']&.to_i, 1096 | authoritative_result_id: body['authoritative_result_id']&.to_i, 1097 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1098 | end_at: body['end_at'].nil? ? nil : default_timezone(body['end_at']), 1099 | grade_passback_guid: body['grade_passback_guid']&.to_s, 1100 | graded_url: body['graded_url']&.to_s, 1101 | id: body['id']&.to_i, 1102 | invalidated_student_access_code_at: body['invalidated_student_access_code_at'].nil? ? nil : default_timezone(body['invalidated_student_access_code_at']), 1103 | one_at_a_time_type: body['one_at_a_time_type']&.to_s, 1104 | passback_url: body['passback_url']&.to_s, 1105 | points_possible: body['points_possible']&.to_f, 1106 | quiz_id: body['quiz_id']&.to_i, 1107 | session_items_count: body['session_items_count']&.to_i, 1108 | start_at: body['start_at'].nil? ? nil : default_timezone(body['start_at']), 1109 | status: body['status']&.to_s, 1110 | submitted_at: body['submitted_at'].nil? ? nil : default_timezone(body['submitted_at']), 1111 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 1112 | exclude_from_stats: body['exclude_from_stats']&.to_s, 1113 | } 1114 | 1115 | when 'quizzes.quiz_updated' 1116 | 1117 | bodydata = { 1118 | id: body['id']&.to_i, 1119 | title: body['title']&.to_s, 1120 | instructions: body['instructions']&.to_s, 1121 | context_id: body['context_id']&.to_i, 1122 | owner: body['owner']&.to_s, 1123 | has_time_limit: body['has_time_limit']&.to_s, 1124 | due_at: body['due_at'].nil? ? nil : default_timezone(body['due_at']), 1125 | lock_at: body['lock_at'].nil? ? nil : default_timezone(body['lock_at']), 1126 | session_time_limit_in_seconds: body['session_time_limit_in_seconds']&.to_i, 1127 | shuffle_answers: body['shuffle_answers']&.to_s, 1128 | shuffle_questions: body['shuffle_questions']&.to_s, 1129 | status: body['status']&.to_s, 1130 | outcome_alignment_set_guid: body['outcome_alignment_set_guid']&.to_s, 1131 | } 1132 | 1133 | when 'quiz_caliper.quiz_session_created' 1134 | 1135 | bodydata = { 1136 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1137 | end_at: body['end_at'].nil? ? nil : default_timezone(body['end_at']), 1138 | id: body['id']&.to_i, 1139 | start_at: body['start_at'].nil? ? nil : default_timezone(body['start_at']), 1140 | status: body['status']&.to_s, 1141 | submitted_at: body['submitted_at'].nil? ? nil : default_timezone(body['submitted_at']), 1142 | } 1143 | 1144 | when 'quiz_caliper.quiz_session_updated' 1145 | 1146 | bodydata = { 1147 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1148 | end_at: body['end_at'].nil? ? nil : default_timezone(body['end_at']), 1149 | id: body['id']&.to_i, 1150 | start_at: body['start_at'].nil? ? nil : default_timezone(body['start_at']), 1151 | status: body['status']&.to_s, 1152 | submitted_at: body['submitted_at'].nil? ? nil : default_timezone(body['submitted_at']), 1153 | } 1154 | 1155 | when 'quiz_caliper.asset_accessed' 1156 | 1157 | bodydata = { 1158 | id: body['id']&.to_i, 1159 | title: body['title']&.to_s, 1160 | } 1161 | 1162 | when 'quiz_caliper.quiz_created' 1163 | 1164 | bodydata = { 1165 | id: body['id']&.to_i, 1166 | title: body['title']&.to_s, 1167 | } 1168 | 1169 | when 'quiz_caliper.quiz_started' 1170 | 1171 | bodydata = { 1172 | id: body['id']&.to_i, 1173 | title: body['title']&.to_s, 1174 | } 1175 | 1176 | when 'quiz_caliper.quiz_submitted' 1177 | 1178 | bodydata = { 1179 | id: body['id']&.to_i, 1180 | title: body['title']&.to_s, 1181 | } 1182 | 1183 | when 'quiz_caliper.quiz_updated' 1184 | 1185 | bodydata = { 1186 | id: body['id']&.to_i, 1187 | title: body['title']&.to_s, 1188 | } 1189 | 1190 | when 'quiz_caliper.item_created' 1191 | 1192 | bodydata = { 1193 | id: body['id']&.to_i, 1194 | title: body['title']&.to_s, 1195 | label: body['label']&.to_s, 1196 | } 1197 | 1198 | when 'quiz_caliper.item_updated' 1199 | bodydata = { 1200 | id: body['id']&.to_i, 1201 | title: body['title']&.to_s, 1202 | label: body['label']&.to_s, 1203 | } 1204 | 1205 | when 'sis_batch_created' 1206 | 1207 | bodydata = { 1208 | sis_batch_id: body['sis_batch_id']&.to_s, 1209 | account_id: body['account_id']&.to_i, 1210 | workflow_state: body['workflow_state']&.to_s, 1211 | } 1212 | 1213 | when 'sis_batch_updated' 1214 | 1215 | bodydata = { 1216 | sis_batch_id: body['sis_batch_id']&.to_s, 1217 | account_id: body['account_id']&.to_i, 1218 | workflow_state: body['workflow_state']&.to_s, 1219 | } 1220 | 1221 | when 'submission_comment_created' 1222 | 1223 | bodydata = { 1224 | submission_comment_id: body['submission_comment_id']&.to_i, 1225 | submission_id: body['submission_id']&.to_i, 1226 | user_id: body['user_id']&.to_i, 1227 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1228 | attachment_ids: body['attachment_ids'].length == 0 ? nil : body['attachment_ids'].join(','), 1229 | body: body['body']&.to_i, 1230 | } 1231 | 1232 | when 'submission_created' 1233 | 1234 | bodydata = { 1235 | submission_id: body['submission_id']&.to_i, 1236 | assignment_id: body['assignment_id']&.to_i, 1237 | user_id: body['user_id']&.to_i, 1238 | submitted_at: body['submitted_at'].nil? ? nil : default_timezone(body['submitted_at']), 1239 | lti_user_id: body['lti_user_id']&.to_s, 1240 | graded_at: body['graded_at'].nil? ? nil : default_timezone(body['graded_at']), 1241 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 1242 | score: body['score']&.to_f, 1243 | grade: body['grade']&.to_s, 1244 | submission_type: body['submission_type']&.to_s, 1245 | body: body['body']&.to_s, 1246 | url: body['url']&.to_s, 1247 | attempt: body['attempt']&.to_i, 1248 | lti_assignment_id: body['lti_assignment_id']&.to_s, 1249 | group_id: body['group_id']&.to_i, 1250 | late: body['late']&.to_s, 1251 | missing: body['missing']&.to_s, 1252 | } 1253 | 1254 | when 'submission_updated' 1255 | 1256 | bodydata = { 1257 | submission_id: body['submission_id']&.to_i, 1258 | assignment_id: body['assignment_id']&.to_i, 1259 | user_id: body['user_id']&.to_i, 1260 | submitted_at: body['submitted_at'].nil? ? nil : default_timezone(body['submitted_at']), 1261 | lti_user_id: body['lti_user_id']&.to_s, 1262 | lti_assignment_id: body['lti_assignment_id']&.to_s, 1263 | graded_at: body['graded_at'].nil? ? nil : default_timezone(body['graded_at']), 1264 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 1265 | score: body['score']&.to_f, 1266 | grade: body['grade']&.to_s, 1267 | submission_type: body['submission_type']&.to_s, 1268 | body: body['body']&.to_s, 1269 | url: body['url']&.to_s, 1270 | attempt: body['attempt']&.to_i, 1271 | group_id: body['group_id']&.to_i, 1272 | late: body['late']&.to_s, 1273 | missing: body['missing']&.to_s, 1274 | } 1275 | 1276 | when 'syllabus_updated' 1277 | 1278 | bodydata = { 1279 | course_id: body['course_id']&.to_i, 1280 | syllabus_body: body['syllabus_body']&.to_s, 1281 | old_syllabus_body: body['old_syllabus_body']&.to_s, 1282 | } 1283 | 1284 | when 'user_account_association_created' 1285 | 1286 | bodydata = { 1287 | user_id: body['user_id']&.to_i, 1288 | account_id: body['account_id']&.to_i, 1289 | account_uuid: body['account_uuid']&.to_s, 1290 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1291 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 1292 | is_admin: body['is_admin']&.to_s, 1293 | } 1294 | 1295 | when 'user_created' 1296 | 1297 | bodydata = { 1298 | user_id: body['user_id']&.to_i, 1299 | uuid: body['uuid']&.to_s, 1300 | name: body['name']&.to_s, 1301 | short_name: body['short_name']&.to_s, 1302 | workflow_state: body['workflow_state']&.to_s, 1303 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1304 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 1305 | user_login: body['user_login']&.to_s, 1306 | user_sis_id: body['user_sis_id']&.to_s, 1307 | } 1308 | 1309 | when 'user_updated' 1310 | 1311 | bodydata = { 1312 | user_id: body['user_id']&.to_i, 1313 | uuid: body['uuid']&.to_s, 1314 | name: body['name']&.to_s, 1315 | short_name: body['short_name']&.to_s, 1316 | workflow_state: body['workflow_state']&.to_s, 1317 | created_at: body['created_at'].nil? ? nil : default_timezone(body['created_at']), 1318 | updated_at: body['updated_at'].nil? ? nil : default_timezone(body['updated_at']), 1319 | user_login: body['user_login']&.to_s, 1320 | user_sis_id: body['user_sis_id']&.to_s, 1321 | } 1322 | 1323 | when 'wiki_page_created' 1324 | 1325 | bodydata = { 1326 | wiki_page_id: body['wiki_page_id']&.to_i, 1327 | title: body['title']&.to_s, 1328 | body: body['body']&.to_s, 1329 | } 1330 | 1331 | when 'wiki_page_deleted' 1332 | 1333 | bodydata = { 1334 | wiki_page_id: body['wiki_page_id']&.to_i, 1335 | title: body['title']&.to_s, 1336 | } 1337 | 1338 | when 'wiki_page_updated' 1339 | 1340 | bodydata = { 1341 | wiki_page_id: body['wiki_page_id']&.to_i, 1342 | title: body['title']&.to_s, 1343 | body: body['body']&.to_s, 1344 | old_title: body['old_title']&.to_s, 1345 | old_body: body['old_body']&.to_s, 1346 | } 1347 | 1348 | # catch and save events, we don't have configured or we aren't expecting 1349 | else 1350 | collect_unknown(event_name, event_data) 1351 | # return if the message cannot be prepped for import 1352 | return 1353 | end 1354 | 1355 | # return parsed event bodydata 1356 | bodydata 1357 | end 1358 | 1359 | def _canvas(event_data) 1360 | meta = _metadata(event_data) 1361 | body = _bodydata(event_data) 1362 | return if !body.is_a? Hash 1363 | 1364 | # check if we missed any new data 1365 | missing_meta(event_data, meta) 1366 | missing_body(event_data, body) 1367 | 1368 | # merge metadata and bodydata, prevent duplicate fields 1369 | # return event data - parsed, flattened, ready for sql 1370 | meta.merge!(body) 1371 | end 1372 | end --------------------------------------------------------------------------------