seth
/
jam-cloud
mirror of https://bitbucket.org/jamkazam/jam-cloud.git
1
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

VRFS-1483 error collector; cleanup

This commit is contained in:
Jonathan Kolyer 2014-04-05 21:25:47 +00:00
parent e59d9cbe7a
commit 15090e3c70
10 changed files with 116 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
admin/app/admin/email_batch.rb
View File

@ -4,18 +4,26 @@ ActiveAdmin.register JamRuby::EmailBatch, :as => 'Batch Emails' do
config.sort_order = 'updated_at DESC'
config.batch_actions = false
# config.clear_action_items!
config.clear_action_items!
config.filters = false
form :partial => 'form'
action_item do
link_to('New Batch Email', new_admin_batch_email_path)
end
action_item :only => [:show, :edit] do
link_to('Clone Batch Email', batch_clone_admin_batch_email_path(resource.id))
end
index do
column 'Subject' do |bb| bb.subject end
column 'Updated' do |bb| bb.updated_at end
column 'Created' do |bb| bb.created_at end
column 'From' do |bb| bb.from_email end
column 'Status' do |bb| bb.aasm_state end
column 'Test Emails' do |bb| bb.test_emails end
column 'Email Count' do |bb| bb.candidate_count end
column 'Email Count' do |bb| bb.opt_in_count end
column 'Sent Count' do |bb| bb.sent_count end
column 'Started' do |bb| bb.started_at end
column 'Completed' do |bb| bb.completed_at end
@ -36,18 +44,6 @@ ActiveAdmin.register JamRuby::EmailBatch, :as => 'Batch Emails' do
default_actions
end
# action_item :only => :show do
# link_to("Send Test Batch (#{resource.test_count})",
# batch_test_admin_batch_email_path(resource.id),
# :confirm => "Run test batch with #{resource.test_count} emails?")
# end
# action_item :only => :show do
# link_to("Send Live Batch (#{User.email_opt_in.count})",
# batch_send_admin_batch_email_path(resource.id),
# :confirm => "Run LIVE batch with #{User.email_opt_in.count} emails?")
# end
show :title => 'Batch Email' do |obj|
panel 'Email Contents' do
attributes_table_for obj do
@ -71,7 +67,11 @@ ActiveAdmin.register JamRuby::EmailBatch, :as => 'Batch Emails' do
end
end
column do
panel 'Send Results' do
panel 'Send Chunks' do
table_for(sets = obj.email_batch_sets) do
column :started_at do |sets| sets.started_at.strftime('%b %d %Y, %H:%M') end
column :batch_count do |sets| sets.batch_count end
end
end
end
end
@ -82,7 +82,6 @@ ActiveAdmin.register JamRuby::EmailBatch, :as => 'Batch Emails' do
def create
batch = EmailBatch.create_with_params(params[:jam_ruby_email_batch])
redirect_to admin_batch_email_path(batch.id)
# redirect_to admin_batch_emails_path
end
def update
@ -93,21 +92,18 @@ ActiveAdmin.register JamRuby::EmailBatch, :as => 'Batch Emails' do
end
member_action :batch_test, :method => :get do
batch = EmailBatch.find(params[:id])
batch.send_test_batch
redirect_to admin_batch_email_path(batch.id)
resource.send_test_batch
redirect_to admin_batch_email_path(resource.id)
end
member_action :batch_send, :method => :get do
batch = EmailBatch.find(params[:id])
batch.deliver_batch
redirect_to admin_batch_email_path(batch.id)
resource.deliver_batch
redirect_to admin_batch_email_path(resource.id)
end
member_action :batch_clone, :method => :get do
batch = EmailBatch.find(params[:id])
batch.clone
redirect_to edit_admin_batch_email_path(batch.id)
bb = resource.clone
redirect_to edit_admin_batch_email_path(bb.id)
end
end

12
admin/app/admin/email_error_batch.rb
View File

@ -1,17 +1,14 @@
ActiveAdmin.register JamRuby::EmailError, :as => 'Batch Email Errors' do
ActiveAdmin.register JamRuby::EmailError, :as => 'Email Errors' do
menu :label => 'Batch Errors', :parent => 'Email'
menu :label => 'Email Errors', :parent => 'Email'
config.batch_actions = false
config.filters = false
config.clear_action_items!
index do
column 'Batch' do |eerr|
link_to(truncate(eerr.batch_subject, :length => 40), admin_batch_email_path(eerr.email_batch_id))
end
column 'User' do |eerr|
eerr.user ? link_to(eerr.user.name, batch_action_admin_users_path(eerr.user_id)) : 'N/A'
eerr.user ? link_to(eerr.user.name, admin_user_path(eerr.user_id)) : 'N/A'
end
column 'Error Type' do |eerr| eerr.error_type end
column 'Email Address' do |eerr| eerr.email_address end
@ -24,8 +21,7 @@ ActiveAdmin.register JamRuby::EmailError, :as => 'Batch Email Errors' do
def scoped_collection
@eerrors ||= end_of_association_chain
.where(['email_batch_id IS NOT NULL'])
.includes([:user, :email_batch])
.includes([:user])
.order('email_date DESC')
end
end

3
db/up/email_batch.sql
View File

@ -14,7 +14,6 @@ CREATE INDEX email_batch_set_fkidx ON email_batch_sets (email_batch_id);
CREATE TABLE email_errors (
id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(),
email_batch_id VARCHAR(64) REFERENCES email_batches(id) ON DELETE CASCADE,
user_id VARCHAR(64) REFERENCES users(id) ON DELETE CASCADE,
error_type VARCHAR(32),
@ -27,5 +26,5 @@ CREATE TABLE email_errors (
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX email_error_batch_fkidx ON email_errors(email_batch_id);
CREATE INDEX email_error_user_fkidx ON email_errors(user_id);
CREATE INDEX email_error_address_idx ON email_errors(email_address);

2
db/up/emails.sql
View File

@ -8,7 +8,7 @@ CREATE TABLE email_batches (
test_emails TEXT NOT NULL default 'test@jamkazam.com',
candidate_count INTEGER NOT NULL default 0,
opt_in_count INTEGER NOT NULL default 0,
sent_count INTEGER NOT NULL default 0,
lock_version INTEGER,

4
ruby/lib/jam_ruby/app/mailers/batch_mailer.rb
View File

@ -24,12 +24,12 @@ module JamRuby
def send_batch_email(batch_id, user_ids)
users = User.find_all_by_id(user_ids)
batch = EmailBatch.where(:id => batch_id).limit(1).first
batch = EmailBatch.find(batch_id)
self._send_batch(batch, users)
end
def send_batch_email_test(batch_id)
batch = EmailBatch.where(:id => batch_id).limit(1).first
batch = EmailBatch.find(batch_id)
users = batch.test_users
self._send_batch(batch, users)
end

89
ruby/lib/jam_ruby/models/email_batch.rb
View File

@ -3,12 +3,11 @@ module JamRuby
self.table_name = "email_batches"
has_many :email_batch_sets, :class_name => 'JamRuby::EmailBatchSet'
has_many :email_errors, :class_name => 'JamRuby::EmailError'
attr_accessible :from_email, :subject, :test_emails, :body
attr_accessible :lock_version, :candidate_count, :sent_count, :started_at, :completed_at
attr_accessible :lock_version, :opt_in_count, :sent_count, :started_at, :completed_at
default_scope :order => 'updated_at DESC'
default_scope :order => 'created_at DESC'
VAR_FIRST_NAME = '@FIRSTNAME'
VAR_LAST_NAME = '@LASTNAME'
@ -24,8 +23,6 @@ Hello #{VAR_FIRST_NAME},
<p>Paragraph 2 ... #{VAR_FIRST_NAME} will be replaced by users first name</p>
<p>Thanks for using JamKazam!</p>
The JamKazam Team
FOO
include AASM
@ -33,45 +30,36 @@ FOO
state :pending, :initial => true
state :testing
state :tested
state :batching
state :batched
state :confirming
state :confirmed
state :delivering
state :delivered
state :disabled
event :enable do
transitions :from => :disabled, :to => :pending
end
event :reset do
transitions :from => [:confirming, :confirmed, :disabled, :testing, :tested, :batching, :batched, :pending], :to => :pending
event :reset, :after => :did_reset do
transitions :from => [:disabled, :testing, :tested, :delivering, :delivered, :pending], :to => :pending
end
event :do_test_run, :before => :running_tests do
transitions :from => [:pending, :tested, :batched], :to => :testing
event :do_test_run, :before => :will_run_tests do
transitions :from => [:pending, :tested, :delivered], :to => :testing
end
event :did_test_run, :after => :ran_tests do
transitions :from => :testing, :to => :tested
end
event :do_batch_run, :before => :running_batch do
transitions :from => [:tested, :pending, :batched], :to => :batching
event :do_batch_run, :before => :will_run_batch do
transitions :from => [:tested, :pending, :delivered], :to => :delivering
end
event :did_batch_run, :after => :ran_batch do
transitions :from => :batching, :to => :batched
transitions :from => :delivering, :to => :delivered
end
event :disable do
transitions :from => [:pending, :tested, :batched], :to => :disabled
end
event :do_confirmation do
transitions :from => [:batched, :tested], :to => :confirming
end
event :did_confirmation do
transitions :from => [:confirming], :to => :confirmed
transitions :from => [:pending, :tested, :delivered], :to => :disabled
end
end
def self.new(*args)
oo = super
oo.body = BODY_TEMPLATE
oo.test_emails = "test@jamkazam.com, test@example.com"
oo
end
@ -115,9 +103,9 @@ FOO
def send_test_batch
self.perform_event('do_test_run!')
if 'test'==Rails.env
BatchMailer.send_batch_email_test(batch.id).deliver!
BatchMailer.send_batch_email_test(self.id).deliver!
else
BatchMailer.send_batch_email_test(batch.id).deliver
BatchMailer.send_batch_email_test(self.id).deliver
end
end
@ -128,8 +116,8 @@ FOO
def did_send(emails)
self.update_with_conflict_validation({ :sent_count => self.sent_count + emails.size })
if self.sent_count >= self.candidate_count
if batching?
if self.sent_count >= self.opt_in_count
if delivering?
self.perform_event('did_batch_run!')
elsif testing?
self.perform_event('did_test_run!')
@ -150,42 +138,37 @@ FOO
def update_with_conflict_validation(*args)
num_try = 0
update_attributes(*args)
rescue ActiveRecord::StaleObjectError
num_try += 1
if 5 > num_try
self.reload
retry
begin
update_attributes(*args)
rescue ActiveRecord::StaleObjectError
num_try += 1
if 5 > num_try
self.reload
sleep(0.25)
retry
end
end
end
def running_batch
self.update_with_conflict_validation({:candidate_count => User.email_opt_in.count,
def will_run_batch
self.update_with_conflict_validation({:opt_in_count => User.email_opt_in.count,
:sent_count => 0,
:started_at => Time.now
})
end
def running_tests
self.update_with_conflict_validation({:candidate_count => self.test_count,
:sent_count => 0,
:started_at => Time.now
def will_run_tests
self.update_with_conflict_validation({:opt_in_count => self.test_count,
:sent_count => 0
})
end
def ran_tests
self.update_with_conflict_validation({ :completed_at => Time.now })
perform_confirmation
end
def ran_batch
self.update_with_conflict_validation({ :completed_at => Time.now })
perform_confirmation
end
def perform_confirmation
self.perform_event('do_confirmation!')
EmailError.confirm_errors(self)
end
def clone
@ -199,7 +182,17 @@ FOO
end
def opting_in_count
0 < candidate_count ? candidate_count : User.email_opt_in.count
0 < opt_in_count ? opt_in_count : User.email_opt_in.count
end
def did_reset
self.email_batch_sets.map(&:destroy)
self.update_with_conflict_validation({
:opt_in_count => 0,
:sent_count => 0,
:started_at => nil,
:completed_at => nil,
})
end
end

2
ruby/lib/jam_ruby/models/email_batch_set.rb
View File

@ -4,8 +4,6 @@ module JamRuby
belongs_to :email_batch, :class_name => 'JamRuby::EmailBatch'
BATCH_SIZE = 1000
def self.deliver_set(batch_id, user_ids)
bset = self.new
bset.email_batch_id = batch_id

54
ruby/lib/jam_ruby/models/email_error.rb
View File

@ -2,7 +2,6 @@ module JamRuby
class EmailError < ActiveRecord::Base
self.table_name = "email_errors"
belongs_to :email_batch, :class_name => 'JamRuby::EmailBatch'
belongs_to :user, :class_name => 'JamRuby::User'
default_scope :order => 'email_date DESC'
@ -14,70 +13,67 @@ module JamRuby
SENDGRID_PASSWD = 'jamjamblueberryjam'
def self.sendgrid_url(resource, action='get', params='')
"https://api.sendgrid.com/api/#{resource}.#{action}.json?api_user=#{EmailError::SENDGRID_UNAME}&api_key=#{EmailError::SENDGRID_PASSWD}&date=1&#{params}"
start_date, end_date = self.date_range
"https://api.sendgrid.com/api/#{resource}.#{action}.json?api_user=#{EmailError::SENDGRID_UNAME}&api_key=#{EmailError::SENDGRID_PASSWD}&date=1&start_date=#{start_date.strftime('%Y-%m-%d')}&end_date=#{end_date.strftime('%Y-%m-%d')}&#{params}"
end
def self.bounce_url(batch)
uu = sendgrid_url('bounces')
uu += "&start_date=#{batch.started_at.strftime('%Y-%m-%d')}&end_date=#{batch.completed_at.strftime('%Y-%m-%d')}" if batch.batched?
uu
def self.date_range
tt = Time.now
if eerr = self.first
return [eerr.email_date, tt]
end
[tt - 1.year, tt]
end
def self.bounce_errors(batch)
uu = self.bounce_url(batch)
def self.did_capture?(email_addy)
self.where(:email_address => email_addy).limit(1).first.present?
end
def self.bounce_errors
uu = self.sendgrid_url('bounces')
response = RestClient.get(uu)
if 200 == response.code
return JSON.parse(response.body).collect do |jj|
next if self.did_capture?(jj['email'])
ee = EmailError.new
ee.error_type = 'bounces'
ee.email_batch_id = batch.id
ee.email_address = jj['email']
ee.user_id = User.where(:email => ee.email_address).pluck(:id).first
ee.status = jj['status']
ee.email_date = jj['created']
ee.reason = jj['reason']
ee.save!
RestClient.delete(self.sendgrid_url('bounces', 'delete', "email=#{ee.email_address}"))
# RestClient.delete(self.sendgrid_url('bounces', 'delete', "email=#{ee.email_address}"))
ee
end
end
end
def self.invalid_url(batch)
uu = sendgrid_url('invalidemails')
uu += "&start_date=#{batch.started_at.strftime('%Y-%m-%d')}&end_date=#{batch.completed_at.strftime('%Y-%m-%d')}" if batch.batched?
uu
end
def self.invalid_errors(batch)
uu = self.invalid_url(batch)
def self.invalid_errors
uu = self.sendgrid_url('invalidemails')
response = RestClient.get(uu)
if 200 == response.code
return JSON.parse(response.body).collect do |jj|
next if self.did_capture?(jj['email'])
ee = EmailError.new
ee.error_type = 'invalidemails'
ee.email_batch_id = batch.id
ee.email_address = jj['email']
ee.user_id = User.where(:email => ee.email_address).pluck(:id).first
ee.email_date = jj['created']
ee.reason = jj['reason']
ee.save!
uu =
RestClient.delete(self.sendgrid_url('invalidemails', 'delete', "email=#{ee.email_address}"))
# RestClient.delete(self.sendgrid_url('invalidemails', 'delete', "email=#{ee.email_address}"))
ee
end
end
end
def self.collect_errors(batch)
if batch.batched?
EmailError.bounce_errors(batch)
EmailError.invalid_errors(batch)
end
end
def batch_subject
self.email_batch.try(:subject)
def self.capture_errors
EmailError.bounce_errors
EmailError.invalid_errors
end
end

15
ruby/lib/jam_ruby/resque/scheduled/email_error_collector.rb Normal file
View File

@ -0,0 +1,15 @@
module JamRuby
class EmailErrorCollector
extend Resque::Plugins::LonelyJob
@queue = :email_error_collector
@@log = Logging.logger[EmailErrorCollector]
def self.perform
@@log.debug("waking up")
EmailError.capture_errors
@@log.debug("done")
end
end
end

5
web/config/scheduler.yml
View File

@ -19,3 +19,8 @@ CleanupFacebookSignup:
class: "JamRuby::CleanupFacebookSignup"
description: "Deletes facebook_signups that are old"
EmailErrorCollector:
cron: "0 14 * * *"
class: "JamRuby::EmailErrorCollector"
description: "Collects sendgrid email errors"