reconnect automatically in resque; kill self for websocket gateway

ruby/lib/jam_ruby.rb

@@ -53,6 +53,7 @@ require "jam_ruby/lib/em_helper"
 require "jam_ruby/lib/nav"
 require "jam_ruby/lib/html_sanitize"
 require "jam_ruby/lib/guitar_center"
+require "jam_ruby/resque/resque_jam_error"
 require "jam_ruby/resque/resque_hooks"
 require "jam_ruby/resque/audiomixer"
 require "jam_ruby/resque/quick_mixer"
@@ -104,6 +105,7 @@ require "jam_ruby/app/uploaders/jam_track_uploader"
 require "jam_ruby/app/uploaders/jam_track_track_uploader"
 require "jam_ruby/app/uploaders/jam_track_right_uploader"
 require "jam_ruby/app/uploaders/max_mind_release_uploader"
+require "jam_ruby/db_util"
 require "jam_ruby/lib/desk_multipass"
 require "jam_ruby/lib/ip"
 require "jam_ruby/lib/subscription_message"

ruby/lib/jam_ruby/db_util.rb

@@ -0,0 +1,18 @@
+module JamRuby
+  class DbUtil
+
+    def self.create(connection_hash)
+      
+    end
+
+    BAD_CONN_EXCEPTIONS = [PG::UnableToSend, PG::ConnectionBad]
+
+    def self.bad_conn_exception?(exception)
+      test_exception = exception
+      if exception.class == ActiveRecord::StatementInvalid
+        test_exception = exception.original_exception
+      end
+      BAD_CONN_EXCEPTIONS.include?(test_exception.class)
+    end
+  end
+end

ruby/lib/jam_ruby/dbutil.rb

@@ -1,7 +0,0 @@
-module JamRuby
-  class DbUtil
-    def self.create(connection_hash)
-      
-    end
-  end
-end

ruby/lib/jam_ruby/resque/resque_jam_error.rb

@@ -0,0 +1,10 @@
+require 'resque'
+class ResqueJamError < Resque::Failure::Base
+
+  def save
+    if DbUtil.bad_conn_exception?(self.exception)
+      puts "RECONNECTING TO DATABASE"
+      ActiveRecord::Base.connection.reconnect!
+    end
+  end
+end

ruby/lib/jam_ruby/resque/stress_job.rb

@@ -13,13 +13,15 @@ module JamRuby
 
     def self.perform
 
+      puts "STARTING BIA"
       @@log.debug("STARTING")
-      100.times do
+      1.times do
         user = User.first.id
         diagnostic = Diagnostic.first.user_id
         count = Diagnostic.all.count
       end
       @@log.debug("ENDING")
+      puts "ENDING BIA"
 
     end
   end

web/config/initializers/resque.rb

@@ -1,5 +1,16 @@
 Resque.redis = Rails.application.config.redis_host
 
+require 'resque_failed_job_mailer'
+
+puts "RESQUE INITIALIZER"
+Resque::Failure::Notifier.configure do |config|
+  config.to = Rails.application.config.email_alerts_alias
+  config.from = Rails.application.config.email_generic_from
+end
+
+#Resque::Failure::Multiple.classes = [ResqueJamError]
+Resque::Failure::Multiple.classes = [Resque::Failure::Notifier, ResqueJamError]
+Resque::Failure.backend = Resque::Failure::Multiple
 
 if File.split($0).last != 'rake' && Rails.env == 'development' && (ENV['RUN_JOBS_INLINE'] == '1' || ENV['RUN_INLINE_JOBS'] == '1')
 

web/config/initializers/resque_failed_job_mailer.rb

@@ -1,6 +1 @@
-require 'resque_failed_job_mailer'
-
-Resque::Failure::Notifier.configure do |config|
-  config.to = Rails.application.config.email_alerts_alias
-  config.from = Rails.application.config.email_generic_from
-end
+puts "DEPRECATED INITILAIZER"

websocket-gateway/lib/jam_websockets/router.rb

@@ -412,11 +412,12 @@ module JamWebsockets
         @log.error "ending client session due to server programming or runtime error. reason=#{e.to_s}"
         @log.error e
 
-        if PG::UnableToSend
+        if DbUtil.bad_conn_exception?(e)
           # indicates connection to server is down; kill self.  Will be restarted; if db is up we will be healthy
-          @log.error "EXITING DUE TO DEAD DBCONN"
+          @log.error "EXITING DUE TO DEAD DBCONN: #{e}"
           Kernel.exit!(1)
         end
+
         begin
           # wrap the message up and send it down
           error_msg = @message_factory.server_generic_error(e.to_s)