jam-cloud/ruby/lib/jam_ruby/resque/audiomixer.rb

require 'json'
require 'resque'
require 'resque-retry'
require 'net/http'
require 'digest/md5'

module JamRuby

  # executes a mix of tracks, creating a final output mix
  class AudioMixer
    extend JamRuby::ResqueStats

    @queue = :audiomixer

    @@log = Logging.logger[AudioMixer]

    attr_accessor :mix_id, :manifest, :manifest_file, :output_filename, :error_out_filename,
                  :postback_ogg_url, :postback_mp3_url,
                  :error_reason, :error_detail

    def self.perform(mix_id, postback_ogg_url, postback_mp3_url)

      JamWebEventMachine.run_wait_stop do
        audiomixer = AudioMixer.new()
        audiomixer.postback_ogg_url = postback_ogg_url
        audiomixer.postback_mp3_url = postback_mp3_url
        audiomixer.mix_id = mix_id
        audiomixer.run
      end

    end

    def self.queue_jobs_needing_retry
      Mix.find_each(:conditions => "completed = FALSE AND (should_retry = TRUE OR (started_at IS NOT NULL AND NOW() - started_at > '1 hour'::INTERVAL))", :batch_size => 100) do |mix|
        mix.enqueue
      end
    end

    def initialize
      #@s3_manager = S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
    end

    def validate
      raise "no manifest specified" unless @manifest

      raise "no files specified" if !@manifest[:files] || @manifest[:files].length == 0

      @manifest[:files].each do |file|
        codec = file[:codec]
        raise "no codec specified" unless codec

        offset = file[:offset]
        raise "no offset specified" unless offset

        filename = file[:filename]
        raise "no filename specified" unless filename
      end

      raise "no output specified" unless @manifest[:output]
      raise "no output codec specified" unless @manifest[:output][:codec]
      raise "no timeline specified" unless @manifest[:timeline]
      raise "no recording_id specified" unless @manifest[:recording_id]
      raise "no mix_id specified" unless @manifest[:mix_id]

    end


    def fetch_audio_files
      @manifest[:files].each do |file|
        filename = file[:filename]
        if filename.start_with? "http"
          # fetch it from wherever, put it somewhere on disk, and replace filename in the file parameter with the local disk one
          download_filename = Dir::Tmpname.make_tmpname(["#{Dir.tmpdir}/audiomixer-file", '.ogg'], nil)

          uri = URI(filename)
          open download_filename, 'wb' do |io|
            begin
            Net::HTTP.start(uri.host, uri.port) do |http|
              request = Net::HTTP::Get.new uri
              http.request request do |response|
                response_code = response.code.to_i
                unless response_code >= 200 && response_code <= 299
                  raise "bad status code: #{response_code}. body: #{response.body}"
                end
                response.read_body do |chunk|
                  io.write chunk
                end
              end
            end
            rescue Exception => e
              @error_reason = "unable to download"
              @error_detail = "url #{filename}, error=#{e}"
              raise e
            end
          end

          @@log.debug("downloaded #{download_filename}")
          filename = download_filename
          file[:filename] = download_filename
        end

        raise "no file located at: #{filename}" unless File.exist? filename
      end
    end

    def prepare
      # make sure there is a place to write the .ogg mix
      prepare_output

      # make sure there is a place to write the error_out json file (if audiomixer fails  this is needed)
      prepare_error_out

      prepare_manifest
    end

    # write the manifest object to file, to pass into audiomixer
    def prepare_manifest

      @manifest_file = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-manifest-#{@manifest[:recording_id]}", '.json'], nil)
      File.open(@manifest_file,"w") do |f|
        f.write(@manifest.to_json)
      end

      @@log.debug("manifest: #{@manifest}")
    end

    # make a suitable location to store the output mix, and pass the chosen filepath into the manifest
    def prepare_output
      @output_ogg_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-output-#{@manifest[:recording_id]}", '.ogg'], nil)
      @output_mp3_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-output-#{@manifest[:recording_id]}", '.mp3'], nil)

      # update manifest so that audiomixer writes here
      @manifest[:output][:filename] = @output_ogg_filename
      # this is not used by audiomixer today; since today the Ruby code handles this
      @manifest[:output][:filename_mp3] = @output_mp3_filename

      @@log.debug("output ogg file: #{@output_ogg_filename}, output mp3 file: #{@output_mp3_filename}")
    end

    # make a suitable location to store an output error file, which will be populated on failure to help diagnose problems.
    def prepare_error_out
      @error_out_filename = Dir::Tmpname.make_tmpname( ["#{Dir.tmpdir}/audiomixer-error-out-#{@manifest[:recording_id]}", '.ogg'], nil)

      # update manifest so that audiomixer writes here
      @manifest[:error_out] = @error_out_filename

      @@log.debug("error_out: #{@error_out_filename}")
    end

    # read in and parse the error file that audiomixer pops out
    def parse_error_out
      error_out_data = File.read(@error_out_filename)
      begin
        @error_out = JSON.parse(error_out_data)
      rescue
        @error_reason = "unable-parse-error-out"
        @@log.error("unable to parse error_out_data: #{error_out_data} from error_out: #{@error_out_filename}")
      end

      @error_reason = @error_out[:reason]
      @error_reason = "unspecified-reason" unless @error_reason
      @error_detail = @error_out[:detail]
    end

    def postback

      @@log.debug("posting ogg mix to #{@postback_ogg_url}")

      uri = URI.parse(@postback_ogg_url)
      http = Net::HTTP.new(uri.host, uri.port)
      request = Net::HTTP::Put.new(uri.request_uri)

      response = nil
      File.open(@output_ogg_filename,"r") do |f|
        request.body_stream=f
        request["Content-Type"] = "audio/ogg"
        request.add_field('Content-Length', File.size(@output_ogg_filename))
        response = http.request(request)
      end

      response_code = response.code.to_i
      unless response_code >= 200 && response_code <= 299
        @error_reason = "postback-ogg-mix-to-s3"
        raise "unable to put to url: #{@postback_ogg_url}, status: #{response.code}, body: #{response.body}"
      end

      
      @@log.debug("posting mp3 mix to #{@postback_mp3_url}")

      uri = URI.parse(@postback_mp3_url)
      http = Net::HTTP.new(uri.host, uri.port)
      request = Net::HTTP::Put.new(uri.request_uri)

      response = nil
      File.open(@output_mp3_filename,"r") do |f|
        request.body_stream=f
        request["Content-Type"] = "audio/mpeg"
        request.add_field('Content-Length', File.size(@output_mp3_filename))
        response = http.request(request)
      end

      response_code = response.code.to_i
      unless response_code >= 200 && response_code <= 299
        @error_reason = "postback-mp3-mix-to-s3"
        raise "unable to put to url: #{@postback_mp3_url}, status: #{response.code}, body: #{response.body}"
      end

    end

    def cleanup_files()
      File.delete(@output_ogg_filename) if File.exists?(@output_ogg_filename)
      File.delete(@output_mp3_filename) if File.exists?(@output_mp3_filename)
      File.delete(@manifest_file) if File.exists?(@manifest_file)
      File.delete(@error_out_filename) if File.exists?(@error_out_filename)

      @manifest[:files].each do |file|
        filename = file[:filename]
        File.delete(filename) if File.exists?(filename)
      end
    end

    def post_success(mix)

      ogg_length = File.size(@output_ogg_filename)
      ogg_md5 = Digest::MD5.new
      File.open(@output_ogg_filename, 'rb').each {|line| ogg_md5.update(line)}

      mp3_length = File.size(@output_mp3_filename)
      mp3_md5 = Digest::MD5.new
      File.open(@output_mp3_filename, 'rb').each {|line| mp3_md5.update(line)}


      mix.finish(ogg_length, ogg_md5.to_s, mp3_length, mp3_md5.to_s)
    end


    def post_error(mix, e)
      begin

        # if error_reason is null, assume this is an unhandled error
        unless @error_reason
          @error_reason = "unhandled-job-exception"
          @error_detail = e.to_s
        end
        mix.errored(error_reason, error_detail)

      rescue Exception => e
        @@log.error "unable to post back to the database the error #{e}"
      end
    end

    def run
      @@log.info("audiomixer job starting. mix_id #{mix_id}")

      mix = Mix.find(mix_id)

      begin
        # bailout check
        if mix.completed
          @@log.debug("mix is already completed. bailing")
          return
        end

        @manifest = symbolize_keys(mix.manifest)
        @manifest[:mix_id] = mix_id # slip in the mix_id so that the job can add it to the ogg comments

        # sanity check the manifest
        validate

        # if http files are specified, bring them local
        fetch_audio_files

        # write the manifest to file, so that it can be passed to audiomixer as an filepath argument
        prepare

        execute(@manifest_file)

        postback

        post_success(mix)

        # only cleanup files if we manage to get this far
        cleanup_files

        @@log.info("audiomixer job successful. mix_id #{mix_id}")
      
      rescue Exception => e
        post_error(mix, e)
        raise
      end

    end

    def manifest=(value)
      @manifest = symbolize_keys(value)
    end

    private

    def execute(manifest_file)

      unless File.exist? APP_CONFIG.audiomixer_path
        @@log.error("unable to find audiomixer")
        error_msg = "audiomixer job failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}"
        @@log.info(error_msg)
        @error_reason = "unable-find-appmixer"
        @error_detail = APP_CONFIG.audiomixer_path
        raise error_msg
      end

      audiomixer_cmd = "#{APP_CONFIG.audiomixer_path} #{manifest_file}"

      @@log.debug("executing #{audiomixer_cmd}")

      system(audiomixer_cmd)

      unless $? == 0
        parse_error_out
        error_msg = "audiomixer job failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}"
        @@log.info(error_msg)
        raise error_msg
      end

      raise "no output ogg file after mix" unless File.exist? @output_ogg_filename
      
      ffmpeg_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{@output_ogg_filename}\" -ab 192k -metadata JamRecordingId=#{@manifest[:recording_id]} -metadata JamMixId=#{@mix_id} -metadata JamType=Mix \"#{@output_mp3_filename}\""

      system(ffmpeg_cmd)

      unless $? == 0
        @error_reason = 'ffmpeg-failed'
        @error_detail = $?.to_s
        error_msg = "ffmpeg failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}"
        @@log.info(error_msg)
        raise error_msg
      end

      raise "no output mp3 file after conversion" unless File.exist? @output_mp3_filename

      # time to normalize both mp3 and ogg files

      normalize_ogg_cmd = "#{APP_CONFIG.normalize_ogg_path} --bitrate 128 \"#{@output_ogg_filename}\""
      system(normalize_ogg_cmd)
      unless $? == 0
        @error_reason = 'normalize-ogg-failed'
        @error_detail = $?.to_s
        error_msg = "normalize-ogg failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}"
        @@log.info(error_msg)
        raise error_msg
      end
      raise "no output ogg file after normalization" unless File.exist? @output_ogg_filename

      normalize_mp3_cmd = "#{APP_CONFIG.normalize_mp3_path} --bitrate 128 \"#{@output_mp3_filename}\""
      system(normalize_mp3_cmd)
      unless $? == 0
        @error_reason = 'normalize-mp3-failed'
        @error_detail = $?.to_s
        error_msg = "normalize-mp3 failed status=#{$?} error_reason=#{@error_reason} error_detail=#{@error_detail}"
        @@log.info(error_msg)
        raise error_msg
      end
      raise "no output mp3 file after conversion" unless File.exist? @output_mp3_filename
    end

    def symbolize_keys(obj)
      case obj
        when Array
          obj.inject([]){|res, val|
            res << case val
                     when Hash, Array
                       symbolize_keys(val)
                     else
                       val
                   end
            res
          }
        when Hash
          obj.inject({}){|res, (key, val)|
            nkey = case key
                     when String
                       key.to_sym
                     else
                       key
                   end
            nval = case val
                     when Hash, Array
                       symbolize_keys(val)
                     else
                       val
                   end
            res[nkey] = nval
            res
          }
        else
          obj
      end
    end
  end

end