jam-cloud/ruby/lib/jam_ruby/jam_track_importer.rb

require 'json'
require 'tempfile'
require 'open3'
require 'fileutils'
require 'open-uri'
require 'yaml'

module JamRuby

  class JamTrackImporter

    @@log = Logging.logger[JamTrackImporter]

    attr_accessor :name
    attr_accessor :reason
    attr_accessor :detail

    def jamkazam_s3_manager
      @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
    end

    def finish(reason, detail)
      self.reason = reason
      self.detail = detail
    end

    def dry_run(metadata, metalocation)
      metadata ||= {}

      parsed_metalocation = parse_metalocation(metalocation)

      return unless parsed_metalocation

      original_artist = parsed_metalocation[1]
      name = parsed_metalocation[2]

      success = dry_run_metadata(metadata, original_artist, name)

      return unless success

      dry_run_audio(metadata, "audio/#{original_artist}/#{name}")

      finish("success", nil)
    end

    def parse_metalocation(metalocation)

      bits = metalocation.split('/')

      if bits.length != 4
        finish("invalid_metalocation", "metalocation not valid #{metalocation}")
        return nil
      end

      if bits[0] != "audio"
        finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}")
        return nil
      end

      if bits[3] != 'meta.yml'
        finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}")
        return nil
      end

      bits
    end

    # if you change this, it will (at least without some work )break development usage of jamtracks
    def gen_plan_code(original_artist, name)
      # remove all non-alphanumeric chars from artist as well as name
      artist_code = original_artist.gsub(/[^0-9a-z]/i, '').downcase
      name_code = name.gsub(/[^0-9a-z]/i, '').downcase
      "jamtrack-#{artist_code[0...20]}-#{name_code}"[0...50] # make sure it's a max of 50 long
    end

    def dry_run_metadata(metadata, original_artist, name)

      self.name = metadata["name"] || name

      original_artist = metadata["original_artist"] || original_artist
      plan_code = metadata["plan_code"] || gen_plan_code(original_artist, self.name)
      description = metadata["description"]

      @@log.debug("#{self.name} original_artist=#{original_artist}")
      @@log.debug("#{self.name} plan_code=#{plan_code}")

      true
    end

    def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name)

      metadata ||= {}
      self.name = metadata["name"] || name

      if jam_track.new_record?
        jam_track.status = 'Staging'
        jam_track.metalocation = metalocation
        jam_track.original_artist = metadata["original_artist"] || original_artist
        jam_track.name = self.name
        jam_track.genre_id = 'rock'
        jam_track.plan_code = metadata["plan_code"] || gen_plan_code(jam_track.original_artist, jam_track.name)
        jam_track.price = 1.99
        jam_track.reproduction_royalty_amount = 0
        jam_track.licensor_royalty_amount = 0
        jam_track.sales_region = 'United States'
        jam_track.recording_type = 'Cover'
        jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"."
      else
        #@@log.debug("#{self.name} skipped because it already exists in database")
        finish("jam_track_exists", "")
        return false
      end

      saved = jam_track.save

      if !saved
        finish("invalid_definition", jam_track.errors.inspect)
      end

      saved
    end

    # oddballs - Guitar Solo.wav
    # Rocket Man Stem - Vocal Back Up
    # Rocket Man Stem - Vocal Lead Double
    # Rock and Roll Stem - Electric Guitar - Main - Solo
    def determine_instrument(potential_instrument_original, potential_part_original = nil)
      potential_instrument = potential_instrument_original.downcase
      potential_part = potential_part_original.downcase if potential_part_original

      instrument = nil
      used_helper = false
      part = nil

      if potential_instrument == 'guitar'
        if potential_part
          if potential_part == 'acoustic'
            instrument = 'acoustic guitar'
            used_helper = true
          elsif potential_part == 'electric'
            instrument = 'electric guitar'
            used_helper = true
          elsif potential_part == 'acoustic solo'
            instrument = 'acoustic guitar'
            used_helper = true
            part = 'Solo'
          elsif potential_part.include?('acoustic')
            used_helper = true # ambiguous
          else
            instrument = 'electric guitar'
            used_helper = false
          end
        else
          instrument = 'electric guitar'
        end
      elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat'
        instrument = 'electric guitar'
      elsif potential_instrument == 'keys'
        instrument = 'keyboard'
      elsif potential_instrument == 'vocal' || potential_instrument == 'vocals'
        instrument = 'voice'
      elsif potential_instrument == 'bass'
        instrument = 'bass guitar'
      elsif potential_instrument == 'drum'
        instrument = 'drums'
      elsif potential_instrument == 'sound effects' || potential_instrument == 'sound efx' || potential_instrument == 'effects'
        instrument = 'computer'

        if potential_part_original
          part = "Sound FX (#{potential_part_original})"
        else
          part = 'Sound FX'
        end


      elsif potential_instrument == "sax"
        instrument = 'saxophone'
      elsif potential_instrument == "vocal back up"
        instrument = "voice"
        part = "Back Up"
      elsif potential_instrument == "vocal lead double"
        instrument = "voice"
        part = "Lead Double"
      elsif potential_instrument == "guitar solo"
        instrument = "electric guitar"
        part = "Solo"
      elsif potential_instrument == 'stadium crowd'
        instrument = 'computer'
        part = 'Crowd Noise'
      elsif potential_instrument == 'cannons'
        instrument = 'computer'
        part = 'Cannons'
      elsif potential_instrument == 'bells'
        instrument = 'computer'
        part = 'Bells'
      elsif potential_instrument == 'percussion'
        instrument = 'drums'
        part = 'Percussion'
      elsif potential_instrument == 'fretless bass'
        instrument = 'bass guitar'
        part = 'Fretless'
      elsif potential_instrument == 'clock percussion'
        instrument = 'computer'
        part = 'Clock'
      elsif potential_instrument == 'horns'
        instrument = 'other'
        part = 'Horns'
      elsif potential_instrument == 'strings'
        instrument = 'other'
        part = 'Strings'
      elsif potential_instrument == 'orchestration'
        instrument = 'computer'
        part = 'Orchestration'
      elsif potential_instrument == 'claps' || potential_instrument == 'hand claps'
        instrument = 'computer'
        part = 'Claps'
      else
        found_instrument = Instrument.find_by_id(potential_instrument)
        if found_instrument
          instrument = found_instrument.id
        end
      end

      if !used_helper && !part
        part = potential_part_original
      end

      part = potential_instrument_original if !part

      {instrument: instrument,
      part: part}

    end

    def parse_wav(file)

      bits = file.split('/')
      filename = bits[bits.length - 1] # remove all but just the filename
      filename_no_ext = filename[0..-5]
      comparable_filename = filename_no_ext.downcase # remove .wav

      master = false
      instrument = nil
      part = nil

      if comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix")
        master = true
      else
        stem_location = comparable_filename.index('stem -')
        unless stem_location
          stem_location = comparable_filename.index('stems -')
        end
        unless stem_location
          stem_location = comparable_filename.index('stem-')
        end
        unless stem_location
          stem_location = comparable_filename.index('stems-')
        end

        if stem_location
          bits = filename_no_ext[stem_location..-1].split('-')
          bits.collect! {|bit| bit.strip}

          possible_instrument = nil
          possible_part = nil


          if bits.length == 2
            # second bit is instrument
            possible_instrument = bits[1]
          elsif bits.length == 3
            # second bit is instrument, third bit is part
            possible_instrument = bits[1]
            possible_part = bits[2]
          elsif bits.length == 4
            possible_instrument = bits[1]
            possible_part = "#{bits[2]} #{bits[3]}"
          end

          result = determine_instrument(possible_instrument, possible_part)
          instrument = result[:instrument]
          part = result[:part]
        end
      end


      {filename: filename, master: master, instrument: instrument, part: part}
    end

    def dry_run_audio(metadata, s3_path)
      all_files = fetch_wav_files(s3_path)

      all_files.each do |file|
        if file.end_with?('.wav')
          parsed_wav = parse_wav(file)
          if parsed_wav[:master]
            @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
          else
            if !parsed_wav[:instrument] || !parsed_wav[:part]
              @@log.warn("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
            else
              @@log.debug("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
            end
          end
        else
          @@log.debug("#{self.name} ignoring non-wav file #{file}")
        end
      end

    end

    def sort_tracks(tracks)

      def set_custom_weight(track)
        weight = 5
        case track.instrument_id
          when 'electric guitar'
            weight = 1
          when 'acoustic guitar'
            weight = 2
          when 'drums'
            weight = 3
          when 'keys'
            weight = 4
          when 'computer'
            weight = 10
          else
            weight = 5
        end
        if track.track_type == 'Master'
          weight = 1000
        end

        weight
      end

      sorted_tracks = tracks.sort do |a, b|
        a_weight = set_custom_weight(a)
        b_weight = set_custom_weight(b)

        a_weight <=> b_weight
      end

      position = 1
      sorted_tracks.each do |track|
        track.position = position
        position = position + 1
      end

      sorted_tracks[sorted_tracks.length - 1].position = 1000

      sorted_tracks
    end

    def synchronize_audio(jam_track, metadata, s3_path, skip_audio_upload)

      wav_files = fetch_wav_files(s3_path)

      tracks = []

      wav_files.each do |wav_file|
        track = JamTrackTrack.new
        track.original_audio_s3_path = wav_file

        parsed_wav = parse_wav(wav_file)

        if parsed_wav[:master]
          track.track_type = 'Master'
          track.part = 'Master'
          @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
        else
          if !parsed_wav[:instrument] || !parsed_wav[:part]
            @@log.warn("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
          else
            @@log.debug("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
          end

          track.instrument_id = parsed_wav[:instrument] || 'other'
          track.track_type = 'Track'
          track.part = parsed_wav[:part] || 'Other'
        end

        tracks << track
      end

      tracks = sort_tracks(tracks)

      jam_track.jam_track_tracks = tracks

      saved = jam_track.save

      if !saved
        finish('invalid_audio', jam_track.errors.inspect)
        return false
      end

      return synchronize_audio_files(jam_track, skip_audio_upload)
    end

    def synchronize_audio_files(jam_track, skip_audio_upload)

      begin
        Dir.mktmpdir do |tmp_dir|

          jam_track.jam_track_tracks.each do |track|

            basename = File.basename(track.original_audio_s3_path)
            s3_dirname = File.dirname(track.original_audio_s3_path)

            # make a 44100 version, and a 48000 version
            ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
            ogg_48000_filename = File.basename(basename, ".wav") + "-48000.ogg"

            ogg_44100_s3_path = track.filename(ogg_44100_filename)
            ogg_48000_s3_path = track.filename(ogg_48000_filename)

            track.skip_uploader = true

            if skip_audio_upload
              track["url_44"] = ogg_44100_s3_path
              track["md5_44"] = 'md5'
              track["length_44"] = 1

              track["url_48"] = ogg_48000_s3_path
              track["md5_48"] = 'md5'
              track["length_48"] = 1
            else
              wav_file = File.join(tmp_dir, basename)

              # bring the original wav file down from S3 to local file system
              JamTrackImporter::s3_manager.download(track.original_audio_s3_path, wav_file)

              sample_rate = `soxi -r "#{wav_file}"`.strip

              ogg_44100 = File.join(tmp_dir, ogg_44100_filename)
              ogg_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.ogg")

              if sample_rate == "44100"
                `oggenc "#{wav_file}" -q 6 -o "#{ogg_44100}"`
              else
                `oggenc "#{wav_file}" --resample 44100 -q 6 -o "#{ogg_44100}"`
              end

              if sample_rate == "48000"
                `oggenc "#{wav_file}" -q 6 -o "#{ogg_48000}"`
              else
                `oggenc "#{wav_file}" --resample 48000 -q 6 -o "#{ogg_48000}"`
              end

              # upload the new ogg files to s3
              @@log.debug("uploading 44100 to #{ogg_44100_s3_path}")

              jamkazam_s3_manager.upload(ogg_44100_s3_path, ogg_44100)

              @@log.debug("uploading 48000 to #{ogg_48000_s3_path}")

              jamkazam_s3_manager.upload(ogg_48000_s3_path, ogg_48000)

              # and finally update the JamTrackTrack with the new info
              track["url_44"] = ogg_44100_s3_path
              track["md5_44"] = ::Digest::MD5.file(ogg_44100).hexdigest
              track["length_44"] = File.new(ogg_44100).size

              track["url_48"] = ogg_48000_s3_path
              track["md5_48"] = ::Digest::MD5.file(ogg_48000).hexdigest
              track["length_48"] = File.new(ogg_48000).size

            end

            track.save!
          end
        end
      rescue Exception => e
        finish("sync_audio_exception", e.to_s)
        return false
      end

      return true
    end

    def fetch_all_files(s3_path)
      JamTrackImporter::s3_manager.list_files(s3_path)
    end

    def fetch_wav_files(s3_path)
      files = fetch_all_files(s3_path)
      files.select { |file| file.end_with?('.wav') }
    end

    def synchronize(jam_track, metadata, metalocation, options)

      # metalocation should be audio/original artist/song name/meta.yml

      metadata ||= {}

      parsed_metalocation = parse_metalocation(metalocation)

      return unless parsed_metalocation

      original_artist = parsed_metalocation[1]
      name = parsed_metalocation[2]

      success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name)

      return unless success

      synchronized_audio = synchronize_audio(jam_track, metadata, "audio/#{original_artist}/#{name}", options[:skip_audio_upload])

      return unless synchronized_audio

      created_plan = synchronize_recurly(jam_track)
      if created_plan
        finish("success", nil)
      end

    end

    def synchronize_recurly(jam_track)
      begin
        recurly = RecurlyClient.new
        recurly.create_jam_track_plan(jam_track) unless recurly.find_jam_track_plan(jam_track)
      rescue RecurlyClientError => x
        finish('recurly_create_plan', x.errors.to_s)
        return false
      end

      true
    end

    class << self

      def s3_manager
        @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end


      def dry_run
        s3_manager.list_directories('audio').each do |original_artist|
          @@log.debug("searching through artist directory '#{original_artist}'")

          songs = s3_manager.list_directories(original_artist)
          songs.each do |song|
            @@log.debug("searching through song directory' #{song}'")

            metalocation = "#{song}meta.yml"

            metadata = load_metalocation(metalocation)

            jam_track_importer = JamTrackImporter.new

            jam_track_importer.dry_run(metadata, metalocation)
          end
        end

      end


      def synchronize_all(options)
        importers = []

        s3_manager.list_directories('audio').each do |original_artist|
          @@log.debug("searching through artist directory '#{original_artist}'")

          songs = s3_manager.list_directories(original_artist)
          songs.each do |song|
            @@log.debug("searching through song directory' #{song}'")

            metalocation = "#{song}meta.yml"

            importer = synchronize_from_meta(metalocation, options)
            importers << importer
          end
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def jam_track_dry_run(metalocation)
        # see if we can find a JamTrack with this metalocation
        jam_track = JamTrack.find_by_metalocation(metalocation)

        meta = load_metalocation(metalocation)

        if jam_track
          @@log.debug("jamtrack #{jam_track.name} located by metalocation")
          jam_track.dry_run(meta, metalocation)
        else
          jam_track = JamTrack.new
          jam_track.dry_run(meta, metalocation)
        end
      end

      def load_metalocation(metalocation)
        begin
        data = s3_manager.read_all(metalocation)
          return YAML.load(data)
        rescue AWS::S3::Errors::NoSuchKey
          return nil
        end
      end

      def create_from_metalocation(meta, metalocation, options = {skip_audio_upload:false})
        jam_track = JamTrack.new
        sync_from_metadata(jam_track, meta, metalocation, options)
      end

      def update_from_metalocation(jam_track, meta, metalocation, options)
        sync_from_metadata(jam_track, meta, metalocation, options)
      end

      def sync_from_metadata(jam_track, meta, metalocation, options)
        jam_track_importer = JamTrackImporter.new

        JamTrack.transaction do
          #begin
            jam_track_importer.synchronize(jam_track, meta, metalocation, options)
          #rescue Exception => e
          #  jam_track_importer.finish("unhandled_exception", e.to_s)
          #end

          if jam_track_importer.reason != "success"
            raise ActiveRecord::Rollback
          end
        end

        jam_track_importer
      end

      def synchronize_from_meta(metalocation, options)
        # see if we can find a JamTrack with this metalocation
        jam_track = JamTrack.find_by_metalocation(metalocation)

        meta = load_metalocation(metalocation)

        jam_track_importer = nil
        if jam_track
          @@log.debug("jamtrack #{jam_track.name} located by metalocation")
          jam_track_importer = update_from_metalocation(jam_track, meta, metalocation, options)
        else
          jam_track_importer = create_from_metalocation(meta, metalocation, options)
        end

        if jam_track_importer.reason == "success"
          @@log.info("#{jam_track_importer.name} successfully imported")
        else
          @@log.error("#{jam_track_importer.name} failed to import.")
          @@log.error("#{jam_track_importer.name} reason=#{jam_track_importer.reason}")
          @@log.error("#{jam_track_importer.name} detail=#{jam_track_importer.detail}")
        end

        jam_track_importer
      end
    end
  end
end