jam-cloud/ruby/lib/jam_ruby/jam_track_importer.rb

require 'json'
require 'tempfile'
require 'open3'
require 'fileutils'
require 'open-uri'
require 'yaml'

module JamRuby

  class JamTrackImporter

    @@log = Logging.logger[JamTrackImporter]

    attr_accessor :name
    attr_accessor :reason
    attr_accessor :detail
    attr_accessor :storage_format

    def jamkazam_s3_manager
      @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
    end

    def public_jamkazam_s3_manager
      @public_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_public, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
    end

    def initialize(storage_format = 'default')
      @storage_format = storage_format
    end

    def finish(reason, detail)
      self.reason = reason
      self.detail = detail
    end


    def synchronize_preview_dev(jam_track)
      jam_track.jam_track_tracks.each do |track|
        next if track.track_type != 'Master'


        most_recent_aac = nil
        most_recent_ogg = nil
        most_recent_mp3 = nil
        public_jamkazam_s3_manager.list_files(track.preview_directory).each do |s3_preview_item|

          s3_object = public_jamkazam_s3_manager.object(s3_preview_item)

          if s3_preview_item.end_with?('.aac')
            if most_recent_aac
              if s3_object.last_modified > most_recent_aac.last_modified
                most_recent_aac = s3_object
              end
            else
              most_recent_aac = s3_object
            end
          end

          if s3_preview_item.end_with?('.mp3')
            if most_recent_mp3
              if s3_object.last_modified > most_recent_mp3.last_modified
                most_recent_mp3 = s3_object
              end
            else
              most_recent_mp3 = s3_object
            end
          end

          if s3_preview_item.end_with?('.ogg')
            if most_recent_ogg
              if s3_object.last_modified > most_recent_ogg.last_modified
                most_recent_ogg = s3_object
              end
            else
              most_recent_ogg = s3_object
            end
          end
        end

        if most_recent_aac
          track['preview_aac_md5'] = 'md5'
          track['preview_aac_url'] = most_recent_aac.key
          track['preview_aac_length'] = most_recent_aac.content_length
        end

        if most_recent_mp3
          track['preview_mp3_md5'] = 'md5'
          track['preview_mp3_url'] = most_recent_mp3.key
          track['preview_mp3_length'] = most_recent_mp3.content_length
        end

        if most_recent_ogg
          track['preview_md5'] = 'md5'
          track['preview_url'] = most_recent_ogg.key
          track['preview_length'] = most_recent_ogg.content_length
        end

        track.save
      end
    end
    # this method was created due to Tency-sourced data having no master track
    # it goes through all audio tracks, and creates a master mix from it.  (mix + normalize)
    def create_master(metadata, metalocation)

      parsed_metalocation = parse_metalocation(metalocation)

      if parsed_metalocation.nil?
        finish("invalid_metalocation", metalocation)
        return
      end

      original_artist = parsed_metalocation[1]
      meta_name = parsed_metalocation[2]

      self.name = metadata[:name] || meta_name


      audio_path = metalocation[0...-"/meta.yml".length]

      all_files = fetch_important_files(audio_path)

      audio_files = []
      master_found = false
      all_files.each do |file|

        parsed_wav = parse_file(file)
        if parsed_wav[:master]
          master_found = true
        elsif parsed_wav[:type] == :track

          audio_files << file
        end
      end

      if master_found
        @@log.debug("master exists... skipping #{self.name} ")
        finish('success', nil)
        return
      else

        tracks = []

        #tmp_dir = Dir.mktmpdir
        #tmp_dir = "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150809-9945-1ykr85u"
        Dir.mktmpdir do |tmp_dir|
          @@log.debug("downloading all audio files in #{tmp_dir}")
          audio_files.each do |s3_track|
            track = File.join(tmp_dir, File.basename(s3_track))
            tracks << track
            JamTrackImporter.song_storage_manager.download(s3_track, track)
          end

          # first have to check if all are the same sample rate.   If not, we have to make it so

          first_sample_rate = nil
          normalize_needed = false
          tracks.each do |track|
            sample_rate = `soxi -r "#{track}"`.strip

            if first_sample_rate.nil?
              first_sample_rate = sample_rate
            else
              if first_sample_rate != sample_rate
                # we need to normalize all of them
                normalize_needed = true
                break
              end
            end
          end

          normalized_tracks = []
          if normalize_needed
            tracks.each do |track|
              normalized_track = File.join(tmp_dir, 'normalized-' + File.basename(track))
              output = `sox "#{track}" "#{normalized_track}" rate #{first_sample_rate}`
              @@log.debug("resampling #{normalized_track}; output: #{output}")
              normalized_tracks << normalized_track
            end
            tracks = normalized_tracks
          end


          temp_file = File.join(tmp_dir, "temp.wav")
          output_filename = JamTrackImporter.remove_s3_special_chars("#{self.name} Master Mix.wav")
          output_file = File.join(tmp_dir, output_filename)
          command = "sox -m "
          tracks.each do |track|
            command << " \"#{track}\""
          end
          command << " \"#{temp_file}\""

          @@log.debug("mixing with cmd: " + command)
          sox_output = `#{command}`
          result_code = $?.to_i

          if result_code != 0
            @@log.error("unable to generate master mix")
            finish("sox_master_mix_failure", sox_output)
          else

            # now normalize the audio

            command = "sox --norm \"#{temp_file}\" \"#{output_file}\""
            @@log.debug("normalizing with cmd: " + command)
            sox_output = `#{command}`
            result_code = $?.to_i
            if result_code != 0
              @@log.error("unable to normalize master mix")
              finish("sox_master_mix_failure", sox_output)
            else

              # now we need to upload the output back up
              s3_target = audio_path + '/' + output_filename
              @@log.debug("uploading #{output_file} to #{s3_target}")
              JamTrackImporter.song_storage_manager.upload(s3_target, output_file)
              finish('success', nil)
            end

          end
        end
      end
    end

    def dry_run(metadata, metalocation)
      metadata ||= {}

      parsed_metalocation = parse_metalocation(metalocation)

      return unless parsed_metalocation

      original_artist = parsed_metalocation[1]
      name = parsed_metalocation[2]

      JamTrackImporter.summaries[:unique_artists] << original_artist

      success = dry_run_metadata(metadata, original_artist, name)

      return unless success

      audio_path = metalocation[0...-"/meta.yml".length]


      dry_run_audio(metadata, audio_path)

      finish("success", nil)
    end

    def add_licensor_metadata(vendor, metalocation)
      Dir.mktmpdir do |tmp_dir|
        @@log.debug("update vendor metadata")
        meta_yml = File.join(tmp_dir, 'meta.yml')
        if jamkazam_s3_manager.exists?(metalocation)
          jamkazam_s3_manager.download(metalocation, meta_yml)
          meta = YAML.load_file(meta_yml)
        else
          meta = {}
        end

        meta[:licensor] = vendor

        File.open(meta_yml, 'w') { |f| f.write meta.to_yaml }

        jamkazam_s3_manager.upload(metalocation, meta_yml)
      end
    end

    def is_tency_storage?
      assert_storage_set
      @storage_format == 'Tency'
    end

    def assert_storage_set
      raise "no storage_format set" if @storage_format.nil?
    end


    def parse_metalocation(metalocation)
      # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml
      if is_tency_storage?

        suffix = '/meta.yml'

        unless metalocation.end_with? suffix
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        metalocation = metalocation[0...-suffix.length]

        first_path = metalocation.index('/')
        if first_path.nil?
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end
        metalocation = metalocation[(first_path + 1)..-1]

        bits = ['audio']
        # example: Sister Hazel - All For You - 10385
        first_dash = metalocation.index(' - ')
        if first_dash
          artist = metalocation[0...(first_dash)].strip
          bits << artist
        else
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        last_dash = metalocation.rindex('-')
        if last_dash
          song = metalocation[(first_dash+3)...last_dash].strip
          bits << song
        else
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        bits << 'meta.yml'
        bits
      else
        bits = metalocation.split('/')

        if bits.length != 4
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        if bits[0] != "audio"
          finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}")
          return nil
        end

        if bits[3] != 'meta.yml'
          finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}")
          return nil
        end

        bits
      end
    end

    # if you change this, it will (at least without some work )break development usage of jamtracks
    def gen_plan_code(original_artist, name)
      # remove all non-alphanumeric chars from artist as well as name
      artist_code = original_artist.gsub(/[^0-9a-z]/i, '').downcase
      name_code = name.gsub(/[^0-9a-z]/i, '').downcase
      "jamtrack-#{artist_code[0...20]}-#{name_code}"[0...50] # make sure it's a max of 50 long
    end

    def dry_run_metadata(metadata, original_artist, name)

      self.name = metadata["name"] || name

      original_artist = metadata["original_artist"] || original_artist
      plan_code = metadata["plan_code"] || gen_plan_code(original_artist, self.name)
      description = metadata["description"]

      @@log.debug("#{self.name} original_artist=#{original_artist}")
      @@log.debug("#{self.name} plan_code=#{plan_code}")

      true
    end

    def determine_genres(metadata)

      genres = []
      if metadata[:genres]
        metadata[:genres].each do |genre|
          if genre == 'hard/metal'
            genres << Genre.find('hard rock')
            genres << Genre.find('metal')
          elsif genre == 'christmas'
            genres << Genre.find('holiday')
          elsif genre == 'alternative'
            genres << Genre.find('alternative rock')
          elsif genre == '80s'
            # swallow
          elsif genre == 'love'
            # swallow
          elsif genre == 'christian' || genre == 'gospel'
            genres << Genre.find('religious')
          elsif genre == 'punk/grunge'
            genres << Genre.find('punk')
          elsif genre == 'electro'
            genres << Genre.find('electronic')
          elsif genre == 'teen pop'
            genres << Genre.find('pop')
          elsif genre == "rock 'n roll"
            genres << Genre.find('rock')
          elsif genre == 'zouk/creole'
            genres << Genre.find('creole')
          elsif genre == 'world/folk'
            genres << Genre.find('world')
            genres << Genre.find('folk')
          elsif genre == 'french pop'
            # swallow
          elsif genre == 'schlager'
            #swallow
          elsif genre == 'humour'
            # swallow
          elsif genre == 'oriental'
            genres << Genre.find('asian')
          else
            found = Genre.find_by_id(genre)
            genres << found if found
          end

        end
      end

      genres
    end

    def determine_language(metadata)

      found = ISO_639.find_by_code('eng')

      language = metadata[:language]

      if language
        language.downcase!

        if language == 'instrumental'
          return 'instrumental'
        end

        if language.include? 'spanish'
          found = ISO_639.find_by_code('spa')
        elsif language.include? 'german'
          found = ISO_639.find_by_code('ger')
        elsif language.include? 'portuguese'
          found = ISO_639.find_by_code('por')
        elsif language.include? 'english'
          found = ISO_639.find_by_code('eng')
        end
      end

      found[0] # 3 letter code
    end

    #http://stackoverflow.com/questions/22740252/how-to-generate-javas-string-hashcode-using-ruby
    def jhash(str)
      result = 0
      mul = 1
      max_mod = 2**31 - 1

      str.chars.reverse_each do |c|
        result += mul * c.ord
        result %= max_mod
        mul *= 31
      end

      result
    end

    def prevent_concurrent_processing(metalocation)

      # use a PG advisory lock to see if someone else is doing this same unit of work right now
      track_code = jhash(metalocation)
      locked = ActiveRecord::Base.connection.execute("SELECT pg_try_advisory_xact_lock(#{track_code})").values[0][0]
      if locked == 'f'
        finish("other_processing", "")
        raise ActiveRecord::Rollback
      end
    end

    def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)

      metadata ||= {}
      self.name = metadata["name"] || name

      prevent_concurrent_processing(metalocation)

      if jam_track.new_record?
        latest_jamtrack = JamTrack.order('created_at desc').first

        id = latest_jamtrack.nil? ? 1 : latest_jamtrack.id.to_i + 1
        if ENV['NODE_NUMBER']
          # complicated goofy code to support parallel processing of importers

          node_number = ENV['NODE_NUMBER'].to_i
          node_count = ENV['NODE_COUNT'].to_i
          raise "NO NODE_COUNT" if node_count == 0
          r = id % node_count
          id = r + id # get to the same base number if both are working at the same time
          id = id + node_number # offset by your node number
          @@log.debug("JAM TRACK ID: #{id}")
        end
        jam_track.id = "#{id}" # default is UUID, but the initial import was based on auto-increment ID, so we'll maintain that
        jam_track.status = 'Staging'
        jam_track.metalocation = metalocation
        jam_track.original_artist = metadata["original_artist"] || original_artist
        jam_track.name = self.name
        jam_track.additional_info = metadata[:additional_info]
        jam_track.year = metadata[:year]
        jam_track.genres = determine_genres(metadata)
        jam_track.language = determine_language(metadata)
        jam_track.plan_code = metadata["plan_code"] || gen_plan_code(jam_track.original_artist, jam_track.name)
        jam_track.price = 1.99
        jam_track.reproduction_royalty_amount = nil
        jam_track.reproduction_royalty = true
        jam_track.public_performance_royalty = true
        jam_track.licensor_royalty_amount = 0.4
        jam_track.sales_region = 'Worldwide'
        jam_track.recording_type = 'Cover'
        jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"."
        jam_track.hfa_license_status = false
        jam_track.alternative_license_status = false
        jam_track.hfa_license_desired = true
        jam_track.server_fixation_date = Time.now
        jam_track.slug = metadata['slug']
        unless jam_track.slug
          jam_track.generate_slug
        end

        if is_tency_storage?
          jam_track.vendor_id = metadata[:id]
          jam_track.licensor = JamTrackLicensor.find_by_name('Tency Music')
          #add_licensor_metadata('Tency Music', metalocation)
        end
      else
        if !options[:resync_audio]
          #@@log.debug("#{self.name} skipped because it already exists in database")
          finish("jam_track_exists", "")
          return false
        else
          # jamtrack exists, leave it be
          return true
        end


      end

      @@log.debug("about to save")
      saved = jam_track.save

      if !saved
        finish("invalid_definition", jam_track.errors.inspect)
      end

      saved
    end

    # oddballs - Guitar Solo.wav
    # Rocket Man Stem - Vocal Back Up
    # Rocket Man Stem - Vocal Lead Double
    # Rock and Roll Stem - Electric Guitar - Main - Solo
    def determine_instrument(potential_instrument_original, potential_part_original = nil)
      potential_instrument = potential_instrument_original.downcase
      potential_part = potential_part_original.downcase if potential_part_original

      instrument = nil
      used_helper = false
      part = nil

      if potential_instrument == 'guitar'
        if potential_part
          if potential_part == 'acoustic'
            instrument = 'acoustic guitar'
            used_helper = true
          elsif potential_part == 'electric'
            instrument = 'electric guitar'
            used_helper = true
          elsif potential_part == 'acoustic solo'
            instrument = 'acoustic guitar'
            used_helper = true
            part = 'Solo'
          elsif potential_part.include?('acoustic')
            used_helper = true # ambiguous
          else
            instrument = 'electric guitar'
            used_helper = false
          end
        else
          instrument = 'electric guitar'
        end
      elsif potential_instrument == 'acoustic'
        instrument = 'acoustic guitar'
      elsif potential_instrument == 'acoutic guitar'
        instrument = 'electric guitar'
      elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary'
        instrument = 'electric guitar'
      elsif potential_instrument == 'keys'
        instrument = 'keyboard'
      elsif potential_instrument == 'vocal' || potential_instrument == 'vocals'
        instrument = 'voice'
      elsif potential_instrument == 'upright bass'
        instrument = 'double bass'
      elsif potential_instrument == 'bass'
        instrument = 'bass guitar'
      elsif potential_instrument == 'drum'
        instrument = 'drums'
      elsif potential_instrument == 'sound effects' || potential_instrument == 'sound efx' || potential_instrument == 'effects'
        instrument = 'computer'

        if potential_part_original
          part = "Sound FX (#{potential_part_original})"
        else
          part = 'Sound FX'
        end
      elsif potential_instrument == 'computer scratches'
        instrument = 'computer'
        part = 'Scratches'
      elsif potential_instrument == "sax"
        instrument = 'saxophone'
      elsif potential_instrument == "vocal back up"
        instrument = "voice"
        part = "Back Up"
      elsif potential_instrument == "vocal lead double"
        instrument = "voice"
        part = "Lead Double"
      elsif potential_instrument == "guitar solo"
        instrument = "electric guitar"
        part = "Solo"
      elsif potential_instrument == 'stadium crowd'
        instrument = 'computer'
        part = 'Crowd Noise'
      elsif potential_instrument == 'cannons'
        instrument = 'computer'
        part = 'Cannons'
      elsif potential_instrument == 'bells'
        instrument = 'computer'
        part = 'Bells'
      elsif potential_instrument == 'percussion'
        instrument = 'drums'
        part = 'Percussion'
      elsif potential_instrument == 'fretless bass'
        instrument = 'bass guitar'
        part = 'Fretless'
      elsif potential_instrument == 'lap steel' || potential_instrument == 'pedal steel'
        instrument = 'steel guitar'
      elsif potential_instrument == 'clock percussion'
        instrument = 'computer'
        part = 'Clock'
      elsif potential_instrument == 'horns' || potential_instrument == 'horn'
        instrument = 'other'
        part = 'Horns' if potential_part.nil?
      elsif potential_instrument == 'english horn'
        instrument = 'other'
        part = 'English Horn'
      elsif potential_instrument == 'bass clarinet'
        instrument = 'other'
        part = 'Bass Clarinet'
      elsif potential_instrument == 'recorder'
        instrument = 'other'
        part = 'Recorder'
      elsif potential_instrument == 'marimba'
        instrument = 'keyboard'
        part = 'Marimba'
      elsif potential_instrument == 'strings'
        instrument = 'orchestra'
        part = 'Strings'
      elsif potential_instrument == 'celesta'
        instrument = 'keyboard'
      elsif potential_instrument == 'balalaika'
        instrument = 'other'
        part = 'Balalaika'
      elsif potential_instrument == 'tanpura'
        instrument = 'other'
        part = 'Tanpura'
      elsif potential_instrument == 'quena'
        instrument = 'other'
        part = 'Quena'
      elsif potential_instrument == 'bouzouki'
        instrument = 'other'
        part = 'Bouzouki'
      elsif potential_instrument == 'claps' || potential_instrument == 'hand claps'
        instrument = 'computer'
        part = 'Claps'
      else
        found_instrument = Instrument.find_by_id(potential_instrument)
        if found_instrument
          instrument = found_instrument.id
        end
      end

      if !used_helper && !part
        part = potential_part_original
      end

      part = potential_instrument_original if !part

      {instrument: instrument,
       part: part}

    end

    def parse_file(file)

      bits = file.split('/')
      filename = bits[bits.length - 1] # remove all but just the filename
      filename_no_ext = filename[0..-5]
      comparable_filename = filename_no_ext.downcase # remove .wav

      type = nil
      master = false
      instrument = nil
      part = nil
      precount_num = nil
      no_precount_detail = nil
      if comparable_filename == "click" || comparable_filename.include?("clicktrack")
        if filename.end_with?('.txt')
          type = :clicktxt
        else
          type = :clickwav
        end
      elsif comparable_filename.include? "precount"
        type = :precount
        index = comparable_filename.index('precount')
        precount = comparable_filename[(index + 'precount'.length)..-1].strip
        if precount.start_with?('_')
          precount = precount[1..-1]
        end
        if precount.to_i == 0
          no_precount_detail = comparable_filename
        else
          precount_num = precount.to_i
        end


      elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix")
        master = true
        type = :master
      else
        type = :track
        stem_location = comparable_filename.index('stem -')
        unless stem_location
          stem_location = comparable_filename.index('stems -')
        end
        unless stem_location
          stem_location = comparable_filename.index('stem-')
        end
        unless stem_location
          stem_location = comparable_filename.index('stems-')
        end

        if stem_location
          bits = filename_no_ext[stem_location..-1].split('-')
          bits.collect! { |bit| bit.strip }

          possible_instrument = nil
          possible_part = nil


          if bits.length == 2
            # second bit is instrument
            possible_instrument = bits[1]
          elsif bits.length == 3
            # second bit is instrument, third bit is part
            possible_instrument = bits[1]
            possible_part = bits[2]
          elsif bits.length == 4
            possible_instrument = bits[1]
            possible_part = "#{bits[2]} #{bits[3]}"
          end

          result = determine_instrument(possible_instrument, possible_part)
          instrument = result[:instrument]
          part = result[:part]
        else
          if is_tency_storage?
            # we can check to see if we can find mapping info for this filename
            mapping = JamTrackImporter.tency_mapping[filename.downcase]

            if mapping && mapping[:trust]
              instrument = mapping[:instrument]
              part = mapping[:part]
            end

            # tency mapping didn't work; let's retry with our own home-grown mapping
            if instrument.nil? && !possible_instrument.nil?
              result = determine_instrument(possible_instrument, possible_part)
              instrument = result[:instrument]
              part = result[:part]
            end
          end
        end

      end


      {filename: filename, master: master, instrument: instrument, part: part, type: type, precount_num: precount_num, no_precount_detail: no_precount_detail}
    end

    def dry_run_audio(metadata, s3_path)
      all_files = fetch_important_files(s3_path)

      all_files.each do |file|

        # ignore click/precount
        parsed_wav = parse_file(file)
        if parsed_wav[:master]
          @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
        elsif parsed_wav[:type] == :track

          JamTrackImporter.summaries[:total_tracks] += 1

          if parsed_wav[:instrument].nil?
            detail = JamTrackImporter.summaries[:no_instrument_detail]
            file_detail = detail[parsed_wav[:filename].downcase]
            if file_detail.nil?
              detail[parsed_wav[:filename].downcase] = 0
            end
            detail[parsed_wav[:filename].downcase] += 1

            JamTrackImporter.summaries[:no_instrument] += 1
          end

          JamTrackImporter.summaries[:no_part] += 1 if parsed_wav[:part].nil?

          if !parsed_wav[:instrument] || !parsed_wav[:part]
            @@log.warn("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
          else
            @@log.debug("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
          end
        elsif parsed_wav[:type] == :clickwav

        elsif parsed_wav[:type] == :clicktxt

        elsif parsed_wav[:type] == :precount
          if parsed_wav[:precount_num].nil?
            JamTrackImporter.summaries[:no_precount_num] += 1
            JamTrackImporter.summaries[:no_precount_detail] << parsed_wav[:no_precount_detail]
          end

        else
          JamTrackImporter.summaries[:unknown_filetype] += 1
        end
      end
    end


    def set_custom_weight(track)

      slop = 800

      instrument_weight = nil
      # if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end

      if track.persisted?
        instrument_weight = track.position
      else
        if track.instrument_id == 'voice'

          if track.part && track.part.start_with?('Lead')
            instrument_weight = 100
          elsif track.part && track.part.start_with?('Backing')
            instrument_weight = 110
          else
            instrument_weight = 120
          end

        elsif track.instrument_id == 'drums'

          if track.part && track.part == 'Drums'
            instrument_weight = 150
          elsif track.part && track.part == 'Percussion'
            instrument_weight = 160
          else
            instrument_weight = 170
          end

        elsif track.instrument_id == 'bass guitar' && track.part && track.part == 'Bass'
          instrument_weight = 180

        elsif track.instrument_id == 'piano' && track.part && track.part == 'Piano'
          instrument_weight = 250

        elsif track.instrument_id == 'keyboard'

          if track.part && track.part.start_with?('Synth')
            instrument_weight = 260
          elsif track.part && track.part.start_with?('Pads')
            instrument_weight = 270
          else
            instrument_weight = 280
          end

        elsif track.instrument_id == 'acoustic guitar'
          if track.part && track.part.start_with?('Lead')
            instrument_weight = 300
          elsif track.part && track.part.start_with?('Rhythm')
            instrument_weight = 310
          else
            instrument_weight = 320
          end
        elsif track.instrument_id == 'electric guitar'
          if track.part && track.part.start_with?('Lead')
            instrument_weight = 400
          elsif track.part && track.part.start_with?('Solo')
            instrument_weight = 410
          elsif track.part && track.part.start_with?('Rhythm')
            instrument_weight = 420
          else
            instrument_weight = 440
          end
        else
          instrument_weight = slop
        end

        if track.track_type == 'Master'
          instrument_weight = 1000
        end
      end


      instrument_weight
    end

    def deduplicate_parts(tracks)
      unique_instruments = {}

      tracks.each do |track|

        key = "#{track.instrument_id} | #{track.part}"
        found = unique_instruments[key]
        if !found
          found = []
          unique_instruments[key] = found
        end

        found << track
      end

      unique_instruments.each do |key, value|
        if value.length > 1
          count = 0

          value.each do |track|
            if track.part.nil?
              track.part = (count + 1).to_s
            else
              track.part = "#{track.part} #{count + 1}"
            end
            count += 1
          end

        end
      end

      # debug output
      tracks.each do |track|
        puts "TRACK #{track.instrument_id} #{track.part}"
      end
    end


    def sort_tracks(tracks)

      sorted_tracks = tracks.sort do |a, b|
        a_weight = set_custom_weight(a)
        b_weight = set_custom_weight(b)

        if a_weight != b_weight
          a_weight <=> b_weight
        elsif a.instrument_id != b.instrument_id
          a.instrument_id <=> b.instrument_id
        else
          a_part = a.part
          b_part = b.part
          a_part <=> b_part
        end
      end

      # default to 1, but if there are any persisted tracks, this will get manipulated to be +1 the highest persisted track
      position = 1
      sorted_tracks.each do |track|
        if track.persisted?
          # persisted tracks should be sorted at the beginning of the sorted_tracks,
          # so this just keeps moving the 'position builder' up to +1 of the last persisted track
          position = track.position + 1
        else
          track.position = position
          position = position + 1
        end


      end

      sorted_tracks[sorted_tracks.length - 1].position = 1000

      sorted_tracks
    end

    def synchronize_audio(jam_track, metadata, s3_path, skip_audio_upload)

      attempt_to_match_existing_tracks = true

      # find all wav files in the JamTracks s3 bucket
      wav_files = fetch_important_files(s3_path)

      tracks = []
      addt_files = []

      wav_files.each do |wav_file|

        if attempt_to_match_existing_tracks
          # try to find a matching track from the JamTrack based on the name of the 44.1 path
          basename = File.basename(wav_file)
          ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"

          found_track = nil
          jam_track.jam_track_tracks.each do |jam_track_track|

            if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename)
              # found a match!
              found_track = jam_track_track
              break
            end
          end

          if found_track
            @@log.debug("found a existing track to reuse")
            found_track.original_audio_s3_path = wav_file
            tracks << found_track
            next
          end
        end

        @@log.debug("no existing track found; creating a new one")

        track = JamTrackTrack.new
        track.original_filename = wav_file
        track.original_audio_s3_path = wav_file

        file = JamTrackFile.new
        file.original_filename = wav_file
        file.original_audio_s3_path = wav_file

        parsed_wav = parse_file(wav_file)

        unknowns = 0
        if parsed_wav[:master]
          track.track_type = 'Master'
          track.part = 'Master Mix'
          track.instrument_id = 'computer'
          tracks << track
          @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
        elsif parsed_wav[:type] == :track

          if !parsed_wav[:instrument] || !parsed_wav[:part]
            @@log.warn("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
            unknowns += 1
          else
            @@log.debug("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
          end


          track.instrument_id = parsed_wav[:instrument] || 'other'
          track.track_type = 'Track'
          track.part = parsed_wav[:part] || "Other #{unknowns}"
          tracks << track
        elsif parsed_wav[:type] == :clicktxt
          file.file_type = 'ClickTxt'
          addt_files << file
        elsif parsed_wav[:type] == :clickwav
          file.file_type = 'ClickWav'
          addt_files << file
        elsif parsed_wav[:type] == :precount
          file.file_type = 'Precount'
          file.precount_num = parsed_wav[:precount_num]
          addt_files << file
        else
          finish("unknown_file_type", "unknown file type #{wave_file}")
          return false
        end

      end

      jam_track.jam_track_tracks.each do |jam_track_track|
        # delete all jam_track_tracks not in the tracks array
        unless tracks.include?(jam_track_track)
          @@log.info("destroying removed JamTrackTrack #{jam_track_track.inspect}")
          jam_track_track.destroy # should also delete s3 files associated with this jamtrack
        end
      end

      jam_track.jam_track_files.each do |jam_track_file|
        unless addt_files.include?(jam_track_file)
          @@log.info("destroying removed JamTrackFile #{jam_track_file.inspect}")
          jam_track_file.destroy # should also delete s3 files associated with this jamtrack
        end
      end

      @@log.info("sorting tracks")
      tracks = sort_tracks(tracks)

      deduplicate_parts(tracks)

      jam_track.jam_track_tracks = tracks
      jam_track.jam_track_files = addt_files

      saved = jam_track.save

      if !saved
        finish('invalid_audio', jam_track.errors.inspect)
        return false
      end

      return synchronize_audio_files(jam_track, skip_audio_upload)
    end

    def synchronize_audio_files(jam_track, skip_audio_upload)

      begin
        Dir.mktmpdir do |tmp_dir|

          jam_track.jam_track_tracks.each do |track|

            basename = File.basename(track.original_audio_s3_path)
            s3_dirname = File.dirname(track.original_audio_s3_path)

            # make a 44100 version, and a 48000 version
            ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
            ogg_48000_filename = File.basename(basename, ".wav") + "-48000.ogg"

            ogg_44100_s3_path = track.filename(ogg_44100_filename)
            ogg_48000_s3_path = track.filename(ogg_48000_filename)

            track.skip_uploader = true

            if skip_audio_upload
              track["url_44"] = ogg_44100_s3_path
              track["md5_44"] = 'md5'
              track["length_44"] = 1

              track["url_48"] = ogg_48000_s3_path
              track["md5_48"] = 'md5'
              track["length_48"] = 1

              # we can't fake the preview as easily because we don't know the MD5 of the current item
              #track["preview_md5"] = 'md5'
              #track["preview_mp3_md5"] = 'md5'
              #track["preview_url"] = track.preview_filename('md5', 'ogg')
              #track["preview_length"] = 1
              #track["preview_mp3_url"] = track.preview_filename('md5', 'mp3')
              #track["preview_mp3_length"] = 1
              #track["preview_start_time"] = 0
            else
              wav_file = File.join(tmp_dir, basename)

              # bring the original wav file down from S3 to local file system
              JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file)

              sample_rate = `soxi -r "#{wav_file}"`.strip

              ogg_44100 = File.join(tmp_dir, ogg_44100_filename)
              ogg_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.ogg")

              if sample_rate == "44100"
                `oggenc "#{wav_file}" -q 6 -o "#{ogg_44100}"`
              else
                `oggenc "#{wav_file}" --resample 44100 -q 6 -o "#{ogg_44100}"`
              end

              if sample_rate == "48000"
                `oggenc "#{wav_file}" -q 6 -o "#{ogg_48000}"`
              else
                `oggenc "#{wav_file}" --resample 48000 -q 6 -o "#{ogg_48000}"`
              end

              # upload the new ogg files to s3
              @@log.debug("uploading 44100 to #{ogg_44100_s3_path}")

              jamkazam_s3_manager.upload(ogg_44100_s3_path, ogg_44100)

              @@log.debug("uploading 48000 to #{ogg_48000_s3_path}")

              jamkazam_s3_manager.upload(ogg_48000_s3_path, ogg_48000)

              ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
              # and finally update the JamTrackTrack with the new info
              track["url_44"] = ogg_44100_s3_path
              track["md5_44"] = ogg_44100_digest.hexdigest
              track["length_44"] = File.new(ogg_44100).size

              track["url_48"] = ogg_48000_s3_path
              track["md5_48"] = ::Digest::MD5.file(ogg_48000).hexdigest
              track["length_48"] = File.new(ogg_48000).size

              synchronize_duration(jam_track, ogg_44100)
              jam_track.save!

              # convert entire master ogg file to mp3, and push both to public destination
              if track.track_type == 'Master'
                preview_succeeded = synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)

                if !preview_succeeded
                  return false
                end
              elsif track.track_type == 'Track'
                synchronize_track_preview(track, tmp_dir, ogg_44100)
              end

            end

            track.save!
          end
        end
      rescue Exception => e
        finish("sync_audio_exception", e.to_s)
        return false
      end

      return true
    end

    def synchronize_duration(jam_track, ogg_44100)
      duration_command = "soxi -D \"#{ogg_44100}\""
      output = `#{duration_command}`

      result_code = $?.to_i

      if result_code == 0
        duration = output.to_f.round
        jam_track.duration = duration
      else
        @@log.warn("unable to determine duration for jam_track #{jam_track.name}. output #{output}")
      end
      true
    end

    def synchronize_track_preview(track, tmp_dir, ogg_44100)

      out_wav = File.join(tmp_dir, 'stripped.wav')

      burp_gaps = ['0.3', '0.2', '0.1', '0.05']

      total_time_command = "soxi -D \"#{ogg_44100}\""
      total_time = `#{total_time_command}`.to_f

      result_code = -20
      stripped_time = total_time # default to the case where we just start the preview at the beginning

      burp_gaps.each do |gap|
        command_strip_lead_silence = "sox \"#{ogg_44100}\" \"#{out_wav}\" silence 1 #{gap} 1%"

        @@log.debug("stripping silence: " + command_strip_lead_silence)

        output = `#{command_strip_lead_silence}`

        result_code = $?.to_i

        if result_code == 0
          stripped_time_command = "soxi -D \"#{out_wav}\""
          stripped_time_test = `#{stripped_time_command}`.to_f

          if stripped_time_test < 1 # meaning a very short duration
            @@log.warn("could not determine the start of non-silencea. assuming beginning")
            stripped_time = total_time # default to the case where we just start the preview at the beginning
          else
            stripped_time = stripped_time_test # accept the measured time of the stripped file and move on by using break
            break
          end
        else
          @@log.warn("unable to determine silence for jam_track #{track.original_filename}, #{output}")
          stripped_time = total_time # default to the case where we just start the preview at the beginning
        end

      end

      preview_start_time = total_time - stripped_time

      # this is in seconds; convert to integer milliseconds
      preview_start_time = (preview_start_time * 1000).to_i

      preview_start_time = nil if preview_start_time < 0

      track.preview_start_time = preview_start_time

      if track.preview_start_time
        @@log.debug("determined track start time to be #{track.preview_start_time}")
      else
        @@log.debug("determined track start time to be #{track.preview_start_time}")
      end

      track.process_preview(ogg_44100, tmp_dir) if track.preview_start_time

      if track.preview_generate_error
        @@log.warn(track.preview_generate_error)
      end

    end

    def synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_digest)
      begin
        aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac')
        convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\""
        @@log.debug("converting to aac using: " + convert_aac_cmd)

        convert_output = `#{convert_aac_cmd}`

        aac_digest = ::Digest::MD5.file(aac_44100)

        track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest

        # upload 44100 aac to public location
        @@log.debug("uploading aac preview to #{track.preview_filename('aac')}")
        public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest)


        track.skip_uploader = true

        original_aac_preview_url = track["preview_aac_url"]

        # and finally update the JamTrackTrack with the new info
        track["preview_aac_url"] = track.preview_filename(aac_md5, 'aac')
        track["preview_aac_length"] = File.new(aac_44100).size
        track["preview_start_time"] = 0

        if !track.save
          finish("save_master_preview", track.errors.to_s)
          return false
        end

        # if all that worked, now delete old previews, if present
        begin
          public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"]
        rescue
          puts "UNABLE TO CLEANUP OLD PREVIEW URL"
        end
      rescue Exception => e
        finish("sync_master_preview_exception", e.to_s)
        return false
      end


      return true

    end

    def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest)

      begin
        mp3_44100 = File.join(tmp_dir, 'output-preview-44100.mp3')
        convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -ab 192k \"#{mp3_44100}\""
        @@log.debug("converting to mp3 using: " + convert_mp3_cmd)

        convert_output = `#{convert_mp3_cmd}`

        mp3_digest = ::Digest::MD5.file(mp3_44100)

        aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac')
        convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\""
        @@log.debug("converting to aac using: " + convert_aac_cmd)

        convert_output = `#{convert_aac_cmd}`

        aac_digest = ::Digest::MD5.file(aac_44100)


        track["preview_md5"] = ogg_md5 = ogg_digest.hexdigest
        track["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest
        track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest

        # upload 44100 ogg, mp3, aac to public location as well
        @@log.debug("uploading ogg preview to #{track.preview_filename('ogg')}")
        public_jamkazam_s3_manager.upload(track.preview_filename(ogg_digest.hexdigest, 'ogg'), ogg_44100, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest)
        @@log.debug("uploading mp3 preview to #{track.preview_filename('mp3')}")
        public_jamkazam_s3_manager.upload(track.preview_filename(mp3_digest.hexdigest, 'mp3'), mp3_44100, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest)
        @@log.debug("uploading aac preview to #{track.preview_filename('aac')}")
        public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest)


        track.skip_uploader = true

        original_ogg_preview_url = track["preview_url"]
        original_mp3_preview_url = track["preview_mp3_url"]
        original_aac_preview_url = track["preview_aac_url"]

        # and finally update the JamTrackTrack with the new info
        track["preview_url"] = track.preview_filename(ogg_md5, 'ogg')
        track["preview_length"] = File.new(ogg_44100).size
        # and finally update the JamTrackTrack with the new info
        track["preview_mp3_url"] = track.preview_filename(mp3_md5, 'mp3')
        track["preview_mp3_length"] = File.new(mp3_44100).size
        track["preview_aac_url"] = track.preview_filename(aac_md5, 'mp3')
        track["preview_aac_length"] = File.new(aac_44100).size
        track["preview_start_time"] = 0

        if !track.save
          finish("save_master_preview", track.errors.to_s)
          return false
        end

        # if all that worked, now delete old previews, if present
        begin
          public_jamkazam_s3_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != track["preview_url"]
          public_jamkazam_s3_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"]
          public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"]
        rescue
          puts "UNABLE TO CLEANUP OLD PREVIEW URL"
        end
      rescue Exception => e
        finish("sync_master_preview_exception", e.to_s)
        return false
      end


      return true

    end

    def fetch_all_files(s3_path)
      JamTrackImporter::song_storage_manager.list_files(s3_path)
    end

    def fetch_important_files(s3_path)
      files = fetch_all_files(s3_path)
      files.select { |file| file.end_with?('.wav') || file.end_with?('.txt') }
    end

    def synchronize(jam_track, metadata, metalocation, options)

      # metalocation should be audio/original artist/song name/meta.yml

      metadata ||= {}

      parsed_metalocation = parse_metalocation(metalocation)

      return unless parsed_metalocation

      original_artist = parsed_metalocation[1]
      name = parsed_metalocation[2]

      success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)

      return unless success

      audio_path = metalocation[0...-"/meta.yml".length]

      synchronized_audio = synchronize_audio(jam_track, metadata, audio_path, options[:skip_audio_upload])

      return unless synchronized_audio

      created_plan = synchronize_recurly(jam_track)
      if created_plan
        finish("success", nil)
      end

      # do a last check on any problems with the jamtrack
      jam_track.sync_onboarding_exceptions

    end

    def synchronize_recurly(jam_track)
      begin
        recurly = RecurlyClient.new
          # no longer create JamTrack plans: VRFS-3028
          # recurly.create_jam_track_plan(jam_track) unless recurly.find_jam_track_plan(jam_track)
      rescue RecurlyClientError => x
        finish('recurly_create_plan', x.errors.to_s)
        return false
      end

      true
    end

    class << self

      attr_accessor :storage_format
      attr_accessor :tency_mapping
      attr_accessor :tency_metadata
      attr_accessor :summaries

      def report_summaries
        @@log.debug("SUMMARIES DUMP")
        @@log.debug("--------------")
        @summaries.each do |k, v|

          if k == :no_instrument_detail
            @@log.debug("#{k}: #{v}")
          elsif k == :no_precount_detail
            v.each do |precount_detail|
              @@log.debug("precount: #{precount_detail}")
            end
          elsif k == :unique_artists
            v.each do |artist|
              @@log.debug("artist: #{artist}")
            end
          else
            @@log.debug("#{k}: #{v}")
          end
        end
      end

      def song_storage_manager
        if is_tency_storage?
          tency_s3_manager
        else
          s3_manager
        end
      end

      def summaries
        @summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}, no_precount_num: 0, no_precount_detail: [], unique_artists: SortedSet.new}
      end

      def tency_s3_manager
        @tency_s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def s3_manager
        @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def private_s3_manager
        @private_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def extract_tency_song_id(metalocation)
        # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml

        first_path = metalocation.index('/')
        return nil unless first_path
        metalocation = metalocation[(first_path + 1)..-1]

        suffix = '/meta.yml'
        metalocation = metalocation[0...-suffix.length]

        last_dash = metalocation.rindex('-')
        return nil if last_dash.nil?

        id = metalocation[(last_dash+1)..-1].strip

        return nil if id.to_i == 0

        id
      end

      def is_tency_storage?
        assert_storage_set
        @storage_format == 'Tency'
      end

      def assert_storage_set
        raise "no storage_format set" if @storage_format.nil?
      end

      def iterate_tency_song_storage(&blk)
        count = 0
        song_storage_manager.list_directories('mapped').each do |song|
          @@log.debug("searching through song directory '#{song}'")

          metalocation = "#{song}meta.yml"

          metadata = load_metalocation(metalocation)

          blk.call(metadata, metalocation)

          count += 1
          #break if count > 100

        end
      end

      def iterate_default_song_storage(&blk)
        song_storage_manager.list_directories('audio').each do |original_artist|
          @@log.debug("searching through artist directory '#{original_artist}'")

          songs = song_storage_manager.list_directories(original_artist)
          songs.each do |song|
            @@log.debug("searching through song directory' #{song}'")

            metalocation = "#{song}meta.yml"

            metadata = load_metalocation(metalocation)

            blk.call(metadata, metalocation)
          end
        end
      end

      def iterate_song_storage(&blk)
        if is_tency_storage?
          iterate_tency_song_storage do |metadata, metalocation|
            blk.call(metadata, metalocation)
          end
        else
          iterate_default_song_storage do |metadata, metalocation|
            blk.call(metadata, metalocation)
          end
        end
      end

      def dry_run
        iterate_song_storage do |metadata, metalocation|
          jam_track_importer = JamTrackImporter.new(@storage_format)

          jam_track_importer.dry_run(metadata, metalocation)
        end

        report_summaries
      end

      # figure out which songs are in S3 that do not exist in the 2k spreadsheet (mapping.csv), and which songs are in the 2k spreadsheet that are not in S3
      def tency_delta
        in_s3 = {}
        in_mapping = {}

        load_tency_mappings

        JamTrackImporter.tency_metadata.each do |song_id, metadata|
          in_mapping[song_id] = {artist: metadata[:original_artist], song: metadata[:name]}
        end

        iterate_song_storage do |metadata, metalocation|

          importer = JamTrackImporter.new(@storage_format)
          song_id = JamTrackImporter.extract_tency_song_id(metalocation)
          parsed_metalocation = importer.parse_metalocation(metalocation)

          next if song_id.nil?
          next if parsed_metalocation.nil?

          original_artist = parsed_metalocation[1]
          meta_name = parsed_metalocation[2]

          in_s3[song_id] = {artist: original_artist, song: meta_name}
        end

        in_s3_keys = Set.new(in_s3.keys)
        in_mapping_keys = Set.new(in_mapping.keys)
        only_in_mapping = in_mapping_keys - in_s3_keys
        only_in_s3 = in_s3_keys - in_mapping_keys

        CSV.open("only_in_s3.csv", "wb") do |csv|
          only_in_s3.each do |song_id|
            csv << [song_id, in_s3[song_id][:artist], in_s3[song_id][:song]]
          end
        end

        CSV.open("only_in_2k_selection.csv", "wb") do |csv|
          only_in_mapping.each do |song_id|
            csv << [song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song]]
          end
        end

      end

      def add_tency_metadata
        JamTrackLicensor.find_by_name('Tency Music').jam_tracks.each do |jam_track|
          jam_track_importer = JamTrackImporter.new(@storage_format)
          jam_track_importer.add_licensor_metadata('Tency Music', jam_track.metalocation)
          break
        end
      end

      def create_masters
        iterate_song_storage do |metadata, metalocation|
          next if metadata.nil?
          jam_track_importer = JamTrackImporter.new(@storage_format)

          jam_track_importer.create_master(metadata, metalocation)
        end
      end

      def create_master(path)
        metalocation = "#{path}/meta.yml"

        metadata = load_metalocation(metalocation)

        jam_track_importer = JamTrackImporter.new(@storage_format)

        jam_track_importer.create_master(metadata, metalocation)
      end

      def dry_run_original
        s3_manager.list_directories('audio').each do |original_artist|
          @@log.debug("searching through artist directory '#{original_artist}'")

          songs = s3_manager.list_directories(original_artist)
          songs.each do |song|
            @@log.debug("searching through song directory' #{song}'")

            metalocation = "#{song}meta.yml"

            metadata = load_metalocation(metalocation)

            jam_track_importer = JamTrackImporter.new

            jam_track_importer.dry_run(metadata, metalocation)
          end
        end

      end

      def synchronize_preview(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        error_occurred = false
        error_msg = nil
        jam_track.jam_track_tracks.each do |track|
          next if track.track_type == 'Master'

          if track.preview_start_time
            track.generate_preview
            if track.preview_generate_error
              error_occurred = true
              error_msg = track.preview_generate_error
            else
            end
          end
        end

        if error_occurred
          importer.finish('preview_error', error_msg)
        else
          importer.finish('success', nil)
        end
        importer
      end

      # hunts for the most recent .aac, .mp3, or .ogg file
      def synchronize_preview_dev(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        importer.synchronize_preview_dev(jam_track)

        importer.finish('success', nil)
        importer
      end

      def synchronize_jamtrack_aac_preview(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        track = jam_track.master_track

        if track
          Dir.mktmpdir do |tmp_dir|
            ogg_44100 = File.join(tmp_dir, 'input.ogg')
            private_s3_manager.download(track.url_by_sample_rate(44), ogg_44100)
            ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
            if importer.synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)
              importer.finish("success", nil)
            end
          end
        else
          importer.finish('no_master_track', nil)
        end
        importer
      end

      def synchronize_jamtrack_master_preview(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        master_track = jam_track.master_track

        if master_track
          Dir.mktmpdir do |tmp_dir|
            ogg_44100 = File.join(tmp_dir, 'input.ogg')
            private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
            ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
            if importer.synchronize_master_preview(master_track, tmp_dir, ogg_44100, ogg_44100_digest)
              importer.finish("success", nil)
            end
          end
        else
          importer.finish('no_master_track', nil)
        end

        importer
      end

      def synchronize_previews_dev
        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_preview_dev(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "no_preview_start_time"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def synchronize_previews
        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_preview(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "no_preview_start_time"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def synchronize_jamtrack_aac_previews

        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_jamtrack_aac_preview(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end


        end
      end

      def synchronize_jamtrack_master_previews
        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_jamtrack_master_preview(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def synchronize_duration(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        master_track = jam_track.master_track
        if master_track
          Dir.mktmpdir do |tmp_dir|
            ogg_44100 = File.join(tmp_dir, 'input.ogg')
            private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)

            if importer.synchronize_duration(jam_track, ogg_44100)
              jam_track.save!
              importer.finish("success", nil)
            end
          end
        else
          importer.finish('no_duration', nil)
        end

        importer
      end

      def synchronize_durations
        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_duration(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def download_master(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        Dir.mkdir('tmp') unless Dir.exists?('tmp')
        Dir.mkdir('tmp/jam_track_masters') unless Dir.exists?('tmp/jam_track_masters')

        master_track = jam_track.master_track
        if master_track
          ogg_44100 = File.join('tmp/jam_track_masters', "#{jam_track.original_artist} - #{jam_track.name}.ogg")
          private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
        end
        importer
      end

      def download_masters
        importers = []

        JamTrack.all.each do |jam_track|
          importers << download_master(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to download.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def remove_s3_special_chars(filename)
        filename.tr('/&@:,$=+?;\^`><{}[]#%~|', '')
      end

      def generate_slugs
        JamTrack.all.each do |jam_track|
          jam_track.generate_slug
          jam_track.save!
        end
      end

      def onboarding_exceptions
        JamTrack.all.each do |jam_track|
          jam_track.onboarding_exceptions
        end
      end

      def synchronize_all(options)
        importers = []

        count = 0
        iterate_song_storage do |metadata, metalocation|

          next if metadata.nil? && is_tency_storage?

          importer = synchronize_from_meta(metalocation, options)
          importers << importer

          if importer.reason != 'jam_track_exists' && importer.reason != "other_processing"
            count+=1
          end

          if count > 500
            #break
          end
        end


        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def jam_track_dry_run(metalocation)
        # see if we can find a JamTrack with this metalocation
        jam_track = JamTrack.find_by_metalocation(metalocation)

        meta = load_metalocation(metalocation)

        if jam_track
          @@log.debug("jamtrack #{jam_track.name} located by metalocation")
          jam_track.dry_run(meta, metalocation)
        else
          jam_track = JamTrack.new
          jam_track.dry_run(meta, metalocation)
        end
      end

      def genre_dump
        load_tency_mappings

        genres = {}
        @tency_metadata.each do |id, value|

          genre1 = value[:genre1]
          genre2 = value[:genre2]
          genre3 = value[:genre3]
          genre4 = value[:genre4]
          genre5 = value[:genre5]

          genres[genre1.downcase.strip] = genre1.downcase.strip if genre1
          genres[genre2.downcase.strip] = genre2.downcase.strip if genre2
          genres[genre3.downcase.strip] = genre3.downcase.strip if genre3
          genres[genre4.downcase.strip] = genre4.downcase.strip if genre4
          genres[genre5.downcase.strip] = genre5.downcase.strip if genre5
        end

        all_genres = Genre.select(:id).all.map(&:id)

        all_genres = Set.new(all_genres)
        genres.each do |genre, value|
          found = all_genres.include? genre

          puts "#{genre}" unless found
        end
      end

      def load_tency_mappings
        Dir.mktmpdir do |tmp_dir|
          mapping_file = File.join(tmp_dir, 'mapping.csv')
          metadata_file = File.join(tmp_dir, 'metadata.csv')

          # this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv
          if ENV['TENCY_ALREADY_DOWNLOADED'] == '1'
            mapping_file = 'mapping.csv'
            metadata_file = 'metadata.csv'
          else
            tency_s3_manager.download('mapping/mapping.csv', mapping_file)
            tency_s3_manager.download('mapping/metadata.csv', metadata_file)
          end

          mapping_csv = CSV.read(mapping_file)
          metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false)

          @tency_mapping = {}
          @tency_metadata = {}
          # convert both to hashes
          mapping_csv.each do |line|
            @tency_mapping[line[0].strip] = {instrument: line[1], part: line[2], count: line[3], trust: line[4]}
          end

          metadata_csv.each do |line|
            @tency_metadata[line[0].strip] = {id: line[0].strip, original_artist: line[1], name: line[2], additional_info: line[3], year: line[4], language: line[5], isrc: line[10], genre1: line[11], genre2: line[12], genre3: line[13], genre4: line[14], genre5: line[15]}
          end


          @tency_metadata.each do |id, value|

            genres = []

            genre1 = value[:genre1]
            genre2 = value[:genre2]
            genre3 = value[:genre3]
            genre4 = value[:genre4]
            genre5 = value[:genre5]

            genres << genre1.downcase.strip if genre1
            genres << genre2.downcase.strip if genre2
            genres << genre3.downcase.strip if genre3
            genres << genre4.downcase.strip if genre4
            genres << genre5.downcase.strip if genre5

            value[:genres] = genres
          end


        end
      end

      def load_metalocation(metalocation)

        if is_tency_storage?
          load_tency_mappings if @tency_mapping.nil?
          song_id = extract_tency_song_id(metalocation)

          if song_id.nil?
            puts "missing_song_id #{metalocation}"
            return nil
          end


          tency_data = @tency_metadata[song_id]

          if tency_data.nil?
            @@log.warn("missing tency metadata '#{song_id}'")
          end

          return tency_data
        else
          begin
            data = s3_manager.read_all(metalocation)
            return YAML.load(data)
          rescue AWS::S3::Errors::NoSuchKey
            return nil
          end
        end
      end

      def create_from_metalocation(meta, metalocation, options = {skip_audio_upload: false})
        jam_track = JamTrack.new
        sync_from_metadata(jam_track, meta, metalocation, options)
      end

      def update_from_metalocation(jam_track, meta, metalocation, options)
        sync_from_metadata(jam_track, meta, metalocation, options)
      end

      def sync_from_metadata(jam_track, meta, metalocation, options)
        jam_track_importer = JamTrackImporter.new(@storage_format)

        JamTrack.connection.execute('SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED')

        JamTrack.transaction do

          #begin
          jam_track_importer.synchronize(jam_track, meta, metalocation, options)
          #rescue Exception => e
          #  jam_track_importer.finish("unhandled_exception", e.to_s)
          #end

          if jam_track_importer.reason != "success"
            raise ActiveRecord::Rollback
          end
        end

        jam_track_importer
      end

      def synchronize_from_meta(metalocation, options)
        # see if we can find a JamTrack with this metalocation
        jam_track = JamTrack.find_by_metalocation(metalocation)

        meta = load_metalocation(metalocation)

        if meta.nil? && is_tency_storage?
          raise "no tency song matching this metalocation #{metalocation}"
        end
        jam_track_importer = nil
        if jam_track
          @@log.debug("jamtrack #{jam_track.name} located by metalocation")
          jam_track_importer = update_from_metalocation(jam_track, meta, metalocation, options)
        else
          jam_track_importer = create_from_metalocation(meta, metalocation, options)
        end

        if jam_track_importer.reason == "success"
          @@log.info("#{jam_track_importer.name} successfully imported")
        else
          @@log.error("#{jam_track_importer.name} failed to import.")
          @@log.error("#{jam_track_importer.name} reason=#{jam_track_importer.reason}")
          @@log.error("#{jam_track_importer.name} detail=#{jam_track_importer.detail}")
        end

        jam_track_importer
      end
    end
  end
end