jam-cloud/ruby/lib/jam_ruby/jam_track_importer.rb

require 'json'
require 'tempfile'
require 'open3'
require 'fileutils'
require 'open-uri'
require 'yaml'

module JamRuby

  class JamTrackImporter

    @@log = Logging.logger[JamTrackImporter]

    attr_accessor :name
    attr_accessor :metadata
    attr_accessor :reason
    attr_accessor :detail
    attr_accessor :storage_format

    def jamkazam_s3_manager
      @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
    end

    def public_jamkazam_s3_manager
      @public_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_public, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
    end

    def initialize(storage_format = 'default')
      @storage_format = storage_format
    end

    def finish(reason, detail)
      @@log.info("JamTrackImporter:#{self.name} #{reason} #{detail}")
      self.reason = reason
      self.detail = detail

      if ENV['END_ON_FAIL'] == "1" && reason != 'success' && reason != 'jam_track_exists'
        raise "#{reason} #{detail}"
      end
    end

    def import_click_track(jam_track)
      # we need to download the click track, if it exists.
      Dir.mktmpdir do |tmp_dir|

        @@log.info("importing clicking track for #{jam_track.original_artist}:#{jam_track.name}")

        if jam_track.click_track
          @@log.info("already has click track: #{jam_track.original_artist}:#{jam_track.name}")
          finish('success', 'already_has_click_track')
          return
        end

        click_track_file = jam_track.click_track_file
        if click_track_file.nil?
          @@log.info("no click track for #{jam_track.original_artist}:#{jam_track.name}")
          finish('success', 'no_click_track')
          return
        end

        original_filename = click_track_file[:original_filename]

        if original_filename.nil?
          @@log.info("no click track s3 path for #{jam_track.original_artist}:#{jam_track.name}")
          finish('no_original_source', 'click track is missing s3 path:' + click_track_file.id)
          return
        end

        #wav_file = File.join(tmp_dir, File.basename(click_track_file[:original_filename]))
        #JamTrackImporter.song_storage_manager.download(click_track_file[:original_filename], wav_file)

        JamTrack.transaction do
          click_track = jam_track.click_track

          if click_track.nil?
            click_track = JamTrackTrack.new
            click_track.original_filename = click_track_file[:original_filename]
            click_track.original_audio_s3_path = click_track_file[:original_filename]
            click_track.track_type = 'Click'
            click_track.part = 'Clicktrack'
            click_track.instrument_id = 'computer'
            click_track.jam_track = jam_track
            click_track.position = 10000
            if !click_track.save
              @@log.error("unable to create jamtrack click track #{click_track.errors.inspect}")
              finish("jam_track_click", "unable to create: #{click_track.errors.inspect}")
              return false
            end
          end

          jam_track.increment_version!

          # with the click track in hand, flesh out the details
          synchronize_audio_track(jam_track, tmp_dir, false, click_track)

          finish('success', nil)
        end
      end
    end

    def generate_jmep(jam_track)

      # https://docs.google.com/spreadsheets/d/1dyUOjWkeU8BXwnJl-ws1Kvxq_twWEG7E78F29haYkLc/edit#gid=987457683
      # cross-check against marks_approved

      if JamTrackImporter.marks_approved && JamTrackImporter.marks_approved.has_key?(jam_track.slug)
        @@log.info("Found track in mark approved list. skipping")
        finish('success', 'mark@jamkazam.com created')
        return
      end

      # can we overwrite this one?
      if jam_track.jmep_text.blank? || jam_track.jmep_text.include?('created via code')
        @@log.info("This is a blank jmep or created earlier by code.")
      else
        @@log.info("This a JMEP that was not created by code. Skip it.")
        return
      end

      # we need to download the click track, if it exists.
      Dir.mktmpdir do |tmp_dir|

        master_track = jam_track.master_track

        click_track = jam_track.click_track_file

        if master_track.nil?
          finish('no_master_track', nil)
          return
        end

        master_track_file = File.join(tmp_dir, File.basename(master_track[:url_48]))
        begin
          JamTrackImporter.private_s3_manager.download(master_track.url_by_sample_rate(44), master_track_file)
        rescue Exception => e
          @@log.error("unable to download master track")
          finish("no-download-master", master_track.url_by_sample_rate(44))
          return
        end

        if click_track
          click_track_file = File.join(tmp_dir, File.basename(click_track[:original_filename]))
          JamTrackImporter.song_storage_manager.download(click_track[:original_filename], click_track_file)
        else
          # we'll use the master for click analysis. not ideal, but would work
          click_track_file = master_track_file
        end


        if click_track
          start_time = determine_start_time(click_track_file, tmp_dir, click_track[:original_filename], false)
        else
          start_time = determine_start_time(master_track_file, tmp_dir, master_track[:url], false)
        end

        trimmed_for_beat_analysis = File.join(tmp_dir, 'trimmed_for_beat.wav')
        # trim out the 1st 5 second after non-silence
        trim_cmd = "sox #{Shellwords.escape(click_track_file)} #{Shellwords.escape(trimmed_for_beat_analysis)} trim #{start_time} #{start_time + 5}"
        cmd = "bash -c #{Shellwords.escape(trim_cmd)}"
        @@log.debug("executing cmd #{cmd}")
        output=`#{cmd}`
        result_code = $?.to_i

        if result_code != 0
          finish("trim-fail", "failed to run trim click track: #{output}")
          return
        end

        # bpm comes from git clone http://www.pogo.org.uk/~mark/bpm-tools.git
        sox="sox #{Shellwords.escape(trimmed_for_beat_analysis)} -t raw -r 44100 -e float -c 1 - | bpm  -m 25 -x 250"
        cmd = "bash -c #{Shellwords.escape(sox)}"
        @@log.debug("executing cmd #{cmd}")
        output=`#{cmd}`
        result_code = $?.to_i

        if result_code == 0
          bpm = output.to_f

          offset = 0.140
          if bpm >= 60 && bpm < 80
            offset = 0.110
          elsif bpm >= 80 && bpm < 100
            offset = 0.080
          elsif bpm >= 100 && bpm < 120
            offset = 0.050
          elsif bpm >= 120
            offset = 0.020
          end

          @@log.debug("bpm: #{bpm} start_time: #{start_time}, offset: #{offset}")

          start_time += offset
          metro_fin = "#{Time.at(start_time).utc.strftime("%H:%M:%S")}:#{((start_time - start_time.to_i) * 1000).round.to_s.rjust(3, "0")}"

          jmep = ""
          jmep << "# created via code using bpm/silence detection (bpm:#{bpm} offset:#{offset})\r\n"
          jmep << "prelude@10.0 #number of seconds before music starts\r\n"
          jmep << "metro_fin@#{metro_fin} bpm=#{bpm}, ticks=8, pmode=stream, name=Beep, play=mono"

          @@log.info("jmep generated: #{jmep}")

          jam_track.jmep_text = jmep
          if jam_track.save
            finish('success', nil)
          else
            @@log.error("jamtrack did not save.  #{jam_track.errors.inspect}")
            finish("no-save", "jamtrack did not save.  #{jam_track.errors.inspect}")
            return
          end
        else
          finish("bpm-fail", "failed to run bpm: #{output}")
          return
        end
      end
    end


    def determine_start_time(audio_file, tmp_dir, original_filename, protect_short = true)
      burp_gaps = ['0.3', '0.2', '0.1', '0.05', '0.025']

      out_wav = File.join(tmp_dir, 'stripped.wav')
      total_time_command = "soxi -D \"#{audio_file}\""
      total_time = `#{total_time_command}`.to_f

      result_code = -20
      stripped_time = total_time # default to the case where we just start the preview at the beginning

      burp_gaps.each do |gap|
        command_strip_lead_silence = "sox \"#{audio_file}\" \"#{out_wav}\" silence 1 #{gap} 1%"

        @@log.debug("stripping silence: " + command_strip_lead_silence)

        output = `#{command_strip_lead_silence}`

        result_code = $?.to_i

        if result_code == 0
          stripped_time_command = "soxi -D \"#{out_wav}\""
          stripped_time_test = `#{stripped_time_command}`.to_f

          if stripped_time_test < 1 # meaning a very short duration
            @@log.warn("could not determine the start of non-silence. assuming beginning")
            stripped_time = total_time # default to the case where we just start the preview at the beginning
          else
            stripped_time = stripped_time_test # accept the measured time of the stripped file and move on by using break
            break
          end
        else
          @@log.warn("unable to determine silence for jam_track #{original_filename}, #{output}")
          stripped_time = total_time # default to the case where we just start the preview at the beginning
        end
      end

      preview_start_time = total_time - stripped_time

      preview_start_time
    end

    def synchronize_preview_dev(jam_track)
      jam_track.jam_track_tracks.each do |track|
        next if track.track_type != 'Master'


        most_recent_aac = nil
        most_recent_ogg = nil
        most_recent_mp3 = nil
        public_jamkazam_s3_manager.list_files(track.preview_directory).each do |s3_preview_item|

          s3_object = public_jamkazam_s3_manager.object(s3_preview_item)

          if s3_preview_item.end_with?('.aac')
            if most_recent_aac
              if s3_object.last_modified > most_recent_aac.last_modified
                most_recent_aac = s3_object
              end
            else
              most_recent_aac = s3_object
            end
          end

          if s3_preview_item.end_with?('.mp3')
            if most_recent_mp3
              if s3_object.last_modified > most_recent_mp3.last_modified
                most_recent_mp3 = s3_object
              end
            else
              most_recent_mp3 = s3_object
            end
          end

          if s3_preview_item.end_with?('.ogg')
            if most_recent_ogg
              if s3_object.last_modified > most_recent_ogg.last_modified
                most_recent_ogg = s3_object
              end
            else
              most_recent_ogg = s3_object
            end
          end
        end

        if most_recent_aac
          track['preview_aac_md5'] = 'md5'
          track['preview_aac_url'] = most_recent_aac.key
          track['preview_aac_length'] = most_recent_aac.content_length
        end

        if most_recent_mp3
          track['preview_mp3_md5'] = 'md5'
          track['preview_mp3_url'] = most_recent_mp3.key
          track['preview_mp3_length'] = most_recent_mp3.content_length
        end

        if most_recent_ogg
          track['preview_md5'] = 'md5'
          track['preview_url'] = most_recent_ogg.key
          track['preview_length'] = most_recent_ogg.content_length
        end

        track.save
      end
    end

    def create_silence(tmp_dir, segment_count, duration, sample_rate, channels = 2)
      file = File.join(tmp_dir, "#{segment_count}.wav")

      # -c 2 means stereo
      cmd("sox -n -r #{sample_rate} -c #{channels} #{file} trim 0.0 #{duration}", "silence")

      file
    end

    # this method was created due to Tency-sourced data having no master track
    # it goes through all audio tracks, and creates a master mix from it.  (mix + normalize)
    def create_master(metadata, metalocation)

      parsed_metalocation = parse_metalocation(metalocation)

      if parsed_metalocation.nil?
        finish("invalid_metalocation", metalocation)
        return
      end

      original_artist = parsed_metalocation[1]
      meta_name = parsed_metalocation[2]

      self.name = metadata[:name] || meta_name


      audio_path = metalocation[0...-"/meta.yml".length]

      all_files = fetch_important_files(audio_path)

      audio_files = []
      master_found = false
      all_files.each do |file|

        parsed_wav = parse_file(file)
        if parsed_wav[:master]
          master_found = true
        elsif parsed_wav[:type] == :track

          audio_files << file
        end
      end

      if master_found
        @@log.debug("master exists... skipping #{self.name} ")
        finish('success', nil)
        return
      else

        tracks = []

        #tmp_dir = Dir.mktmpdir
        #tmp_dir = "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150809-9945-1ykr85u"
        Dir.mktmpdir do |tmp_dir|
          @@log.debug("downloading all audio files in #{tmp_dir}")
          audio_files.each do |s3_track|
            track = File.join(tmp_dir, File.basename(s3_track))
            tracks << track
            JamTrackImporter.song_storage_manager.download(s3_track, track)
          end

          # first have to check if all are the same sample rate.   If not, we have to make it so

          first_sample_rate = nil
          normalize_needed = false
          tracks.each do |track|
            sample_rate = `soxi -r "#{track}"`.strip

            if first_sample_rate.nil?
              first_sample_rate = sample_rate
            else
              if first_sample_rate != sample_rate
                # we need to normalize all of them
                normalize_needed = true
                break
              end
            end
          end

          normalized_tracks = []
          if normalize_needed
            tracks.each do |track|
              normalized_track = File.join(tmp_dir, 'normalized-' + File.basename(track))
              output = `sox "#{track}" "#{normalized_track}" rate #{first_sample_rate}`
              @@log.debug("resampling #{normalized_track}; output: #{output}")
              normalized_tracks << normalized_track
            end
            tracks = normalized_tracks
          end


          temp_file = File.join(tmp_dir, "temp.wav")
          output_filename = JamTrackImporter.remove_s3_special_chars("#{self.name} Master Mix.wav")
          output_file = File.join(tmp_dir, output_filename)
          command = "sox -m "
          tracks.each do |track|
            command << " \"#{track}\""
          end
          command << " \"#{temp_file}\""

          @@log.debug("mixing with cmd: " + command)
          sox_output = `#{command}`
          result_code = $?.to_i

          if result_code != 0
            @@log.error("unable to generate master mix")
            finish("sox_master_mix_failure", sox_output)
          else

            # now normalize the audio

            command = "sox --norm \"#{temp_file}\" \"#{output_file}\""
            @@log.debug("normalizing with cmd: " + command)
            sox_output = `#{command}`
            result_code = $?.to_i
            if result_code != 0
              @@log.error("unable to normalize master mix")
              finish("sox_master_mix_failure", sox_output)
            else

              # now we need to upload the output back up
              s3_target = audio_path + '/' + output_filename
              @@log.debug("uploading #{output_file} to #{s3_target}")
              JamTrackImporter.song_storage_manager.upload(s3_target, output_file)

              finish('success', nil)
            end

          end
        end
      end
    end

    def dry_run(metadata, metalocation)
      # STDIN.gets

      @@log.debug("dry_run: #{metadata.inspect}")
      metadata ||= {}

      parsed_metalocation = parse_metalocation(metalocation)

      return unless parsed_metalocation

      original_artist = parsed_metalocation[1]
      name = parsed_metalocation[2]

      JamTrackImporter.summaries[:unique_artists] << original_artist

      success = dry_run_metadata(metadata, original_artist, name)

      return unless success

      audio_path = metalocation[0...-"/meta.yml".length]


      dry_run_audio(metadata, audio_path)

      finish("success", nil)
    end

    def add_licensor_metadata(vendor, metalocation)
      Dir.mktmpdir do |tmp_dir|
        @@log.debug("update vendor metadata")
        meta_yml = File.join(tmp_dir, 'meta.yml')
        if jamkazam_s3_manager.exists?(metalocation)
          jamkazam_s3_manager.download(metalocation, meta_yml)
          meta = YAML.load_file(meta_yml)
        else
          meta = {}
        end

        meta[:licensor] = vendor

        File.open(meta_yml, 'w') { |f| f.write meta.to_yaml }

        jamkazam_s3_manager.upload(metalocation, meta_yml)
      end
    end

    def is_tency_storage?
      assert_storage_set
      @storage_format == 'Tency'
    end

    def is_paris_storage?
      assert_storage_set
      @storage_format == 'Paris'
    end

    def is_tim_tracks_storage?
      assert_storage_set
      @storage_format == 'TimTracks'
    end

    def is_drumma_storage?
      assert_storage_set
      @storage_format == 'Drumma'
    end

    def assert_storage_set
      raise "no storage_format set" if @storage_format.nil?
    end


    def parse_metalocation(metalocation)
      # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml
      if is_drumma_storage?

        suffix = '/meta.yml'

        unless metalocation.end_with? suffix
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        metalocation = metalocation[0...-suffix.length]

        bits = ['audio']
        first_dash = metalocation.index(' - ')
        if first_dash
          artist = metalocation[0...(first_dash)].strip
          bits << artist
        else
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end
        song = metalocation[(first_dash+3)..-1].strip
        bits << song

      elsif is_tency_storage? || is_tim_tracks_storage?

        suffix = '/meta.yml'

        unless metalocation.end_with? suffix
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        metalocation = metalocation[0...-suffix.length]

        first_path = metalocation.index('/')
        if first_path.nil?
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end
        metalocation = metalocation[(first_path + 1)..-1]

        bits = ['audio']
        # example: Sister Hazel - All For You - 10385
        first_dash = metalocation.index(' - ')
        if first_dash
          artist = metalocation[0...(first_dash)].strip
          bits << artist
        else
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        if is_tim_tracks_storage?
          song = metalocation[(first_dash+3)..-1].strip
          bits << song
        elsif is_tency_storage?
          last_dash = metalocation.rindex('-')
          if last_dash
            song = metalocation[(first_dash+3)...last_dash].strip
            bits << song
          else
            finish("invalid_metalocation", "metalocation not valid #{metalocation}")
            return nil
          end
        end

        bits << 'meta.yml'
        bits
      elsif is_paris_storage?
        suffix = '/meta.yml'

        unless metalocation.end_with? suffix
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        metalocation = metalocation[0...-suffix.length]

        first_path = metalocation.index('/')
        if first_path.nil?
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end
        metalocation = metalocation[(first_path + 1)..-1]

        bits = ['audio']


        last_slash = metalocation.rindex('/')

        # example: S4863-Mike Oldfield-Moonlight Shadow-000bpm


        if last_slash
          paris_artist_song_id = metalocation[0...last_slash]
        else
          paris_artist_song_id = metalocation
        end


        bitbits = paris_artist_song_id.split('-')
        song_id = bitbits[0].strip

        artist = bitbits[1]
        song_name = bitbits[2]
        bpm = bitbits[-1]

        bits << artist
        bits << song_name

        bits << 'meta.yml'
        bits << song_id
        bits << bpm
        bits
      else
        bits = metalocation.split('/')

        if bits.length != 4
          finish("invalid_metalocation", "metalocation not valid #{metalocation}")
          return nil
        end

        if bits[0] != "audio"
          finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}")
          return nil
        end

        if bits[3] != 'meta.yml'
          finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}")
          return nil
        end

        bits
      end
    end

    def dry_run_metadata(metadata, original_artist, name)

      self.name = metadata["name"] || name

      original_artist = metadata["original_artist"] || original_artist
      description = metadata["description"]

      @@log.debug("#{self.name} original_artist=#{original_artist}")

      true
    end

    def determine_genres(metadata)

      genres = []
      if metadata[:genres]
        metadata[:genres].each do |genre|
          if genre == 'hard/metal'
            genres << Genre.find('hard rock')
            genres << Genre.find('metal')
          elsif genre == 'christmas'
            genres << Genre.find('holiday')
          elsif genre == 'alternative'
            genres << Genre.find('alternative rock')
          elsif genre == '80s' || genre == "50's" || genre == "60's" || genre == "70's" || genre == "80's" || genre == "90's" || genre == "50/60's" || genre == "00's" || genre == "2010's"
            # swallow
          elsif genre == 'love'
            # swallow
          elsif genre == 'christian' || genre == 'gospel'
            genres << Genre.find('religious')
          elsif genre == 'punk/grunge'
            genres << Genre.find('punk')
          elsif genre == 'electro'
            genres << Genre.find('electronic')
          elsif genre == 'teen pop'
            genres << Genre.find('pop')
          elsif genre == "rock 'n roll"
            genres << Genre.find('rock')
          elsif genre == 'zouk/creole'
            genres << Genre.find('creole')
          elsif genre == 'world/folk'
            genres << Genre.find('world')
            genres << Genre.find('folk')
          elsif genre == 'french pop'
            # swallow
          elsif genre == 'schlager'
            #swallow
          elsif genre == 'humour'
            # swallow
          elsif genre == 'oriental'
            genres << Genre.find('asian')
          elsif genre == 'abba'
            genres << Genre.find('pop')
          elsif genre == 'movies tv show' || genre == "movies"
            genres << Genre.find('tv & movie soundtrack')
          elsif genre == 'ballad'
            # swallow
          elsif genre == "r'n'b" || genre == "pop rnb"
            genres << Genre.find("r&b")
          elsif genre == "rock & roll"
            genres << Genre.find('rock')
          elsif genre == "dance pop"
            genres << Genre.find('dance')
          elsif genre == "soul/motown" || genre == "soul motown"
            genres << Genre.find('soul')
          elsif genre == "party"
            # swallow
          elsif genre == "reggae/ska" || genre == "reggae ska"
            genres << Genre.find('reggae')
            genres << Genre.find('ska')
          elsif genre == "pop rock" || genre == "pop/rock"
            genres << Genre.find("rock")
            genres << Genre.find("pop")
          elsif genre == "singalong"
            #swallow
          elsif genre == "folk rock"
            genres << Genre.find('folk')
            genres << Genre.find('rock')
          elsif genre == "swing" || genre == "swing/big band" || genre == "swing big band"
            genres << Genre.find('oldies')
          elsif genre == "rap/hip hop" || genre == "rap hip hop"
            genres << Genre.find("rap")
          elsif genre == "folk traditional" || genre == "folk/traditional"
            genres << Genre.find('folk')
          elsif genre == "elvis"
            genres << Genre.find('rock')
          elsif genre == "irish"
            genres << Genre.find('celtic')
          elsif genre == "dance/pop"
            genres << Genre.find('dance')
            genres << Genre.find('pop')
          elsif genre == "the beatles"
            genres << Genre.find("rock")
          else
            found = Genre.find_by_id(genre)
            genres << found if found
          end

        end
      end

      # just throw them into rock/pop if not known.  can fix later...
      if genres.length == 0
        genres << Genre.find('rock')
        genres << Genre.find('pop')
      end

      genres
    end

    def determine_language(metadata)

      found = ISO_639.find_by_code('eng')

      language = metadata[:language]

      if language
        language.downcase!

        if language == 'instrumental'
          return 'instrumental'
        end

        if language.include? 'spanish'
          found = ISO_639.find_by_code('spa')
        elsif language.include? 'german'
          found = ISO_639.find_by_code('ger')
        elsif language.include? 'portuguese'
          found = ISO_639.find_by_code('por')
        elsif language.include? 'english'
          found = ISO_639.find_by_code('eng')
        end
      end

      found[0] # 3 letter code
    end

    #http://stackoverflow.com/questions/22740252/how-to-generate-javas-string-hashcode-using-ruby
    def jhash(str)
      result = 0
      mul = 1
      max_mod = 2**31 - 1

      str.chars.reverse_each do |c|
        result += mul * c.ord
        result %= max_mod
        mul *= 31
      end

      result
    end

    def prevent_concurrent_processing(metalocation)

      # use a PG advisory lock to see if someone else is doing this same unit of work right now
      track_code = jhash(metalocation)
      locked = ActiveRecord::Base.connection.execute("SELECT pg_try_advisory_xact_lock(#{track_code})").values[0][0]
      if locked == 'f'
        finish("other_processing", "")
        raise ActiveRecord::Rollback
      end
    end

    def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)

      metadata ||= {}
      self.name = metadata["name"] || name

      prevent_concurrent_processing(metalocation)

      if jam_track.new_record?
        latest_jamtrack = JamTrack.order('id::int desc').first

        id = latest_jamtrack.nil? ? 1 : latest_jamtrack.id.to_i + 1
        if ENV['NODE_NUMBER']
          # complicated goofy code to support parallel processing of importers

          node_number = ENV['NODE_NUMBER'].to_i
          node_count = ENV['NODE_COUNT'].to_i
          raise "NO NODE_COUNT" if node_count == 0
          r = id % node_count
          id = (node_count - r) + id # get to the same base number if both are working at the same time
          id = id + node_number # offset by your node number
          @@log.info("JAM TRACK ID: #{id}")
        end
        jam_track.id = "#{id}" # default is UUID, but the initial import was based on auto-increment ID, so we'll maintain that
        jam_track.status = 'Staging'
        jam_track.metalocation = metalocation
        jam_track.original_artist = metadata["original_artist"] || original_artist
        jam_track.name = self.name
        jam_track.additional_info = metadata[:additional_info]
        jam_track.year = metadata[:year]
        jam_track.genres = determine_genres(metadata)
        jam_track.language = determine_language(metadata)
        jam_track.price = 1.99
        jam_track.reproduction_royalty_amount = nil
        jam_track.reproduction_royalty = true
        jam_track.public_performance_royalty = true
        jam_track.licensor_royalty_amount = 0.4
        jam_track.sales_region = 'Worldwide'
        jam_track.recording_type = 'Cover'
        jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"."
        jam_track.hfa_license_status = false
        jam_track.alternative_license_status = false
        jam_track.hfa_license_desired = true
        jam_track.server_fixation_date = Time.now
        if is_tency_storage?
          jam_track.vendor_id = metadata[:id]
          jam_track.licensor = JamTrackLicensor.find_by_name!('Tency Music')
          #add_licensor_metadata('Tency Music', metalocation)
        elsif is_paris_storage?
          raise 'no vendor id' if metadata[:id].nil?
          jam_track.vendor_id = metadata[:id]
          jam_track.licensor = JamTrackLicensor.find_by_name!('Paris Music')
          jam_track.bpm = metadata[:bpm]
        elsif is_tim_tracks_storage?
          jam_track.vendor_id = metadata[:id]
          jam_track.licensor = JamTrackLicensor.find_by_name!('Tim Waurick')
        elsif is_drumma_storage?
          jam_track.vendor_id = metadata[:id]
          jam_track.licensor = JamTrackLicensor.find_by_name!('Drumma Boy')
        end
        jam_track.slug = metadata['slug']
        if jam_track.slug.nil?
          jam_track.generate_slug
        end
        jam_track.plan_code = metadata["plan_code"]
        if jam_track.plan_code.nil?
          jam_track.gen_plan_code
        end


      else
        if !options[:resync_audio]
          #@@log.debug("#{self.name} skipped because it already exists in database")
          finish("jam_track_exists", "")
          return false
        else
          # jamtrack exists, leave it be
          return true
        end


      end

      @@log.debug("about to save")
      saved = jam_track.save

      if !saved
        finish("invalid_definition", jam_track.errors.inspect)
      end

      saved
    end

    # oddballs - Guitar Solo.wav
    # Rocket Man Stem - Vocal Back Up
    # Rocket Man Stem - Vocal Lead Double
    # Rock and Roll Stem - Electric Guitar - Main - Solo
    def determine_instrument(potential_instrument_original, potential_part_original = nil)
      potential_instrument = potential_instrument_original.downcase
      potential_part = potential_part_original.downcase if potential_part_original

      instrument = nil
      used_helper = false
      part = nil

      if potential_instrument == 'guitar'
        if potential_part
          if potential_part == 'acoustic'
            instrument = 'acoustic guitar'
            used_helper = true
          elsif potential_part == 'electric'
            instrument = 'electric guitar'
            used_helper = true
          elsif potential_part == 'acoustic solo'
            instrument = 'acoustic guitar'
            used_helper = true
            part = 'Solo'
          elsif potential_part.include?('acoustic')
            used_helper = true # ambiguous
          else
            instrument = 'electric guitar'
            used_helper = false
          end
        else
          instrument = 'electric guitar'
        end
      elsif potential_instrument == 'acoustic'
        instrument = 'acoustic guitar'
      elsif potential_instrument == 'acoutic guitar'
        instrument = 'electric guitar'
      elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary' || potential_instrument == 'elec guitar'
        instrument = 'electric guitar'
      elsif potential_instrument == 'lead guitar'
        instrument = 'electric guitar'
        part = 'Lead'
      elsif potential_instrument == 'keys'
        instrument = 'keyboard'
      elsif potential_instrument == 'vocal' || potential_instrument == 'vocals'
        instrument = 'voice'
      elsif potential_instrument == 'upright bass'
        instrument = 'double bass'
      elsif potential_instrument == 'bass'
        instrument = 'bass guitar'
      elsif potential_instrument == 'drum'
        instrument = 'drums'
      elsif potential_instrument == 'sound effects' || potential_instrument == 'sound efx' || potential_instrument == 'effects'
        instrument = 'computer'

        if potential_part_original
          part = "Sound FX (#{potential_part_original})"
        else
          part = 'Sound FX'
        end
      elsif potential_instrument == 'computer scratches'
        instrument = 'computer'
        part = 'Scratches'
      elsif potential_instrument == "sax"
        instrument = 'saxophone'
      elsif potential_instrument == "vocal back up"
        instrument = "voice"
        part = "Back Up"
      elsif potential_instrument == "vocal lead double"
        instrument = "voice"
        part = "Lead Double"
      elsif potential_instrument == "guitar solo"
        instrument = "electric guitar"
        part = "Solo"
      elsif potential_instrument == 'stadium crowd'
        instrument = 'computer'
        part = 'Crowd Noise'
      elsif potential_instrument == 'cannons'
        instrument = 'computer'
        part = 'Cannons'
      elsif potential_instrument == 'bells'
        instrument = 'computer'
        part = 'Bells'
      elsif potential_instrument == 'percussion'
        instrument = 'percussion'
        part = 'Percussion'
      elsif potential_instrument == 'fretless bass'
        instrument = 'bass guitar'
        part = 'Fretless'
      elsif potential_instrument == 'lap steel' || potential_instrument == 'pedal steel'
        instrument = 'steel guitar'
      elsif potential_instrument == 'clock percussion'
        instrument = 'computer'
        part = 'Clock'
      elsif potential_instrument == 'horns' || potential_instrument == 'horn'
        instrument = 'other'
        part = 'Horns' if potential_part.nil?
      elsif potential_instrument == 'english horn'
        instrument = 'other'
        part = 'English Horn'
      elsif potential_instrument == 'bass clarinet'
        instrument = 'other'
        part = 'Bass Clarinet'
      elsif potential_instrument == 'recorder'
        instrument = 'other'
        part = 'Recorder'
      elsif potential_instrument == 'marimba'
        instrument = 'keyboard'
        part = 'Marimba'
      elsif potential_instrument == 'strings'
        instrument = 'orchestra'
        part = 'Strings'
      elsif potential_instrument == 'celesta' || potential_instrument == 'celeste'
        instrument = 'keyboard'
        part = 'Celesta'
      elsif potential_instrument == 'balalaika'
        instrument = 'other'
        part = 'Balalaika'
      elsif potential_instrument == 'tanpura'
        instrument = 'other'
        part = 'Tanpura'
      elsif potential_instrument == 'quena'
        instrument = 'other'
        part = 'Quena'
      elsif potential_instrument == 'bouzouki'
        instrument = 'other'
        part = 'Bouzouki'
      elsif potential_instrument == 'claps' || potential_instrument == 'hand claps'
        instrument = 'other'
        part = 'Claps'
      elsif potential_instrument == 'snaps' || potential_instrument == 'snap'
        instrument = 'other'
        part = 'Snaps'
      else
        found_instrument = Instrument.find_by_id(potential_instrument)
        if found_instrument
          instrument = found_instrument.id
        end
      end

      if !used_helper && !part
        part = potential_part_original
      end

      part = potential_instrument_original if !part

      {instrument: instrument,
       part: part}

    end

    def parse_file(file)

      bits = file.split('/')
      filename = bits[bits.length - 1] # remove all but just the filename
      filename_no_ext = filename[0..-5]
      comparable_filename = filename_no_ext.downcase # remove .wav

      type = nil
      master = false
      instrument = nil
      part = nil
      precount_num = nil
      no_precount_detail = nil
      if comparable_filename == "click" || comparable_filename.include?("clicktrack") || comparable_filename.include?("click track") || comparable_filename.end_with?('click') || comparable_filename.end_with?('click trac')
        if filename.end_with?('.txt')
          type = :clicktxt
        else
          type = :clickwav
        end
      elsif comparable_filename.include? "precount"
        type = :precount
        index = comparable_filename.index('precount')
        precount = comparable_filename[(index + 'precount'.length)..-1].strip
        if precount.start_with?('_')
          precount = precount[1..-1]
        end
        if precount.to_i == 0
          no_precount_detail = comparable_filename
        else
          precount_num = precount.to_i
        end

      elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") || (@metadata && (@metadata[:id] && comparable_filename.start_with?(@metadata[:id].downcase)))
        master = true
        type = :master
      else
        type = :track
        stem_location = comparable_filename.index('stem -')
        unless stem_location
          stem_location = comparable_filename.index('stems -')
        end
        unless stem_location
          stem_location = comparable_filename.index('stem-')
        end
        unless stem_location
          stem_location = comparable_filename.index('stems-')
        end
        unless stem_location
          stem_location = comparable_filename.index(' -')
        end

        if stem_location
          bits = filename_no_ext[stem_location..-1].split('-')
          bits.collect! { |bit| bit.strip }

          possible_instrument = nil
          possible_part = nil


          if bits.length == 2
            # second bit is instrument
            possible_instrument = bits[1]
          elsif bits.length == 3
            # second bit is instrument, third bit is part
            possible_instrument = bits[1]
            possible_part = bits[2]
          elsif bits.length == 4
            possible_instrument = bits[1]
            possible_part = "#{bits[2]} #{bits[3]}"
          end

          result = determine_instrument(possible_instrument, possible_part)
          instrument = result[:instrument]
          part = result[:part]


          if is_tim_tracks_storage?
            if instrument.nil?
              # some stems don't include 'voice'. but... they are all voice :)
              instrument = 'voice'
            end
          end
        else
          if is_tency_storage?
            # we can check to see if we can find mapping info for this filename
            mapping = JamTrackImporter.tency_mapping[filename.downcase]

            if mapping && mapping[:trust]
              instrument = mapping[:instrument]
              part = mapping[:part]
            end

            # tency mapping didn't work; let's retry with our own home-grown mapping
            if instrument.nil? && !possible_instrument.nil?
              result = determine_instrument(possible_instrument, possible_part)
              instrument = result[:instrument]
              part = result[:part]
            end
          elsif is_paris_storage?
            # example: Eternal Flame-Guide Lead Vocal.wav
            # or with a part: Eternal Flame-Keyboard-Stab.wav
            bits = filename_no_ext.split('-')
            bits.collect! { |bit| bit.strip }

            while true
              instrument, part = paris_instrument_parse(bits)

              if instrument.nil? && bits.length > 2
                bits.shift
                instrument, part = paris_instrument_parse(bits)
              else
                break
              end
            end
          end

        end
      end

      {filename: filename, master: master, instrument: instrument, part: part, type: type, precount_num: precount_num, no_precount_detail: no_precount_detail}
    end

    def paris_instrument_parse(bits)
      instrument = nil
      part = nil
      possible_instrument = nil
      possible_part = nil

      if bits.length == 2
        # second bit is instrument
        possible_instrument = bits[1]
      elsif bits.length == 3
        # second bit is instrument, third bit is part
        possible_instrument = bits[1]
        possible_part = bits[2]
      elsif bits.length >= 4
        possible_instrument = bits[1]
        possible_part = "#{bits[2]} #{bits[3]}"
      end

      # otherwise, try mapping
      if instrument.nil? && possible_instrument
        mapping = JamTrackImporter.paris_mapping[possible_instrument.downcase]
        if mapping
          instrument = mapping[:instrument].downcase
          part = mapping[:part]
          part = nil if part.blank?
        end
      end

      # paris mapping didn't work; let's retry  one more time with our own home-grown mapping
      if instrument.nil?
        result = determine_instrument(possible_instrument, possible_part)
        instrument = result[:instrument]
        part = result[:part]
      end
      return instrument, part
    end

    def dry_run_audio(metadata, s3_path)
      all_files = fetch_important_files(s3_path)

      masters = 0
      all_files.each do |file|

        # ignore click/precount
        parsed_wav = parse_file(file)
        if parsed_wav[:master]
          @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
          masters += 1
          if masters > 1
            JamTrackImporter.summaries[:multiple_masters] += 1
          end
        elsif parsed_wav[:type] == :track

          JamTrackImporter.summaries[:total_tracks] += 1

          if parsed_wav[:instrument].nil?
            detail = JamTrackImporter.summaries[:no_instrument_detail]
            file_detail = detail[parsed_wav[:filename].downcase]
            if file_detail.nil?
              detail[parsed_wav[:filename].downcase] = 0
            end
            detail[parsed_wav[:filename].downcase] += 1

            JamTrackImporter.summaries[:no_instrument] += 1
          end

          JamTrackImporter.summaries[:no_part] += 1 if parsed_wav[:part].nil?

          if !parsed_wav[:instrument] || !parsed_wav[:part]
            @@log.warn("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
          else
            @@log.debug("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
          end
        elsif parsed_wav[:type] == :clickwav

        elsif parsed_wav[:type] == :clicktxt

        elsif parsed_wav[:type] == :precount
          if parsed_wav[:precount_num].nil?
            JamTrackImporter.summaries[:no_precount_num] += 1
            JamTrackImporter.summaries[:no_precount_detail] << parsed_wav[:no_precount_detail]
          end

        else
          JamTrackImporter.summaries[:unknown_filetype] += 1
        end
      end
    end


    def set_custom_weight(track)

      slop = 800

      instrument_weight = nil
      # if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end

      #if track.persisted?
      #  instrument_weight = track.position
      #else
      if track.instrument_id == 'voice'

        if track.part && track.part.start_with?('Lead')
          instrument_weight = 100
        elsif track.part && track.part.start_with?('Backing')
          instrument_weight = 110
        else
          instrument_weight = 120
        end

      elsif track.instrument_id == 'drums'

        if track.part && track.part == 'Drums'
          instrument_weight = 150
        elsif track.part && track.part == 'Percussion'
          instrument_weight = 160
        else
          instrument_weight = 170
        end
      elsif track.instrument_id == 'percussion'
        instrument_weight = 175
      elsif track.instrument_id == 'bass guitar' && track.part && track.part == 'Bass'
        instrument_weight = 180

      elsif track.instrument_id == 'piano' && track.part && track.part == 'Piano'
        instrument_weight = 250

      elsif track.instrument_id == 'keyboard'

        if track.part && track.part.start_with?('Synth')
          instrument_weight = 260
        elsif track.part && track.part.start_with?('Pads')
          instrument_weight = 270
        else
          instrument_weight = 280
        end

      elsif track.instrument_id == 'acoustic guitar'
        if track.part && track.part.start_with?('Lead')
          instrument_weight = 300
        elsif track.part && track.part.start_with?('Rhythm')
          instrument_weight = 310
        else
          instrument_weight = 320
        end
      elsif track.instrument_id == 'electric guitar'
        if track.part && track.part.start_with?('Lead')
          instrument_weight = 400
        elsif track.part && track.part.start_with?('Solo')
          instrument_weight = 410
        elsif track.part && track.part.start_with?('Rhythm')
          instrument_weight = 420
        else
          instrument_weight = 440
        end
      else
        instrument_weight = slop
      end

      if track.track_type == 'Master'
        instrument_weight = 1000
      end

      if track.track_type == 'Click'
        instrument_weight = 10000
      end
      #end


      instrument_weight
    end

    def deduplicate_parts(tracks)
      unique_instruments = {}

      tracks.each do |track|

        key = "#{track.instrument_id} | #{track.part}"
        found = unique_instruments[key]
        if !found
          found = []
          unique_instruments[key] = found
        end

        found << track
      end

      unique_instruments.each do |key, value|
        if value.length > 1
          count = 0

          value.each do |track|
            if track.part.nil?
              track.part = (count + 1).to_s
            else
              track.part = "#{track.part} #{count + 1}"
            end
            count += 1
          end

        end
      end

      # debug output
      tracks.each do |track|
        puts "TRACK #{track.instrument_id} #{track.part}"
      end
    end


    def sort_tracks(tracks)

      sorted_tracks = tracks.sort do |a, b|
        a_weight = set_custom_weight(a)
        b_weight = set_custom_weight(b)

        if a_weight != b_weight
          a_weight <=> b_weight
        elsif a.instrument_id != b.instrument_id
          a.instrument_id <=> b.instrument_id
        else
          a_part = a.part
          b_part = b.part
          a_part <=> b_part
        end
      end

      position = 1
      sorted_tracks.each do |track|
        track.position = position
        position = position + 1
      end

      # get click/master tracks position re-set correctly

      last_track = sorted_tracks[sorted_tracks.length - 1]
      second_to_last = sorted_tracks[sorted_tracks.length - 2]

      if last_track.track_type == 'Master'
        last_track.position = 1000
      elsif last_track.track_type == 'Click'
        last_track.position = 10000
      end

      if second_to_last.track_type == 'Master'
        second_to_last.position = 1000
      elsif second_to_last.track_type == 'Click'
        second_to_last.position = 10000
      end

      sorted_tracks
    end

    # this will put original_audio_s3_path on each jam_track_track
    def associate_tracks_with_original_stems(jam_track, s3_path)
      attempt_to_match_existing_tracks = true

      # find all wav files in the JamTracks s3 bucket
      wav_files = fetch_important_files(s3_path)

      tracks = []

      wav_files.each do |wav_file|

        if attempt_to_match_existing_tracks
          # try to find a matching track from the JamTrack based on the name of the 44.1 path
          basename = File.basename(wav_file)
          ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"

          found_track = nil
          jam_track.jam_track_tracks.each do |jam_track_track|

            if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename)
              # found a match!
              found_track = jam_track_track
              break
            end
          end

          if found_track
            @@log.debug("found a existing track to reuse")
            found_track.original_audio_s3_path = wav_file
            tracks << found_track
            next
          end
        end
      end

      tracks
    end

    def synchronize_audio(jam_track, metadata, s3_path, skip_audio_upload)

      attempt_to_match_existing_tracks = true

      # find all wav files in the JamTracks s3 bucket
      wav_files = fetch_important_files(s3_path)

      tracks = []
      addt_files = []

      wav_files.each do |wav_file|

        if attempt_to_match_existing_tracks
          # try to find a matching track from the JamTrack based on the name of the 44.1 path
          basename = File.basename(wav_file)
          ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"

          found_track = nil
          jam_track.jam_track_tracks.each do |jam_track_track|

            if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename)
              # found a match!
              found_track = jam_track_track
              break
            end
          end

          if found_track
            @@log.debug("found a existing track to reuse")
            found_track.original_audio_s3_path = wav_file
            tracks << found_track
            next
          end
        end

        @@log.debug("no existing track found; creating a new one")

        if !assign_instrument_parts(wav_file, tracks, addt_files)
          return false
        end
      end

      jam_track.jam_track_tracks.each do |jam_track_track|
        # delete all jam_track_tracks not in the tracks array
        unless tracks.include?(jam_track_track)
          @@log.info("destroying removed JamTrackTrack #{jam_track_track.inspect}")
          jam_track_track.destroy # should also delete s3 files associated with this jamtrack
        end
      end

      jam_track.jam_track_files.each do |jam_track_file|
        unless addt_files.include?(jam_track_file)
          @@log.info("destroying removed JamTrackFile #{jam_track_file.inspect}")
          jam_track_file.destroy # should also delete s3 files associated with this jamtrack
        end
      end

      @@log.info("sorting tracks")
      tracks = sort_tracks(tracks)

      deduplicate_parts(tracks)

      jam_track.jam_track_tracks = tracks
      jam_track.jam_track_files = addt_files

      saved = jam_track.save

      if !saved
        finish('invalid_audio', jam_track.errors.inspect)
        return false
      end

      return synchronize_audio_files(jam_track, skip_audio_upload)
    end

    def reassign_instrument_parts(jam_track)

      tracks = []
      addt_files = []

      jam_track.jam_track_tracks.each do |track|
        return if !assign_instrument_parts(track.original_filename, tracks, addt_files, true)
      end

      @@log.info("sorting #{tracks.length} tracks")
      tracks = sort_tracks(tracks)

      deduplicate_parts(tracks)

      changed = false
      tracks.each do |track|
        if track.changed?
          changed = true
          puts "CHANGE: #{track.changes.inspect}"
          track.skip_inst_part_uniq = true
          track.save!
        end
      end

      if changed
        # if we messed up any instrument/parts by making a dup, this will catch it
        tracks.each do |track|
          track.skip_inst_part_uniq = false
          track.save!
        end
      end
    end

    def assign_instrument_parts(wav_file, tracks, addt_files, reassign = false)
      if !reassign
        track = JamTrackTrack.new
        track.original_filename = wav_file
        track.original_audio_s3_path = wav_file

        file = JamTrackFile.new
        file.original_filename = wav_file
        file.original_audio_s3_path = wav_file
      else
        matches = JamTrackTrack.where(original_filename: wav_file)
        if matches.count > 1
          raise "multiple jam track tracks encountered with #{wav_file} as original_filename"
        elsif matches.count == 0
          raise "unable to locate jam track wit h#{wav_file} as original_filename"
        end
        track = matches[0]
        track.original_audio_s3_path = wav_file
        file = nil
      end

      parsed_wav = parse_file(wav_file)

      unknowns = 0
      if parsed_wav[:master]
        track.track_type = 'Master'
        track.part = 'Master Mix'
        track.instrument_id = 'computer'
        tracks << track
        @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}")
      elsif parsed_wav[:type] == :track

        if !parsed_wav[:instrument] || !parsed_wav[:part]
          @@log.warn("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
          unknowns += 1
        else
          @@log.debug("#{self.name} track!  instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ")
        end

        track.instrument_id = parsed_wav[:instrument] || 'other'
        track.track_type = 'Track'
        track.part = parsed_wav[:part];
        tracks << track
      elsif parsed_wav[:type] == :clicktxt
        file.file_type = 'ClickTxt'
        addt_files << file
      elsif parsed_wav[:type] == :clickwav
        if file
          file.file_type = 'ClickWav'
          addt_files << file
        end

        # and also add a JamTrackTrack for this click track
        track.track_type = 'Click'
        track.part = 'Clicktrack'
        track.instrument_id = 'computer'
        track.position = 10000
        tracks << track
      elsif parsed_wav[:type] == :precount
        file.file_type = 'Precount'
        file.precount_num = parsed_wav[:precount_num]
        addt_files << file
      else
        finish("unknown_file_type", "unknown file type #{wave_file}")
        return false
      end

      return true
    end

    def synchronize_audio_files(jam_track, skip_audio_upload)

      begin
        Dir.mktmpdir do |tmp_dir|

          # download each jam track here, and then do processing to determine:
          # what's the longest stem
          # and to then pad the rest of the tracks to make them all match in length
          jam_track.jam_track_tracks.each do |track|
            basename = File.basename(track.original_audio_s3_path)
            wav_file = File.join(tmp_dir, basename)

            # bring the original wav file down from S3 to local file system
            JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file)
            track.wav_file = wav_file
          end

          same_lengthening(jam_track, tmp_dir)

          jam_track.jam_track_tracks.each do |track|
            synchronize_audio_track(jam_track, tmp_dir, skip_audio_upload, track)
          end
        end

        generate_jmep(jam_track)
      rescue Exception => e
        finish("sync_audio_exception", e.to_s)
        return false
      end

      return true
    end

    # make all stems be the same length
    def same_lengthening(jam_track, tmp_dir)
      longest_duration = nil
      jam_track.jam_track_tracks.each do |track|
        duration_command = "soxi -D \"#{track.wav_file}\""
        output = `#{duration_command}`

        result_code = $?.to_i

        if result_code == 0
          duration = output.to_f.round

          track.tmp_duration = duration
          if longest_duration.nil?
            longest_duration = duration
          else
            if duration > longest_duration
              longest_duration = duration
            end
          end
        else
          @@log.warn("unable to determine duration for jam_track track #{jam_track.name} #{jam_track_track.instrument} #{jam_track_track.part}. output #{output}")
        end
      end

      @@log.info("duration determined to be #{longest_duration}")
      jam_track.duration = longest_duration
      jam_track.jam_track_tracks.each do |track|
        if track.tmp_duration < longest_duration
          # need to pad with silence to make all match in length

          amount = longest_duration - track.tmp_duration

          @@log.info("track #{track.instrument_id}:#{track.part} needs to be lengthened by #{amount}")

          output = cmd("soxi -c \"#{track.wav_file}\"", "padded_silence")
          channels = output.to_i

          output = cmd("soxi -r \"#{track.wav_file}\"", "get_sample_rate")
          sample_rate = output.to_i

          padding_file = create_silence(tmp_dir, "padded_silence#{track.id}", amount, sample_rate, channels)

          output_file = File.join(tmp_dir, "with_padding_#{track.id}.wav")

          cmd("sox \"#{track.wav_file}\" \"#{padding_file}\" \"#{output_file}\"", "same_lengthening")

          track.wav_file = output_file
        end
      end
    end

    def cmd(cmd, type)

      @@log.debug("executing #{cmd}")

      output = `#{cmd}`

      result_code = $?.to_i

      if result_code == 0
        output
      else
        @error_reason = type + "_fail"
        @error_detail = "#{cmd}, #{output}"
        raise "command `#{cmd}` failed. #{type}, #{output}"
      end
    end

    def synchronize_audio_track(jam_track, tmp_dir, skip_audio_upload, track)
      basename = File.basename(track.original_audio_s3_path)

      # make a 44100 version, and a 48000 version
      ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg"
      ogg_48000_filename = File.basename(basename, ".wav") + "-48000.ogg"


      # make a 44100 version, and a 48000 version
      mp3_48000_filename = File.basename(basename, ".wav") + "-48000.mp3"
      aac_48000_filename = File.basename(basename, ".wav") + "-48000.aac"

      ogg_44100_s3_path = track.filename(ogg_44100_filename)
      ogg_48000_s3_path = track.filename(ogg_48000_filename)

      mp3_48000_s3_path = track.filename(mp3_48000_filename)
      aac_48000_s3_path = track.filename(aac_48000_filename)

      track.skip_uploader = true

      if skip_audio_upload
        track["url_44"] = ogg_44100_s3_path
        track["md5_44"] = 'md5'
        track["length_44"] = 1

        track["url_48"] = ogg_48000_s3_path
        track["md5_48"] = 'md5'
        track["length_48"] = 1

        track["url_mp3_48"] = mp3_48000_filename
        track["md5_mp3_48"] = 'md5'
        track["length_mp3_48"] = 1

        track["url_aac_48"] = aac_48000_filename
        track["md5_aac_48"] = 'md5'
        track["length_aac_48"] = 1

        # we can't fake the preview as easily because we don't know the MD5 of the current item
        #track["preview_md5"] = 'md5'
        #track["preview_mp3_md5"] = 'md5'
        #track["preview_url"] = track.preview_filename('md5', 'ogg')
        #track["preview_length"] = 1
        #track["preview_mp3_url"] = track.preview_filename('md5', 'mp3')
        #track["preview_mp3_length"] = 1
        #track["preview_start_time"] = 0
      else
        wav_file = track.wav_file

        sample_rate = `soxi -r "#{wav_file}"`.strip

        ogg_44100 = File.join(tmp_dir, ogg_44100_filename)
        ogg_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.ogg")

        if sample_rate == "44100"
          `oggenc "#{wav_file}" -q 6 -o "#{ogg_44100}"`
        else
          `oggenc "#{wav_file}" --resample 44100 -q 6 -o "#{ogg_44100}"`
        end

        if sample_rate == "48000"
          `oggenc "#{wav_file}" -q 6 -o "#{ogg_48000}"`
        else
          `oggenc "#{wav_file}" --resample 48000 -q 6 -o "#{ogg_48000}"`
        end

        # upload the new ogg files to s3
        @@log.debug("uploading 44100 to #{ogg_44100_s3_path}")

        jamkazam_s3_manager.upload(ogg_44100_s3_path, ogg_44100)

        @@log.debug("uploading 48000 to #{ogg_48000_s3_path}")

        jamkazam_s3_manager.upload(ogg_48000_s3_path, ogg_48000)

        ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
        # and finally update the JamTrackTrack with the new info
        track["url_44"] = ogg_44100_s3_path
        track["md5_44"] = ogg_44100_digest.hexdigest
        track["length_44"] = File.new(ogg_44100).size

        track["url_48"] = ogg_48000_s3_path
        track["md5_48"] = ::Digest::MD5.file(ogg_48000).hexdigest
        track["length_48"] = File.new(ogg_48000).size

        # now create mp3 and aac files
        mp3_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.mp3")
        aac_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.aac")

        `ffmpeg -i "#{wav_file}" -ar 48000 -ab 192k "#{mp3_48000}"`

        `ffmpeg -i "#{wav_file}" -c:a libfdk_aac -b:a 192k "#{aac_48000}"`

        # upload the new ogg files to s3
        @@log.debug("uploading mp3 48000 to #{mp3_48000_s3_path}")

        jamkazam_s3_manager.upload(mp3_48000_s3_path, mp3_48000)

        @@log.debug("uploading aac 48000 to #{aac_48000_s3_path}")

        jamkazam_s3_manager.upload(aac_48000_s3_path, aac_48000)

        mp3_48000_digest = ::Digest::MD5.file(mp3_48000)
        # and finally update the JamTrackTrack with the new info
        track["url_mp3_48"] = mp3_48000_s3_path
        track["md5_mp3_48"] = mp3_48000_digest.hexdigest
        track["length_mp3_48"] = File.new(mp3_48000).size

        track["url_aac_48"] = aac_48000_s3_path
        track["md5_aac_48"] = ::Digest::MD5.file(aac_48000).hexdigest
        track["length_aac_48"] = File.new(aac_48000).size

        jam_track.save!

        # convert entire master ogg file to mp3, and push both to public destination
        if track.track_type == 'Master'
          preview_succeeded = synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)

          if !preview_succeeded
            return false
          end
        elsif track.track_type == 'Track' || track.track_type == 'Click'
          synchronize_track_preview(track, tmp_dir, ogg_44100)
        end

      end

      track.save!
    end

    def generate_mp3_aac_stem(jam_track, tmp_dir, skip_audio_upload)
      jam_track.jam_track_tracks.each do |track|

        if track.original_audio_s3_path.nil?

          @@log.error("jam_track #{jam_track.name} has empty stem. stem: #{track.id}")
          next
        end

        puts "track.original_audio_s3_path #{track.original_audio_s3_path}"
        basename = File.basename(track.original_audio_s3_path)
        s3_dirname = File.dirname(track.original_audio_s3_path)

        # make a 44100 version, and a 48000 version
        mp3_48000_filename = File.basename(basename, ".wav") + "-48000.mp3"
        aac_48000_filename = File.basename(basename, ".wav") + "-48000.aac"

        mp3_48000_s3_path = track.filename(mp3_48000_filename)
        aac_48000_s3_path = track.filename(aac_48000_filename)

        puts "mp3_48000_s3_path #{mp3_48000_s3_path}"
        track.skip_uploader = true

        if skip_audio_upload
          track["url_mp3_48"] = mp3_48000_filename
          track["md5_mp3_48"] = 'md5'
          track["length_mp3_48"] = 1

          track["url_aac_48"] = aac_48000_filename
          track["md5_aac_48"] = 'md5'
          track["length_aac_48"] = 1

          # we can't fake the preview as easily because we don't know the MD5 of the current item
          #track["preview_md5"] = 'md5'
          #track["preview_mp3_md5"] = 'md5'
          #track["preview_url"] = track.preview_filename('md5', 'ogg')
          #track["preview_length"] = 1
          #track["preview_mp3_url"] = track.preview_filename('md5', 'mp3')
          #track["preview_mp3_length"] = 1
          #track["preview_start_time"] = 0
        else
          wav_file = File.join(tmp_dir, basename)

          # the wave file might already be on the system...

          # don't bother with the same track twice

          next if track["url_mp3_48"] && track["url_aac_48"]

          # bring the original wav file down from S3 to local file system
          JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file) unless File.exists?(wav_file)

          mp3_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.mp3")
          aac_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.aac")

          `ffmpeg -i "#{wav_file}" -ar 48000 -ab 192k "#{mp3_48000}"`

          `ffmpeg -i "#{wav_file}" -c:a libfdk_aac -b:a 192k "#{aac_48000}"`

          # upload the new ogg files to s3
          @@log.debug("uploading mp3 48000 to #{mp3_48000_s3_path}")

          jamkazam_s3_manager.upload(mp3_48000_s3_path, mp3_48000)

          @@log.debug("uploading aac 48000 to #{aac_48000_s3_path}")

          jamkazam_s3_manager.upload(aac_48000_s3_path, aac_48000)

          mp3_48000_digest = ::Digest::MD5.file(mp3_48000)
          # and finally update the JamTrackTrack with the new info
          track["url_mp3_48"] = mp3_48000_s3_path
          track["md5_mp3_48"] = mp3_48000_digest.hexdigest
          track["length_mp3_48"] = File.new(mp3_48000).size

          track["url_aac_48"] = aac_48000_s3_path
          track["md5_aac_48"] = ::Digest::MD5.file(aac_48000).hexdigest
          track["length_aac_48"] = File.new(aac_48000).size
          track.save
        end
      end
    end


    def synchronize_duration(jam_track, ogg_44100)
      duration_command = "soxi -D \"#{ogg_44100}\""
      output = `#{duration_command}`

      result_code = $?.to_i

      if result_code == 0
        duration = output.to_f.round
        jam_track.duration = duration
      else
        @@log.warn("unable to determine duration for jam_track #{jam_track.name}. output #{output}")
      end
      true
    end

    def synchronize_track_preview(track, tmp_dir, ogg_44100)

      preview_start_time = determine_start_time(ogg_44100, tmp_dir, track.original_filename)

      # this is in seconds; convert to integer milliseconds
      preview_start_time = (preview_start_time * 1000).to_i

      preview_start_time = 0 if preview_start_time < 0

      track.preview_start_time = preview_start_time

      track.process_preview(ogg_44100, tmp_dir) if track.preview_start_time

      if track.preview_generate_error
        @@log.warn(track.preview_generate_error)
      end

    end

    def synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_digest)
      begin
        aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac')
        convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\""
        @@log.debug("converting to aac using: " + convert_aac_cmd)

        convert_output = `#{convert_aac_cmd}`

        aac_digest = ::Digest::MD5.file(aac_44100)

        track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest

        # upload 44100 aac to public location
        @@log.debug("uploading aac preview to #{track.preview_filename('aac')}")
        public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest)


        track.skip_uploader = true

        original_aac_preview_url = track["preview_aac_url"]

        # and finally update the JamTrackTrack with the new info
        track["preview_aac_url"] = track.preview_filename(aac_md5, 'aac')
        track["preview_aac_length"] = File.new(aac_44100).size
        track["preview_start_time"] = 0

        if !track.save
          finish("save_master_preview", track.errors.to_s)
          return false
        end

        # if all that worked, now delete old previews, if present
        begin
          public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"]
        rescue
          puts "UNABLE TO CLEANUP OLD PREVIEW URL"
        end
      rescue Exception => e
        finish("sync_master_preview_exception", e.to_s)
        return false
      end


      return true

    end

    def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest)

      begin
        mp3_44100 = File.join(tmp_dir, 'output-preview-44100.mp3')
        convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -ab 192k \"#{mp3_44100}\""
        @@log.debug("converting to mp3 using: " + convert_mp3_cmd)

        convert_output = `#{convert_mp3_cmd}`

        mp3_digest = ::Digest::MD5.file(mp3_44100)

        aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac')
        convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\""
        @@log.debug("converting to aac using: " + convert_aac_cmd)

        convert_output = `#{convert_aac_cmd}`

        aac_digest = ::Digest::MD5.file(aac_44100)


        track["preview_md5"] = ogg_md5 = ogg_digest.hexdigest
        track["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest
        track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest

        # upload 44100 ogg, mp3, aac to public location as well
        @@log.debug("uploading ogg preview to #{track.preview_filename('ogg')}")
        public_jamkazam_s3_manager.upload(track.preview_filename(ogg_digest.hexdigest, 'ogg'), ogg_44100, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest)
        @@log.debug("uploading mp3 preview to #{track.preview_filename('mp3')}")
        public_jamkazam_s3_manager.upload(track.preview_filename(mp3_digest.hexdigest, 'mp3'), mp3_44100, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest)
        @@log.debug("uploading aac preview to #{track.preview_filename('aac')}")
        public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest)


        track.skip_uploader = true

        original_ogg_preview_url = track["preview_url"]
        original_mp3_preview_url = track["preview_mp3_url"]
        original_aac_preview_url = track["preview_aac_url"]

        # and finally update the JamTrackTrack with the new info
        track["preview_url"] = track.preview_filename(ogg_md5, 'ogg')
        track["preview_length"] = File.new(ogg_44100).size
        # and finally update the JamTrackTrack with the new info
        track["preview_mp3_url"] = track.preview_filename(mp3_md5, 'mp3')
        track["preview_mp3_length"] = File.new(mp3_44100).size
        track["preview_aac_url"] = track.preview_filename(aac_md5, 'mp3')
        track["preview_aac_length"] = File.new(aac_44100).size
        track["preview_start_time"] = 0

        if !track.save
          finish("save_master_preview", track.errors.to_s)
          return false
        end

        # if all that worked, now delete old previews, if present
        begin
          public_jamkazam_s3_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != track["preview_url"]
          public_jamkazam_s3_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"]
          public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"]
        rescue
          puts "UNABLE TO CLEANUP OLD PREVIEW URL"
        end
      rescue Exception => e
        finish("sync_master_preview_exception", e.to_s)
        return false
      end


      return true

    end

    def fetch_all_files(s3_path)
      JamTrackImporter::song_storage_manager.list_files(s3_path)
    end

    def fetch_important_files(s3_path)
      files = fetch_all_files(s3_path)
      files.select { |file| file.end_with?('.wav') || file.end_with?('.txt') }
    end

    def synchronize(jam_track, metadata, metalocation, options)

      # metalocation should be audio/original artist/song name/meta.yml

      metadata ||= {}

      parsed_metalocation = parse_metalocation(metalocation)

      return unless parsed_metalocation

      original_artist = parsed_metalocation[1]
      name = parsed_metalocation[2]

      if is_paris_storage?
        bpm = parsed_metalocation[-1]
        bpm.downcase!
        if bpm.end_with?('bpm')
          bpm = bpm[0..-4].to_f
        end
        metadata[:bpm] = bpm
      end

      success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options)

      return unless success

      audio_path = metalocation[0...-"/meta.yml".length]

      synchronized_audio = synchronize_audio(jam_track, metadata, audio_path, options[:skip_audio_upload])

      return unless synchronized_audio

      created_plan = synchronize_recurly(jam_track)
      if created_plan
        finish("success", nil)
      end

      # do a last check on any problems with the jamtrack
      jam_track.sync_onboarding_exceptions

    end

    def synchronize_recurly(jam_track)
      begin
        recurly = RecurlyClient.new
          # no longer create JamTrack plans: VRFS-3028
          # recurly.create_jam_track_plan(jam_track) unless recurly.find_jam_track_plan(jam_track)
      rescue RecurlyClientError => x
        finish('recurly_create_plan', x.errors.to_s)
        return false
      end

      true
    end

    class << self

      attr_accessor :storage_format
      attr_accessor :tency_mapping
      attr_accessor :tency_metadata
      attr_accessor :paris_mapping
      attr_accessor :paris_metadata
      attr_accessor :summaries
      attr_accessor :marks_approved

      def report_summaries
        @@log.debug("SUMMARIES DUMP")
        @@log.debug("--------------")
        @summaries.each do |k, v|

          if k == :no_instrument_detail
            @@log.debug("#{k}: #{v}")
          elsif k == :no_precount_detail
            v.each do |precount_detail|
              @@log.debug("precount: #{precount_detail}")
            end
          elsif k == :unique_artists
            v.each do |artist|
              @@log.debug("artist: #{artist}")
            end
          else
            @@log.debug("#{k}: #{v}")
          end
        end
      end

      def song_storage_manager
        if is_tency_storage?
          tency_s3_manager
        elsif is_paris_storage?
          paris_s3_manager
        elsif is_tim_tracks_storage?
          tim_tracks_s3_manager
        elsif is_drumma_storage?
          drumma_s3_manager
        else
          s3_manager
        end
      end

      def summaries
        @summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}, no_precount_num: 0, no_precount_detail: [], unique_artists: SortedSet.new, multiple_masters: 0, total: 0}
      end

      def drumma_s3_manager
        @drumma_s3_manager ||= S3Manager.new('jamkazam-drumma', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def tency_s3_manager
        @tency_s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def paris_s3_manager
        @paris_s3_manager ||= S3Manager.new('jamkazam-paris', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def tim_tracks_s3_manager
        @tim_tracks_s3_manager ||= S3Manager.new('jamkazam-timtracks', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def s3_manager
        @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def private_s3_manager
        @private_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key)
      end

      def extract_paris_song_id(metalocation)

        first_path = metalocation.index('/')
        return nil unless first_path
        metalocation = metalocation[(first_path + 1)..-1]

        suffix = '/meta.yml'
        metalocation = metalocation[0...-suffix.length]

        first_dash = metalocation.index('-')
        return nil if first_dash.nil?

        id = metalocation[0...first_dash].strip

        return nil unless id.start_with?('S') # all start with S
        return nil if id[1..-1].to_i == 0 # and number after that
        id
      end

      def extract_tency_song_id(metalocation)
        # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml

        first_path = metalocation.index('/')
        return nil unless first_path
        metalocation = metalocation[(first_path + 1)..-1]

        suffix = '/meta.yml'
        metalocation = metalocation[0...-suffix.length]

        last_dash = metalocation.rindex('-')
        return nil if last_dash.nil?

        id = metalocation[(last_dash+1)..-1].strip

        return nil if id.to_i == 0

        id
      end

      def is_default_storage?
        assert_storage_set
        @storage_format == 'default'
      end

      def is_drumma_storage?
        assert_storage_set
        @storage_format == 'Drumma'
      end

      def is_tency_storage?
        assert_storage_set
        @storage_format == 'Tency'
      end

      def is_paris_storage?
        assert_storage_set
        @storage_format == 'Paris'
      end

      def is_tim_tracks_storage?
        assert_storage_set
        @storage_format == 'TimTracks'
      end

      def assert_storage_set
        raise "no storage_format set" if @storage_format.nil?
      end

      def iterate_tim_tracks_song_storage(&blk)
        count = 0
        song_storage_manager.list_directories('mapped').each do |song|
          @@log.debug("searching through song directory '#{song}'")

          metalocation = "#{song}meta.yml"

          metadata = load_metalocation(metalocation)

          blk.call(metadata, metalocation)

          count += 1
          #break if count > 100

        end
      end

      def iterate_paris_song_storage(&blk)
        count = 0
        song_storage_manager.list_directories('mapped').each do |song|
          @@log.debug("searching through song directory '#{song}'")

          #next if song != 'mapped/S1555-Ashlee Simpson-L-O-V-E-96bpm/'

          metalocation = "#{song}meta.yml"

          metadata = load_metalocation(metalocation)

          if metadata.nil?
            # we don't do a paris song unless it has metadata
            next
          end
          blk.call(metadata, metalocation)

          count += 1
          #break if count > 1000
        end
      end

      def iterate_tency_song_storage(&blk)
        count = 0
        song_storage_manager.list_directories('mapped').each do |song|
          @@log.debug("searching through song directory '#{song}'")

          metalocation = "#{song}meta.yml"

          metadata = load_metalocation(metalocation)

          blk.call(metadata, metalocation)

          count += 1
          #break if count > 100

        end
      end

      def iterate_default_song_storage(&blk)
        song_storage_manager.list_directories('audio').each do |original_artist|
          @@log.debug("searching through artist directory '#{original_artist}'")

          songs = song_storage_manager.list_directories(original_artist)
          songs.each do |song|
            @@log.debug("searching through song directory' #{song}'")

            metalocation = "#{song}meta.yml"

            metadata = load_metalocation(metalocation)

            blk.call(metadata, metalocation)
          end
        end
      end

      def iterate_drumma_song_storage(&blk)
        song_storage_manager.list_directories.each do |song|
          @@log.debug("searching through song directory '#{song}'")

          metalocation = "#{song}meta.yml"

          metadata = load_metalocation(metalocation)

          blk.call(metadata, metalocation)

        end
      end

      def iterate_song_storage(&blk)
        if is_tency_storage?
          iterate_tency_song_storage do |metadata, metalocation|
            blk.call(metadata, metalocation)
          end
        elsif is_paris_storage?
          iterate_paris_song_storage do |metadata, metalocation|
            blk.call(metadata, metalocation)
          end
        elsif is_tim_tracks_storage?
          iterate_tim_tracks_song_storage do |metadata, metalocation|
            blk.call(metadata, metalocation)
          end
        elsif is_drumma_storage?
          iterate_drumma_song_storage do |metadata, metalocation|
            blk.call(metadata, metalocation)
          end
        else
          iterate_default_song_storage do |metadata, metalocation|
            blk.call(metadata, metalocation)
          end
        end
      end

      def dry_run
        iterate_song_storage do |metadata, metalocation|
          jam_track_importer = JamTrackImporter.new(@storage_format)
          jam_track_importer.metadata = metadata

          JamTrackImporter.summaries[:total] += 1

          jam_track_importer.dry_run(metadata, metalocation)
        end

        report_summaries
      end

      # figure out which songs are in S3 that do not exist in the 2k spreadsheet (mapping.csv), and which songs are in the 2k spreadsheet that are not in S3
      def tency_delta
        in_s3 = {}
        in_mapping = {}

        load_tency_mappings

        JamTrackImporter.tency_metadata.each do |song_id, metadata|
          in_mapping[song_id] = {artist: metadata[:original_artist], song: metadata[:name]}
        end

        iterate_song_storage do |metadata, metalocation|

          importer = JamTrackImporter.new(@storage_format)
          importer.metadata = metadata
          song_id = JamTrackImporter.extract_tency_song_id(metalocation)
          parsed_metalocation = importer.parse_metalocation(metalocation)

          next if song_id.nil?
          next if parsed_metalocation.nil?

          original_artist = parsed_metalocation[1]
          meta_name = parsed_metalocation[2]

          in_s3[song_id] = {artist: original_artist, song: meta_name}
        end

        in_s3_keys = Set.new(in_s3.keys)
        in_mapping_keys = Set.new(in_mapping.keys)
        only_in_mapping = in_mapping_keys - in_s3_keys
        only_in_s3 = in_s3_keys - in_mapping_keys

        CSV.open("only_in_s3.csv", "wb") do |csv|
          only_in_s3.each do |song_id|
            csv << [song_id, in_s3[song_id][:artist], in_s3[song_id][:song]]
          end
        end

        CSV.open("only_in_2k_selection.csv", "wb") do |csv|
          only_in_mapping.each do |song_id|
            csv << [song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song]]
          end
        end

      end

      def add_tency_metadata
        JamTrackLicensor.find_by_name('Tency Music').jam_tracks.each do |jam_track|
          jam_track_importer = JamTrackImporter.new(@storage_format)
          jam_track_importer.add_licensor_metadata('Tency Music', jam_track.metalocation)
          break
        end
      end

      def create_masters
        iterate_song_storage do |metadata, metalocation|
          next if metadata.nil?
          jam_track_importer = JamTrackImporter.new(@storage_format)
          jam_track_importer.metadata = metadata
          jam_track_importer.create_master(metadata, metalocation)
        end
      end

      def create_master(path)
        metalocation = "#{path}/meta.yml"

        metadata = load_metalocation(metalocation)

        jam_track_importer = JamTrackImporter.new(@storage_format)

        jam_track_importer.create_master(metadata, metalocation)
      end

      def dry_run_original
        s3_manager.list_directories('audio').each do |original_artist|
          @@log.debug("searching through artist directory '#{original_artist}'")

          songs = s3_manager.list_directories(original_artist)
          songs.each do |song|
            @@log.debug("searching through song directory' #{song}'")

            metalocation = "#{song}meta.yml"

            metadata = load_metalocation(metalocation)

            jam_track_importer = JamTrackImporter.new
            jam_track_importer.metadata = metadata

            jam_track_importer.dry_run(metadata, metalocation)
          end
        end

      end

      def synchronize_preview(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        error_occurred = false
        error_msg = nil
        jam_track.jam_track_tracks.each do |track|
          next if track.track_type == 'Master'

          if track.preview_start_time
            track.generate_preview
            if track.preview_generate_error
              error_occurred = true
              error_msg = track.preview_generate_error
            else
            end
          end
        end

        if error_occurred
          importer.finish('preview_error', error_msg)
        else
          importer.finish('success', nil)
        end
        importer
      end

      # hunts for the most recent .aac, .mp3, or .ogg file
      def synchronize_preview_dev(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        importer.synchronize_preview_dev(jam_track)

        importer.finish('success', nil)
        importer
      end

      def synchronize_jamtrack_aac_preview(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        track = jam_track.master_track

        if track
          Dir.mktmpdir do |tmp_dir|
            ogg_44100 = File.join(tmp_dir, 'input.ogg')
            private_s3_manager.download(track.url_by_sample_rate(44), ogg_44100)
            ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
            if importer.synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_44100_digest)
              importer.finish("success", nil)
            end
          end
        else
          importer.finish('no_master_track', nil)
        end
        importer
      end

      def synchronize_jamtrack_master_preview(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        master_track = jam_track.master_track

        if master_track
          Dir.mktmpdir do |tmp_dir|
            ogg_44100 = File.join(tmp_dir, 'input.ogg')
            private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
            ogg_44100_digest = ::Digest::MD5.file(ogg_44100)
            if importer.synchronize_master_preview(master_track, tmp_dir, ogg_44100, ogg_44100_digest)
              importer.finish("success", nil)
            end
          end
        else
          importer.finish('no_master_track', nil)
        end

        importer
      end

      def synchronize_previews_dev
        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_preview_dev(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "no_preview_start_time"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def import_click_track(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name
        importer.import_click_track(jam_track)

        importer
      end

      def generate_jmep(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name
        importer.generate_jmep(jam_track)

        importer
      end

      def import_click_tracks
        importers = []

        JamTrack.all.each do |jam_track|
          #jam_track = JamTrack.find('126')
          importers << import_click_track(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to generate jmep.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def generate_jmeps
        importers = []
        if is_tency_storage?
          licensor = JamTrackLicensor.find_by_name!('Tency Music')
        elsif is_paris_storage?
          licensor = JamTrackLicensor.find_by_name!('Paris Music')
        end

        JamTrack.where(licensor_id: licensor).each do |jam_track|
          importers << generate_jmep(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to generate jmep.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end


      def synchronize_previews
        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_preview(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "no_preview_start_time"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def synchronize_jamtrack_aac_previews

        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_jamtrack_aac_preview(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end


        end
      end

      def synchronize_jamtrack_master_previews
        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_jamtrack_master_preview(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def synchronize_duration(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        master_track = jam_track.master_track
        if master_track
          Dir.mktmpdir do |tmp_dir|
            ogg_44100 = File.join(tmp_dir, 'input.ogg')
            private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)

            if importer.synchronize_duration(jam_track, ogg_44100)
              jam_track.save!
              importer.finish("success", nil)
            end
          end
        else
          importer.finish('no_duration', nil)
        end

        importer
      end

      def synchronize_durations
        importers = []

        JamTrack.all.each do |jam_track|
          importers << synchronize_duration(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def download_master(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        Dir.mkdir('tmp') unless Dir.exists?('tmp')
        Dir.mkdir('tmp/jam_track_masters') unless Dir.exists?('tmp/jam_track_masters')

        master_track = jam_track.master_track
        if master_track
          ogg_44100 = File.join('tmp/jam_track_masters', "#{jam_track.original_artist} - #{jam_track.name}.ogg")
          private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100)
        end
        importer
      end

      def generate_mp3_aac_stem(jam_track)
        importer = JamTrackImporter.new
        importer.name = jam_track.name

        Dir.mktmpdir do |tmp_dir|

          audio_path = jam_track.metalocation[0...-"/meta.yml".length]
          importer.associate_tracks_with_original_stems(jam_track, audio_path)
          importer.generate_mp3_aac_stem(jam_track, tmp_dir, false)
        end

        importer
      end

      def download_masters
        importers = []

        JamTrack.all.each do |jam_track|
          importers << download_master(jam_track)
        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to download.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def remove_s3_special_chars(filename)
        filename.tr('/&@:,$=+?;\^`><{}[]#%~|', '')
      end

      def generate_mp3_aac_stems(format)
        importers = []

        jam_tracks = []

        tency = JamTrackLicensor.find_by_name('Tency Music')

        @@log.info("processing storage #{@storage_format}")
        if is_tency_storage?
          tency = JamTrackLicensor.find_by_name!('Tency Music')
          jam_tracks = JamTrack.where(licensor_id: tency.id)
        elsif is_paris_storage?
          paris = JamTrackLicensor.find_by_name!('Paris Music')
          jam_tracks = JamTrack.where(licensor_id: paris.id)
        elsif is_default_storage?
          # XXX IF WE ADD ANOTHER STORAGE, UPDATE THE WHERE TO EXCLUDE IT AS WELL
          jam_tracks = JamTrack.where('licensor_id is null OR licensor_id != ?', tency.id)
        else
          raise 'unknown storage format!'
        end

        jam_tracks.each do |jam_track|

          if ENV['NODE_COUNT']
            node_count = ENV['NODE_COUNT'].to_i
            node_number = ENV['NODE_NUMBER'].to_i
            raise "NO NODE_COUNT" if node_count == 0

            jam_track_id = jam_track.id.to_i
            jam_track_id = jam_track_id + node_number
            if jam_track_id == 0
              @@log.warn("skipping #{jam_track_id} because non-numeric ID")
              next
            elsif jam_track_id % node_count == 0
              @@log.warn("starting JamTrack #{jam_track.id} (#{jam_track_id})")
              importers << generate_mp3_aac_stem(jam_track)
            else
              @@log.warn("skipping #{jam_track_id}")
              next
            end
          else
            importers << generate_mp3_aac_stem(jam_track)
          end

        end

        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to download.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def generate_slugs
        JamTrack.all.each do |jam_track|
          jam_track.generate_slug
          jam_track.save!
        end
      end

      def onboarding_exceptions
        JamTrack.all.each do |jam_track|
          jam_track.onboarding_exceptions
        end
      end

      def synchronize_all(options)
        importers = []

        count = 0
        iterate_song_storage do |metadata, metalocation|

          next if metadata.nil? && (is_tency_storage? || is_paris_storage?)

          importer = synchronize_from_meta(metalocation, options)
          importers << importer

          if importer.reason != 'jam_track_exists' && importer.reason != "other_processing"
            count+=1
          end

          if count > 500
            #break
          end
        end


        @@log.info("SUMMARY")
        @@log.info("-------")
        importers.each do |importer|
          if importer
            if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing"
              @@log.info("#{importer.name} #{importer.reason}")
            else
              @@log.error("#{importer.name} failed to import.")
              @@log.error("#{importer.name} reason=#{importer.reason}")
              @@log.error("#{importer.name} detail=#{importer.detail}")
            end
          else
            @@log.error("NULL IMPORTER")
          end

        end
      end

      def jam_track_dry_run(metalocation)
        # see if we can find a JamTrack with this metalocation
        jam_track = JamTrack.find_by_metalocation(metalocation)

        meta = load_metalocation(metalocation)

        if jam_track
          @@log.debug("jamtrack #{jam_track.name} located by metalocation")
          jam_track.dry_run(meta, metalocation)
        else
          jam_track = JamTrack.new
          jam_track.dry_run(meta, metalocation)
        end
      end

      def paris_genre_dump
        load_paris_mappings

        genres = {}
        @paris_metadata.each do |id, value|
          genre1 = value[:genre1]
          genre2 = value[:genre2]
          genre3 = value[:genre3]

          genres[genre1.downcase.strip] = genre1.downcase.strip if genre1
          genres[genre2.downcase.strip] = genre2.downcase.strip if genre2
          genres[genre3.downcase.strip] = genre3.downcase.strip if genre3
        end

        all_genres = Genre.select(:id).all.map(&:id)

        all_genres = Set.new(all_genres)
        genres.each do |genre, value|
          found = all_genres.include? genre

          puts "#{genre}" unless found
        end
      end


      def create_importer_from_existing(jam_track)
        importer = JamTrackImporter.new(@storage_format)
        importer.name = jam_track.name
        importer.metadata = load_metalocation(jam_track.metalocation)
        importer
      end

      def resync_instruments(licensor)

        load_paris_mappings if @paris_mapping.nil?

        JamTrack.where(licensor_id: licensor.id).each do |jam_track|

          if @paris_metadata[jam_track.vendor_id].nil?
            next
          end
          puts "RESYNCING JAMTRACK #{jam_track.id}"
          JamTrackTrack.where(jam_track_id: jam_track.id).order(:position).each do |track|
            puts "BEFORE TRACK #{track.instrument_id} #{track.part}"
          end

          importer = create_importer_from_existing(jam_track)
          importer.reassign_instrument_parts(jam_track)

          #puts ">>>>>>>>> HIT KEY TO CONTINUE <<<<<<<<<<"
          #STDIN.gets
        end

      end


      def fix_artist_song_name (licensor)

        load_paris_mappings if @paris_mapping.nil?

        JamTrack.where(licensor_id: licensor.id).each do |jam_track|

          metadata = @paris_metadata[jam_track.vendor_id]

          if metadata.nil?
            puts "OH NO!  A Paris Song that does not belong!  #{jam_track.id} #{jam_track.vendor_id}"
            next
          end

          puts "STARTING JAM_TRACK #{jam_track.id} #{jam_track.original_artist} #{jam_track.name}"

          jam_track.generate_slug

          if jam_track.changed?
            puts "SLUG CHANGED! #{jam_track.changes.inspect}"
          end

          if !jam_track.save
            puts "dup slug!!!!: #{jam_track.id} #{jam_track.name} #{jam_track.original_artist}"
          end

          if jam_track.changed?
            jam_track.reload
          end

          jam_track.original_artist = metadata[:original_artist]
          jam_track.name = metadata[:name]
          if jam_track.changed?
            puts "ARTIST/NAME CHANGE: #{jam_track.changes.inspect}"
          end

          if !jam_track.save
            puts "unable to save new artist/song!"
          end
        end
      end

      def fix_slugs(licensor)

        JamTrack.where(licensor_id: licensor.id).each do |jam_track|

          if jam_track.slug.end_with?('-')
            puts "removing trailing dash"
            jam_track.slug = jam_track.slug[0...-1]
            if !jam_track.save
              puts "dup slug!!!!: #{jam_track.id} #{jam_track.name} #{jam_track.original_artist}"
            end
          end
        end
      end

      def missing_masters(licensor)

        count = 0
        JamTrack.where(licensor_id: licensor.id).each do |jam_track|


          if jam_track.master_track.nil?
            puts "MISSING #{jam_track.metalocation}"
            count += 1
          end

        end

        puts "missing master count: #{count}"
      end

      def tency_genre_dump
        load_tency_mappings

        genres = {}
        @tency_metadata.each do |id, value|

          genre1 = value[:genre1]
          genre2 = value[:genre2]
          genre3 = value[:genre3]
          genre4 = value[:genre4]
          genre5 = value[:genre5]

          genres[genre1.downcase.strip] = genre1.downcase.strip if genre1
          genres[genre2.downcase.strip] = genre2.downcase.strip if genre2
          genres[genre3.downcase.strip] = genre3.downcase.strip if genre3
          genres[genre4.downcase.strip] = genre4.downcase.strip if genre4
          genres[genre5.downcase.strip] = genre5.downcase.strip if genre5
        end

        all_genres = Genre.select(:id).all.map(&:id)

        all_genres = Set.new(all_genres)
        genres.each do |genre, value|
          found = all_genres.include? genre

          puts "#{genre}" unless found
        end
      end

      def load_marks_approved
        @marks_approved = {}
        Dir.mktmpdir do |tmp_dir|
          mapping_file = 'marks_approved.csv'
          mapping_csv = CSV.read(mapping_file, headers: true, return_headers: false)

          mapping_csv.each do |line|
            approved = line[3]
            track_url = line[2]
            comments = line[7]

            approved.strip! if approved
            track_url.strip! if track_url
            comments.strip! if comments

            if approved == 'MJ'
              prefix = 'https://www.jamkazam.com/landing/jamtracks/'.length
              slug = track_url[prefix..-1]

              puts "MARKS APPROVED #{slug} #{comments}"
              @marks_approved[slug] = comments
            end
          end
        end
      end

      def load_paris_mappings
        Dir.mktmpdir do |tmp_dir|
          mapping_file = File.join(tmp_dir, 'mapping.csv')
          metadata_file = File.join(tmp_dir, 'metadata.csv')

          # this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv
          if ENV['PARIS_ALREADY_DOWNLOADED'] == '1'
            mapping_file = 'paris_mapping.csv'
            metadata_file = 'paris_metadata.csv'
          else
            paris_s3_manager.download('mapping/mapping.csv', mapping_file)
            paris_s3_manager.download('mapping/metadata.csv', metadata_file)
          end

          mapping_csv = CSV.read(mapping_file)
          metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false)

          @paris_mapping = {}
          @paris_metadata = {}
          # convert both to hashes
          mapping_csv.each do |line|
            instrument = line[1]
            instrument.strip! if instrument

            part = line[2]
            part.strip! if part
            @paris_mapping[line[0].strip.downcase] = {instrument: instrument, part: part}
          end

          metadata_csv.each do |line|
            paris_artist = line[2]
            # Paris artist in metadata file is often all caps
            artist = paris_artist.split(' ').collect do |item|
              if item == 'DJ'
                'DJ'
              else
                item.titleize
              end
            end.join(' ')
            @paris_metadata[line[1].strip] = {id: line[1].strip, original_artist: artist, name: line[3], genre1: line[4], genre2: line[5], genre3: line[6]}
          end

          @paris_metadata.each do |id, value|

            genres = []

            genre1 = value[:genre1]
            genre2 = value[:genre2]
            genre3 = value[:genre3]

            genres << genre1.downcase.strip if genre1
            genres << genre2.downcase.strip if genre2
            genres << genre3.downcase.strip if genre3

            value[:genres] = genres
          end
        end
      end

      def load_tency_mappings
        Dir.mktmpdir do |tmp_dir|
          mapping_file = File.join(tmp_dir, 'mapping.csv')
          metadata_file = File.join(tmp_dir, 'metadata.csv')

          # this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv
          if ENV['TENCY_ALREADY_DOWNLOADED'] == '1'
            mapping_file = 'mapping.csv'
            metadata_file = 'metadata.csv'
          else
            tency_s3_manager.download('mapping/mapping.csv', mapping_file)
            tency_s3_manager.download('mapping/metadata.csv', metadata_file)
          end

          mapping_csv = CSV.read(mapping_file)
          metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false)

          @tency_mapping = {}
          @tency_metadata = {}
          # convert both to hashes
          mapping_csv.each do |line|
            @tency_mapping[line[0].strip] = {instrument: line[1], part: line[2], count: line[3], trust: line[4]}
          end

          metadata_csv.each do |line|
            @tency_metadata[line[0].strip] = {id: line[0].strip, original_artist: line[1], name: line[2], additional_info: line[3], year: line[4], language: line[5], isrc: line[10], genre1: line[11], genre2: line[12], genre3: line[13], genre4: line[14], genre5: line[15]}
          end


          @tency_metadata.each do |id, value|

            genres = []

            genre1 = value[:genre1]
            genre2 = value[:genre2]
            genre3 = value[:genre3]
            genre4 = value[:genre4]
            genre5 = value[:genre5]

            genres << genre1.downcase.strip if genre1
            genres << genre2.downcase.strip if genre2
            genres << genre3.downcase.strip if genre3
            genres << genre4.downcase.strip if genre4
            genres << genre5.downcase.strip if genre5

            value[:genres] = genres
          end


        end
      end

      def load_metalocation(metalocation)

        if is_tency_storage?
          load_tency_mappings if @tency_mapping.nil?
          song_id = extract_tency_song_id(metalocation)

          if song_id.nil?
            puts "missing_song_id #{metalocation}"
            return nil
          end


          tency_data = @tency_metadata[song_id]

          if tency_data.nil?
            @@log.warn("missing tency metadata '#{song_id}'")
          end

          return tency_data
        elsif is_paris_storage?
          load_paris_mappings if @paris_mapping.nil?
          song_id = extract_paris_song_id(metalocation)

          if song_id.nil?
            puts "missing_song_id #{metalocation}"
            return nil
          end

          paris_data = @paris_metadata[song_id]

          if paris_data.nil?
            @@log.warn("missing paris metadata '#{song_id}'")
          end

          return paris_data
        elsif is_drumma_storage?

          data = {}
          begin
            data = drumma_s3_manager.read_all(metalocation)
          rescue AWS::S3::Errors::NoSuchKey
            return {}
          end
          meta = YAML.load(data)
          meta[:genres] = ['r&b'] if !meta[:genres]
          meta
        else
          begin
            data = s3_manager.read_all(metalocation)
            meta = YAML.load(data)

            if is_tim_tracks_storage?
              meta[:genres] = ['acapella']
            end
            meta
          rescue AWS::S3::Errors::NoSuchKey
            return nil
          end
        end
      end

      def create_from_metalocation(meta, metalocation, options = {skip_audio_upload: false})
        jam_track = JamTrack.new
        sync_from_metadata(jam_track, meta, metalocation, options)
      end

      def update_from_metalocation(jam_track, meta, metalocation, options)
        sync_from_metadata(jam_track, meta, metalocation, options)
      end

      def sync_from_metadata(jam_track, meta, metalocation, options)
        jam_track_importer = JamTrackImporter.new(@storage_format)
        jam_track_importer.metadata = meta

        JamTrack.connection.execute('SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED')

        JamTrack.transaction do

          #begin
          jam_track_importer.synchronize(jam_track, meta, metalocation, options)
          #rescue Exception => e
          #  jam_track_importer.finish("unhandled_exception", e.to_s)
          #end

          if jam_track_importer.reason != "success"
            raise ActiveRecord::Rollback
          end
        end

        jam_track_importer
      end

      def synchronize_from_meta(metalocation, options)
        # see if we can find a JamTrack with this metalocation
        jam_track = JamTrack.find_by_metalocation(metalocation)

        meta = load_metalocation(metalocation)

        if meta.nil? && is_paris_storage?
          raise "no tency song matching this metalocation #{metalocation}"
        end
        jam_track_importer = nil
        if jam_track
          @@log.debug("jamtrack #{jam_track.name} located by metalocation")
          jam_track_importer = update_from_metalocation(jam_track, meta, metalocation, options)
        else
          jam_track_importer = create_from_metalocation(meta, metalocation, options)
        end

        if jam_track_importer.reason == "success"
          @@log.info("#{jam_track_importer.name} successfully imported")
        else
          @@log.error("#{jam_track_importer.name} failed to import.")
          @@log.error("#{jam_track_importer.name} reason=#{jam_track_importer.reason}")
          @@log.error("#{jam_track_importer.name} detail=#{jam_track_importer.detail}")
        end

        jam_track_importer
      end
    end
  end
end