require 'json' require 'tempfile' require 'open3' require 'fileutils' require 'open-uri' require 'yaml' module JamRuby class JamTrackImporter @@log = Logging.logger[JamTrackImporter] attr_accessor :name attr_accessor :metadata attr_accessor :reason attr_accessor :detail attr_accessor :storage_format def jamkazam_s3_manager @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def public_jamkazam_s3_manager @public_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_public, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def initialize(storage_format = 'default') @storage_format = storage_format end def finish(reason, detail) @@log.info("JamTrackImporter:#{self.name} #{reason} #{detail}") self.reason = reason self.detail = detail if ENV['END_ON_FAIL'] == "1" && reason != 'success' && reason != 'jam_track_exists' raise "#{reason} #{detail}" end end def import_click_track(jam_track) # we need to download the click track, if it exists. Dir.mktmpdir do |tmp_dir| @@log.info("importing clicking track for #{jam_track.original_artist}:#{jam_track.name}") if jam_track.click_track @@log.info("already has click track: #{jam_track.original_artist}:#{jam_track.name}") finish('success', 'already_has_click_track') return end click_track_file = jam_track.click_track_file if click_track_file.nil? @@log.info("no click track for #{jam_track.original_artist}:#{jam_track.name}") finish('success', 'no_click_track') return end original_filename = click_track_file[:original_filename] if original_filename.nil? @@log.info("no click track s3 path for #{jam_track.original_artist}:#{jam_track.name}") finish('no_original_source', 'click track is missing s3 path:' + click_track_file.id) return end #wav_file = File.join(tmp_dir, File.basename(click_track_file[:original_filename])) #JamTrackImporter.song_storage_manager.download(click_track_file[:original_filename], wav_file) JamTrack.transaction do click_track = jam_track.click_track if click_track.nil? click_track = JamTrackTrack.new click_track.original_filename = click_track_file[:original_filename] click_track.original_audio_s3_path = click_track_file[:original_filename] click_track.track_type = 'Click' click_track.part = 'Clicktrack' click_track.instrument_id = 'computer' click_track.jam_track = jam_track click_track.position = 10000 if !click_track.save @@log.error("unable to create jamtrack click track #{click_track.errors.inspect}") finish("jam_track_click", "unable to create: #{click_track.errors.inspect}") return false end end jam_track.increment_version! # with the click track in hand, flesh out the details synchronize_audio_track(jam_track, tmp_dir, false, click_track) finish('success', nil) end end end def generate_jmep(jam_track) # https://docs.google.com/spreadsheets/d/1dyUOjWkeU8BXwnJl-ws1Kvxq_twWEG7E78F29haYkLc/edit#gid=987457683 # cross-check against marks_approved if JamTrackImporter.marks_approved && JamTrackImporter.marks_approved.has_key?(jam_track.slug) @@log.info("Found track in mark approved list. skipping") finish('success', 'mark@jamkazam.com created') return end # can we overwrite this one? if jam_track.jmep_text.blank? || jam_track.jmep_text.include?('created via code') @@log.info("This is a blank jmep or created earlier by code.") else @@log.info("This a JMEP that was not created by code. Skip it.") return end # we need to download the click track, if it exists. Dir.mktmpdir do |tmp_dir| master_track = jam_track.master_track click_track = jam_track.click_track_file if master_track.nil? finish('no_master_track', nil) return end master_track_file = File.join(tmp_dir, File.basename(master_track[:url_48])) begin JamTrackImporter.private_s3_manager.download(master_track.url_by_sample_rate(44), master_track_file) rescue Exception => e @@log.error("unable to download master track") finish("no-download-master", master_track.url_by_sample_rate(44)) return end if click_track click_track_file = File.join(tmp_dir, File.basename(click_track[:original_filename])) JamTrackImporter.song_storage_manager.download(click_track[:original_filename], click_track_file) else # we'll use the master for click analysis. not ideal, but would work click_track_file = master_track_file end if click_track start_time = determine_start_time(click_track_file, tmp_dir, click_track[:original_filename], false) else start_time = determine_start_time(master_track_file, tmp_dir, master_track[:url], false) end trimmed_for_beat_analysis = File.join(tmp_dir, 'trimmed_for_beat.wav') # trim out the 1st 5 second after non-silence trim_cmd = "sox #{Shellwords.escape(click_track_file)} #{Shellwords.escape(trimmed_for_beat_analysis)} trim #{start_time} #{start_time + 5}" cmd = "bash -c #{Shellwords.escape(trim_cmd)}" @@log.debug("executing cmd #{cmd}") output=`#{cmd}` result_code = $?.to_i if result_code != 0 finish("trim-fail", "failed to run trim click track: #{output}") return end # bpm comes from git clone http://www.pogo.org.uk/~mark/bpm-tools.git sox="sox #{Shellwords.escape(trimmed_for_beat_analysis)} -t raw -r 44100 -e float -c 1 - | bpm -m 25 -x 250" cmd = "bash -c #{Shellwords.escape(sox)}" @@log.debug("executing cmd #{cmd}") output=`#{cmd}` result_code = $?.to_i if result_code == 0 bpm = output.to_f offset = 0.140 if bpm >= 60 && bpm < 80 offset = 0.110 elsif bpm >= 80 && bpm < 100 offset = 0.080 elsif bpm >= 100 && bpm < 120 offset = 0.050 elsif bpm >= 120 offset = 0.020 end @@log.debug("bpm: #{bpm} start_time: #{start_time}, offset: #{offset}") start_time += offset metro_fin = "#{Time.at(start_time).utc.strftime("%H:%M:%S")}:#{((start_time - start_time.to_i) * 1000).round.to_s.rjust(3, "0")}" jmep = "" jmep << "# created via code using bpm/silence detection (bpm:#{bpm} offset:#{offset})\r\n" jmep << "prelude@10.0 #number of seconds before music starts\r\n" jmep << "metro_fin@#{metro_fin} bpm=#{bpm}, ticks=8, pmode=stream, name=Beep, play=mono" @@log.info("jmep generated: #{jmep}") jam_track.jmep_text = jmep if jam_track.save finish('success', nil) else @@log.error("jamtrack did not save. #{jam_track.errors.inspect}") finish("no-save", "jamtrack did not save. #{jam_track.errors.inspect}") return end else finish("bpm-fail", "failed to run bpm: #{output}") return end end end def determine_start_time(audio_file, tmp_dir, original_filename, protect_short = true) burp_gaps = ['0.3', '0.2', '0.1', '0.05', '0.025'] out_wav = File.join(tmp_dir, 'stripped.wav') total_time_command = "soxi -D \"#{audio_file}\"" total_time = `#{total_time_command}`.to_f result_code = -20 stripped_time = total_time # default to the case where we just start the preview at the beginning burp_gaps.each do |gap| command_strip_lead_silence = "sox \"#{audio_file}\" \"#{out_wav}\" silence 1 #{gap} 1%" @@log.debug("stripping silence: " + command_strip_lead_silence) output = `#{command_strip_lead_silence}` result_code = $?.to_i if result_code == 0 stripped_time_command = "soxi -D \"#{out_wav}\"" stripped_time_test = `#{stripped_time_command}`.to_f if stripped_time_test < 1 # meaning a very short duration @@log.warn("could not determine the start of non-silence. assuming beginning") stripped_time = total_time # default to the case where we just start the preview at the beginning else stripped_time = stripped_time_test # accept the measured time of the stripped file and move on by using break break end else @@log.warn("unable to determine silence for jam_track #{original_filename}, #{output}") stripped_time = total_time # default to the case where we just start the preview at the beginning end end preview_start_time = total_time - stripped_time preview_start_time end def synchronize_preview_dev(jam_track) jam_track.jam_track_tracks.each do |track| next if track.track_type != 'Master' most_recent_aac = nil most_recent_ogg = nil most_recent_mp3 = nil public_jamkazam_s3_manager.list_files(track.preview_directory).each do |s3_preview_item| s3_object = public_jamkazam_s3_manager.object(s3_preview_item) if s3_preview_item.end_with?('.aac') if most_recent_aac if s3_object.last_modified > most_recent_aac.last_modified most_recent_aac = s3_object end else most_recent_aac = s3_object end end if s3_preview_item.end_with?('.mp3') if most_recent_mp3 if s3_object.last_modified > most_recent_mp3.last_modified most_recent_mp3 = s3_object end else most_recent_mp3 = s3_object end end if s3_preview_item.end_with?('.ogg') if most_recent_ogg if s3_object.last_modified > most_recent_ogg.last_modified most_recent_ogg = s3_object end else most_recent_ogg = s3_object end end end if most_recent_aac track['preview_aac_md5'] = 'md5' track['preview_aac_url'] = most_recent_aac.key track['preview_aac_length'] = most_recent_aac.content_length end if most_recent_mp3 track['preview_mp3_md5'] = 'md5' track['preview_mp3_url'] = most_recent_mp3.key track['preview_mp3_length'] = most_recent_mp3.content_length end if most_recent_ogg track['preview_md5'] = 'md5' track['preview_url'] = most_recent_ogg.key track['preview_length'] = most_recent_ogg.content_length end track.save end end def create_silence(tmp_dir, segment_count, duration, sample_rate, channels = 2) file = File.join(tmp_dir, "#{segment_count}.wav") # -c 2 means stereo cmd("sox -n -r #{sample_rate} -c #{channels} #{file} trim 0.0 #{duration}", "silence") file end # this method was created due to Tency-sourced data having no master track # it goes through all audio tracks, and creates a master mix from it. (mix + normalize) def create_master(metadata, metalocation) parsed_metalocation = parse_metalocation(metalocation) if parsed_metalocation.nil? finish("invalid_metalocation", metalocation) return end original_artist = parsed_metalocation[1] meta_name = parsed_metalocation[2] self.name = metadata[:name] || meta_name audio_path = metalocation[0...-"/meta.yml".length] all_files = fetch_important_files(audio_path) audio_files = [] master_found = false all_files.each do |file| parsed_wav = parse_file(file) if parsed_wav[:master] master_found = true elsif parsed_wav[:type] == :track audio_files << file end end if master_found @@log.debug("master exists... skipping #{self.name} ") finish('success', nil) return else tracks = [] #tmp_dir = Dir.mktmpdir #tmp_dir = "/var/folders/05/1jpzfcln1hq9p666whnd7chr0000gn/T/d20150809-9945-1ykr85u" Dir.mktmpdir do |tmp_dir| @@log.debug("downloading all audio files in #{tmp_dir}") audio_files.each do |s3_track| track = File.join(tmp_dir, File.basename(s3_track)) tracks << track JamTrackImporter.song_storage_manager.download(s3_track, track) end # first have to check if all are the same sample rate. If not, we have to make it so first_sample_rate = nil normalize_needed = false tracks.each do |track| sample_rate = `soxi -r "#{track}"`.strip if first_sample_rate.nil? first_sample_rate = sample_rate else if first_sample_rate != sample_rate # we need to normalize all of them normalize_needed = true break end end end normalized_tracks = [] if normalize_needed tracks.each do |track| normalized_track = File.join(tmp_dir, 'normalized-' + File.basename(track)) output = `sox "#{track}" "#{normalized_track}" rate #{first_sample_rate}` @@log.debug("resampling #{normalized_track}; output: #{output}") normalized_tracks << normalized_track end tracks = normalized_tracks end temp_file = File.join(tmp_dir, "temp.wav") output_filename = JamTrackImporter.remove_s3_special_chars("#{self.name} Master Mix.wav") output_file = File.join(tmp_dir, output_filename) command = "sox -m " tracks.each do |track| command << " \"#{track}\"" end command << " \"#{temp_file}\"" @@log.debug("mixing with cmd: " + command) sox_output = `#{command}` result_code = $?.to_i if result_code != 0 @@log.error("unable to generate master mix") finish("sox_master_mix_failure", sox_output) else # now normalize the audio command = "sox --norm \"#{temp_file}\" \"#{output_file}\"" @@log.debug("normalizing with cmd: " + command) sox_output = `#{command}` result_code = $?.to_i if result_code != 0 @@log.error("unable to normalize master mix") finish("sox_master_mix_failure", sox_output) else # now we need to upload the output back up s3_target = audio_path + '/' + output_filename @@log.debug("uploading #{output_file} to #{s3_target}") JamTrackImporter.song_storage_manager.upload(s3_target, output_file) finish('success', nil) end end end end end def dry_run(metadata, metalocation) # STDIN.gets @@log.debug("dry_run: #{metadata.inspect}") metadata ||= {} parsed_metalocation = parse_metalocation(metalocation) return unless parsed_metalocation original_artist = parsed_metalocation[1] name = parsed_metalocation[2] JamTrackImporter.summaries[:unique_artists] << original_artist success = dry_run_metadata(metadata, original_artist, name) return unless success audio_path = metalocation[0...-"/meta.yml".length] dry_run_audio(metadata, audio_path) finish("success", nil) end def add_licensor_metadata(vendor, metalocation) Dir.mktmpdir do |tmp_dir| @@log.debug("update vendor metadata") meta_yml = File.join(tmp_dir, 'meta.yml') if jamkazam_s3_manager.exists?(metalocation) jamkazam_s3_manager.download(metalocation, meta_yml) meta = YAML.load_file(meta_yml) else meta = {} end meta[:licensor] = vendor File.open(meta_yml, 'w') { |f| f.write meta.to_yaml } jamkazam_s3_manager.upload(metalocation, meta_yml) end end def is_tency_storage? assert_storage_set @storage_format == 'Tency' end def is_helbing_storage? assert_storage_set @storage_format == 'Helbing' end def is_paris_storage? assert_storage_set @storage_format == 'Paris' end def is_tim_tracks_storage? assert_storage_set @storage_format == 'TimTracks' end def is_drumma_storage? assert_storage_set @storage_format == 'Drumma' end def assert_storage_set raise "no storage_format set" if @storage_format.nil? end def parse_metalocation(metalocation) # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml if is_drumma_storage? suffix = '/meta.yml' unless metalocation.end_with? suffix finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end metalocation = metalocation[0...-suffix.length] bits = ['audio'] first_dash = metalocation.index(' - ') if first_dash artist = metalocation[0...(first_dash)].strip bits << artist else finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end song = metalocation[(first_dash+3)..-1].strip bits << song elsif is_tency_storage? || is_tim_tracks_storage? || is_helbing_storage? suffix = '/meta.yml' unless metalocation.end_with? suffix finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end metalocation = metalocation[0...-suffix.length] first_path = metalocation.index('/') if first_path.nil? finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end metalocation = metalocation[(first_path + 1)..-1] bits = ['audio'] # example: Sister Hazel - All For You - 10385 first_dash = metalocation.index(' - ') if first_dash artist = metalocation[0...(first_dash)].strip bits << artist else finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end if is_tim_tracks_storage? song = metalocation[(first_dash+3)..-1].strip bits << song elsif is_helbing_storage? song = metalocation[(first_dash+3)..-1].strip bits << song elsif is_tency_storage? last_dash = metalocation.rindex('-') if last_dash song = metalocation[(first_dash+3)...last_dash].strip bits << song else finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end end bits << 'meta.yml' bits elsif is_paris_storage? suffix = '/meta.yml' unless metalocation.end_with? suffix finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end metalocation = metalocation[0...-suffix.length] first_path = metalocation.index('/') if first_path.nil? finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end metalocation = metalocation[(first_path + 1)..-1] bits = ['audio'] last_slash = metalocation.rindex('/') # example: S4863-Mike Oldfield-Moonlight Shadow-000bpm if last_slash paris_artist_song_id = metalocation[0...last_slash] else paris_artist_song_id = metalocation end bitbits = paris_artist_song_id.split('-') song_id = bitbits[0].strip artist = bitbits[1] song_name = bitbits[2] bpm = bitbits[-1] bits << artist bits << song_name bits << 'meta.yml' bits << song_id bits << bpm bits else bits = metalocation.split('/') if bits.length != 4 finish("invalid_metalocation", "metalocation not valid #{metalocation}") return nil end if bits[0] != "audio" finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}") return nil end if bits[3] != 'meta.yml' finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}") return nil end bits end end def dry_run_metadata(metadata, original_artist, name) self.name = metadata["name"] || name original_artist = metadata["original_artist"] || original_artist description = metadata["description"] @@log.debug("#{self.name} original_artist=#{original_artist}") true end def determine_genres(metadata) genres = [] if metadata[:genres] metadata[:genres].each do |genre| if genre == 'hard/metal' genres << Genre.find('hard rock') genres << Genre.find('metal') elsif genre == 'christmas' genres << Genre.find('holiday') elsif genre == 'alternative' genres << Genre.find('alternative rock') elsif genre == '80s' || genre == "50's" || genre == "60's" || genre == "70's" || genre == "80's" || genre == "90's" || genre == "50/60's" || genre == "00's" || genre == "2010's" # swallow elsif genre == 'love' # swallow elsif genre == 'christian' || genre == 'gospel' genres << Genre.find('religious') elsif genre == 'punk/grunge' genres << Genre.find('punk') elsif genre == 'electro' genres << Genre.find('electronic') elsif genre == 'teen pop' genres << Genre.find('pop') elsif genre == "rock 'n roll" genres << Genre.find('rock') elsif genre == 'zouk/creole' genres << Genre.find('creole') elsif genre == 'world/folk' genres << Genre.find('world') genres << Genre.find('folk') elsif genre == 'french pop' # swallow elsif genre == 'schlager' #swallow elsif genre == 'humour' # swallow elsif genre == 'oriental' genres << Genre.find('asian') elsif genre == 'abba' genres << Genre.find('pop') elsif genre == 'movies tv show' || genre == "movies" genres << Genre.find('tv & movie soundtrack') elsif genre == 'ballad' # swallow elsif genre == "r'n'b" || genre == "pop rnb" genres << Genre.find("r&b") elsif genre == "rock & roll" genres << Genre.find('rock') elsif genre == "dance pop" genres << Genre.find('dance') elsif genre == "soul/motown" || genre == "soul motown" genres << Genre.find('soul') elsif genre == "party" # swallow elsif genre == "reggae/ska" || genre == "reggae ska" genres << Genre.find('reggae') genres << Genre.find('ska') elsif genre == "pop rock" || genre == "pop/rock" genres << Genre.find("rock") genres << Genre.find("pop") elsif genre == "singalong" #swallow elsif genre == "folk rock" genres << Genre.find('folk') genres << Genre.find('rock') elsif genre == "swing" || genre == "swing/big band" || genre == "swing big band" genres << Genre.find('oldies') elsif genre == "rap/hip hop" || genre == "rap hip hop" genres << Genre.find("rap") elsif genre == "folk traditional" || genre == "folk/traditional" genres << Genre.find('folk') elsif genre == "elvis" genres << Genre.find('rock') elsif genre == "irish" genres << Genre.find('celtic') elsif genre == "dance/pop" genres << Genre.find('dance') genres << Genre.find('pop') elsif genre == "the beatles" genres << Genre.find("rock") else found = Genre.find_by_id(genre) genres << found if found end end end # just throw them into rock/pop if not known. can fix later... if genres.length == 0 genres << Genre.find('rock') genres << Genre.find('pop') end genres end def determine_language(metadata) found = ISO_639.find_by_code('eng') language = metadata[:language] if language language.downcase! if language == 'instrumental' return 'instrumental' end if language.include? 'spanish' found = ISO_639.find_by_code('spa') elsif language.include? 'german' found = ISO_639.find_by_code('ger') elsif language.include? 'portuguese' found = ISO_639.find_by_code('por') elsif language.include? 'english' found = ISO_639.find_by_code('eng') end end found[0] # 3 letter code end #http://stackoverflow.com/questions/22740252/how-to-generate-javas-string-hashcode-using-ruby def jhash(str) result = 0 mul = 1 max_mod = 2**31 - 1 str.chars.reverse_each do |c| result += mul * c.ord result %= max_mod mul *= 31 end result end def prevent_concurrent_processing(metalocation) # use a PG advisory lock to see if someone else is doing this same unit of work right now track_code = jhash(metalocation) locked = ActiveRecord::Base.connection.execute("SELECT pg_try_advisory_xact_lock(#{track_code})").values[0][0] if locked == 'f' finish("other_processing", "") raise ActiveRecord::Rollback end end def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options) metadata ||= {} self.name = metadata["name"] || name prevent_concurrent_processing(metalocation) if jam_track.new_record? latest_jamtrack = JamTrack.order('id::int desc').first id = latest_jamtrack.nil? ? 1 : latest_jamtrack.id.to_i + 1 if ENV['NODE_NUMBER'] # complicated goofy code to support parallel processing of importers node_number = ENV['NODE_NUMBER'].to_i node_count = ENV['NODE_COUNT'].to_i raise "NO NODE_COUNT" if node_count == 0 r = id % node_count id = (node_count - r) + id # get to the same base number if both are working at the same time id = id + node_number # offset by your node number @@log.info("JAM TRACK ID: #{id}") end jam_track.id = "#{id}" # default is UUID, but the initial import was based on auto-increment ID, so we'll maintain that jam_track.status = 'Staging' jam_track.metalocation = metalocation jam_track.original_artist = metadata["original_artist"] || original_artist jam_track.name = self.name jam_track.additional_info = metadata[:additional_info] jam_track.year = metadata[:year] jam_track.genres = determine_genres(metadata) jam_track.language = determine_language(metadata) jam_track.price = 1.99 jam_track.reproduction_royalty_amount = nil jam_track.reproduction_royalty = true jam_track.public_performance_royalty = true jam_track.licensor_royalty_amount = 0.4 jam_track.sales_region = 'Worldwide' jam_track.recording_type = 'Cover' jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"." jam_track.hfa_license_status = false jam_track.alternative_license_status = false jam_track.hfa_license_desired = true jam_track.server_fixation_date = Time.now if is_tency_storage? jam_track.vendor_id = metadata[:id] jam_track.licensor = JamTrackLicensor.find_by_name!('Tency Music') #add_licensor_metadata('Tency Music', metalocation) elsif is_paris_storage? raise 'no vendor id' if metadata[:id].nil? jam_track.vendor_id = metadata[:id] jam_track.licensor = JamTrackLicensor.find_by_name!('Paris Music') jam_track.bpm = metadata[:bpm] elsif is_tim_tracks_storage? jam_track.vendor_id = metadata[:id] jam_track.licensor = JamTrackLicensor.find_by_name!('Tim Waurick') elsif is_helbing_storage? jam_track.vendor_id = metadata[:id] jam_track.licensor = JamTrackLicensor.find_by_name!('Stockton Helbing') elsif is_drumma_storage? jam_track.vendor_id = metadata[:id] jam_track.licensor = JamTrackLicensor.find_by_name!('Drumma Boy') end jam_track.slug = metadata['slug'] if jam_track.slug.nil? jam_track.generate_slug end jam_track.plan_code = metadata["plan_code"] if jam_track.plan_code.nil? jam_track.gen_plan_code end else if !options[:resync_audio] #@@log.debug("#{self.name} skipped because it already exists in database") finish("jam_track_exists", "") return false else # jamtrack exists, leave it be return true end end @@log.debug("about to save") saved = jam_track.save if !saved finish("invalid_definition", jam_track.errors.inspect) end saved end # oddballs - Guitar Solo.wav # Rocket Man Stem - Vocal Back Up # Rocket Man Stem - Vocal Lead Double # Rock and Roll Stem - Electric Guitar - Main - Solo def determine_instrument(potential_instrument_original, potential_part_original = nil) potential_instrument = potential_instrument_original.downcase potential_part = potential_part_original.downcase if potential_part_original instrument = nil used_helper = false part = nil if potential_instrument == 'guitar' if potential_part if potential_part == 'acoustic' instrument = 'acoustic guitar' used_helper = true elsif potential_part == 'electric' instrument = 'electric guitar' used_helper = true elsif potential_part == 'acoustic solo' instrument = 'acoustic guitar' used_helper = true part = 'Solo' elsif potential_part.include?('acoustic') used_helper = true # ambiguous else instrument = 'electric guitar' used_helper = false end else instrument = 'electric guitar' end elsif potential_instrument == 'acoustic' instrument = 'acoustic guitar' elsif potential_instrument == 'acoutic guitar' instrument = 'electric guitar' elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary' || potential_instrument == 'elec guitar' instrument = 'electric guitar' elsif potential_instrument == 'lead guitar' instrument = 'electric guitar' part = 'Lead' elsif potential_instrument == 'keys' instrument = 'keyboard' elsif potential_instrument == 'vocal' || potential_instrument == 'vocals' instrument = 'voice' elsif potential_instrument == 'upright bass' instrument = 'double bass' elsif potential_instrument == 'bass' instrument = 'bass guitar' elsif potential_instrument == 'drum' instrument = 'drums' elsif potential_instrument == 'sound effects' || potential_instrument == 'sound efx' || potential_instrument == 'effects' instrument = 'computer' if potential_part_original part = "Sound FX (#{potential_part_original})" else part = 'Sound FX' end elsif potential_instrument == 'computer scratches' instrument = 'computer' part = 'Scratches' elsif potential_instrument == "sax" instrument = 'saxophone' elsif potential_instrument == "vocal back up" instrument = "voice" part = "Back Up" elsif potential_instrument == "vocal lead double" instrument = "voice" part = "Lead Double" elsif potential_instrument == "guitar solo" instrument = "electric guitar" part = "Solo" elsif potential_instrument == 'stadium crowd' instrument = 'computer' part = 'Crowd Noise' elsif potential_instrument == 'cannons' instrument = 'computer' part = 'Cannons' elsif potential_instrument == 'bells' instrument = 'computer' part = 'Bells' elsif potential_instrument == 'percussion' instrument = 'percussion' part = 'Percussion' elsif potential_instrument == 'fretless bass' instrument = 'bass guitar' part = 'Fretless' elsif potential_instrument == 'lap steel' || potential_instrument == 'pedal steel' instrument = 'steel guitar' elsif potential_instrument == 'clock percussion' instrument = 'computer' part = 'Clock' elsif potential_instrument == 'horns' || potential_instrument == 'horn' instrument = 'other' part = 'Horns' if potential_part.nil? elsif potential_instrument == 'english horn' instrument = 'other' part = 'English Horn' elsif potential_instrument == 'bass clarinet' instrument = 'other' part = 'Bass Clarinet' elsif potential_instrument == 'recorder' instrument = 'other' part = 'Recorder' elsif potential_instrument == 'marimba' instrument = 'keyboard' part = 'Marimba' elsif potential_instrument == 'strings' instrument = 'orchestra' part = 'Strings' elsif potential_instrument == 'celesta' || potential_instrument == 'celeste' instrument = 'keyboard' part = 'Celesta' elsif potential_instrument == 'balalaika' instrument = 'other' part = 'Balalaika' elsif potential_instrument == 'tanpura' instrument = 'other' part = 'Tanpura' elsif potential_instrument == 'quena' instrument = 'other' part = 'Quena' elsif potential_instrument == 'bouzouki' instrument = 'other' part = 'Bouzouki' elsif potential_instrument == 'claps' || potential_instrument == 'hand claps' instrument = 'other' part = 'Claps' elsif potential_instrument == 'snaps' || potential_instrument == 'snap' instrument = 'other' part = 'Snaps' else found_instrument = Instrument.find_by_id(potential_instrument) if found_instrument instrument = found_instrument.id end end if !used_helper && !part part = potential_part_original end part = potential_instrument_original if !part {instrument: instrument, part: part} end def parse_file(file) bits = file.split('/') filename = bits[bits.length - 1] # remove all but just the filename filename_no_ext = filename[0..-5] comparable_filename = filename_no_ext.downcase # remove .wav type = nil master = false instrument = nil part = nil precount_num = nil no_precount_detail = nil if comparable_filename == "click" || comparable_filename.include?("clicktrack") || comparable_filename.include?("click track") || comparable_filename.end_with?('click') || comparable_filename.end_with?('click trac') if filename.end_with?('.txt') type = :clicktxt else type = :clickwav end elsif comparable_filename.include? "precount" type = :precount index = comparable_filename.index('precount') precount = comparable_filename[(index + 'precount'.length)..-1].strip if precount.start_with?('_') precount = precount[1..-1] end if precount.to_i == 0 no_precount_detail = comparable_filename else precount_num = precount.to_i end elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") || (@metadata && (@metadata[:id] && comparable_filename.start_with?(@metadata[:id].downcase))) master = true type = :master else type = :track stem_location = comparable_filename.index('stem -') unless stem_location stem_location = comparable_filename.index('stems -') end unless stem_location stem_location = comparable_filename.index('stem-') end unless stem_location stem_location = comparable_filename.index('stems-') end unless stem_location stem_location = comparable_filename.index(' -') end if stem_location bits = filename_no_ext[stem_location..-1].split('-') bits.collect! { |bit| bit.strip } possible_instrument = nil possible_part = nil if bits.length == 2 # second bit is instrument possible_instrument = bits[1] elsif bits.length == 3 # second bit is instrument, third bit is part possible_instrument = bits[1] possible_part = bits[2] elsif bits.length == 4 possible_instrument = bits[1] possible_part = "#{bits[2]} #{bits[3]}" end result = determine_instrument(possible_instrument, possible_part) instrument = result[:instrument] part = result[:part] if is_tim_tracks_storage? if instrument.nil? # some stems don't include 'voice'. but... they are all voice :) instrument = 'voice' end end else if is_tency_storage? # we can check to see if we can find mapping info for this filename mapping = JamTrackImporter.tency_mapping[filename.downcase] if mapping && mapping[:trust] instrument = mapping[:instrument] part = mapping[:part] end # tency mapping didn't work; let's retry with our own home-grown mapping if instrument.nil? && !possible_instrument.nil? result = determine_instrument(possible_instrument, possible_part) instrument = result[:instrument] part = result[:part] end elsif is_paris_storage? # example: Eternal Flame-Guide Lead Vocal.wav # or with a part: Eternal Flame-Keyboard-Stab.wav bits = filename_no_ext.split('-') bits.collect! { |bit| bit.strip } while true instrument, part = paris_instrument_parse(bits) if instrument.nil? && bits.length > 2 bits.shift instrument, part = paris_instrument_parse(bits) else break end end end end end {filename: filename, master: master, instrument: instrument, part: part, type: type, precount_num: precount_num, no_precount_detail: no_precount_detail} end def paris_instrument_parse(bits) instrument = nil part = nil possible_instrument = nil possible_part = nil if bits.length == 2 # second bit is instrument possible_instrument = bits[1] elsif bits.length == 3 # second bit is instrument, third bit is part possible_instrument = bits[1] possible_part = bits[2] elsif bits.length >= 4 possible_instrument = bits[1] possible_part = "#{bits[2]} #{bits[3]}" end # otherwise, try mapping if instrument.nil? && possible_instrument mapping = JamTrackImporter.paris_mapping[possible_instrument.downcase] if mapping instrument = mapping[:instrument].downcase part = mapping[:part] part = nil if part.blank? end end # paris mapping didn't work; let's retry one more time with our own home-grown mapping if instrument.nil? result = determine_instrument(possible_instrument, possible_part) instrument = result[:instrument] part = result[:part] end return instrument, part end def dry_run_audio(metadata, s3_path) all_files = fetch_important_files(s3_path) masters = 0 all_files.each do |file| # ignore click/precount parsed_wav = parse_file(file) if parsed_wav[:master] @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") masters += 1 if masters > 1 JamTrackImporter.summaries[:multiple_masters] += 1 end elsif parsed_wav[:type] == :track JamTrackImporter.summaries[:total_tracks] += 1 if parsed_wav[:instrument].nil? detail = JamTrackImporter.summaries[:no_instrument_detail] file_detail = detail[parsed_wav[:filename].downcase] if file_detail.nil? detail[parsed_wav[:filename].downcase] = 0 end detail[parsed_wav[:filename].downcase] += 1 JamTrackImporter.summaries[:no_instrument] += 1 end JamTrackImporter.summaries[:no_part] += 1 if parsed_wav[:part].nil? if !parsed_wav[:instrument] || !parsed_wav[:part] @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") else @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") end elsif parsed_wav[:type] == :clickwav elsif parsed_wav[:type] == :clicktxt elsif parsed_wav[:type] == :precount if parsed_wav[:precount_num].nil? JamTrackImporter.summaries[:no_precount_num] += 1 JamTrackImporter.summaries[:no_precount_detail] << parsed_wav[:no_precount_detail] end else JamTrackImporter.summaries[:unknown_filetype] += 1 end end end def set_custom_weight(track) slop = 800 instrument_weight = nil # if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end #if track.persisted? # instrument_weight = track.position #else if track.instrument_id == 'voice' if track.part && track.part.start_with?('Lead') instrument_weight = 100 elsif track.part && track.part.start_with?('Backing') instrument_weight = 110 else instrument_weight = 120 end elsif track.instrument_id == 'drums' if track.part && track.part == 'Drums' instrument_weight = 150 elsif track.part && track.part == 'Percussion' instrument_weight = 160 else instrument_weight = 170 end elsif track.instrument_id == 'percussion' instrument_weight = 175 elsif track.instrument_id == 'bass guitar' && track.part && track.part == 'Bass' instrument_weight = 180 elsif track.instrument_id == 'piano' && track.part && track.part == 'Piano' instrument_weight = 250 elsif track.instrument_id == 'keyboard' if track.part && track.part.start_with?('Synth') instrument_weight = 260 elsif track.part && track.part.start_with?('Pads') instrument_weight = 270 else instrument_weight = 280 end elsif track.instrument_id == 'acoustic guitar' if track.part && track.part.start_with?('Lead') instrument_weight = 300 elsif track.part && track.part.start_with?('Rhythm') instrument_weight = 310 else instrument_weight = 320 end elsif track.instrument_id == 'electric guitar' if track.part && track.part.start_with?('Lead') instrument_weight = 400 elsif track.part && track.part.start_with?('Solo') instrument_weight = 410 elsif track.part && track.part.start_with?('Rhythm') instrument_weight = 420 else instrument_weight = 440 end else instrument_weight = slop end if track.track_type == 'Master' instrument_weight = 1000 end if track.track_type == 'Click' instrument_weight = 10000 end #end instrument_weight end def deduplicate_parts(tracks) unique_instruments = {} tracks.each do |track| key = "#{track.instrument_id} | #{track.part}" found = unique_instruments[key] if !found found = [] unique_instruments[key] = found end found << track end unique_instruments.each do |key, value| if value.length > 1 count = 0 value.each do |track| if track.part.nil? track.part = (count + 1).to_s else track.part = "#{track.part} #{count + 1}" end count += 1 end end end # debug output tracks.each do |track| puts "TRACK #{track.instrument_id} #{track.part}" end end def sort_tracks(tracks) sorted_tracks = tracks.sort do |a, b| a_weight = set_custom_weight(a) b_weight = set_custom_weight(b) if a_weight != b_weight a_weight <=> b_weight elsif a.instrument_id != b.instrument_id a.instrument_id <=> b.instrument_id else a_part = a.part b_part = b.part a_part <=> b_part end end position = 1 sorted_tracks.each do |track| track.position = position position = position + 1 end # get click/master tracks position re-set correctly last_track = sorted_tracks[sorted_tracks.length - 1] second_to_last = sorted_tracks[sorted_tracks.length - 2] if last_track.track_type == 'Master' last_track.position = 1000 elsif last_track.track_type == 'Click' last_track.position = 10000 end if second_to_last.track_type == 'Master' second_to_last.position = 1000 elsif second_to_last.track_type == 'Click' second_to_last.position = 10000 end sorted_tracks end # this will put original_audio_s3_path on each jam_track_track def associate_tracks_with_original_stems(jam_track, s3_path) attempt_to_match_existing_tracks = true # find all wav files in the JamTracks s3 bucket wav_files = fetch_important_files(s3_path) tracks = [] wav_files.each do |wav_file| if attempt_to_match_existing_tracks # try to find a matching track from the JamTrack based on the name of the 44.1 path basename = File.basename(wav_file) ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg" found_track = nil jam_track.jam_track_tracks.each do |jam_track_track| if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename) # found a match! found_track = jam_track_track break end end if found_track @@log.debug("found a existing track to reuse") found_track.original_audio_s3_path = wav_file tracks << found_track next end end end tracks end def synchronize_audio(jam_track, metadata, s3_path, skip_audio_upload) attempt_to_match_existing_tracks = true # find all wav files in the JamTracks s3 bucket wav_files = fetch_important_files(s3_path) tracks = [] addt_files = [] wav_files.each do |wav_file| if attempt_to_match_existing_tracks # try to find a matching track from the JamTrack based on the name of the 44.1 path basename = File.basename(wav_file) ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg" found_track = nil jam_track.jam_track_tracks.each do |jam_track_track| if jam_track_track["url_44"] && jam_track_track["url_44"].end_with?(ogg_44100_filename) # found a match! found_track = jam_track_track break end end if found_track @@log.debug("found a existing track to reuse") found_track.original_audio_s3_path = wav_file tracks << found_track next end end @@log.debug("no existing track found; creating a new one") if !assign_instrument_parts(wav_file, tracks, addt_files) return false end end jam_track.jam_track_tracks.each do |jam_track_track| # delete all jam_track_tracks not in the tracks array unless tracks.include?(jam_track_track) @@log.info("destroying removed JamTrackTrack #{jam_track_track.inspect}") jam_track_track.destroy # should also delete s3 files associated with this jamtrack end end jam_track.jam_track_files.each do |jam_track_file| unless addt_files.include?(jam_track_file) @@log.info("destroying removed JamTrackFile #{jam_track_file.inspect}") jam_track_file.destroy # should also delete s3 files associated with this jamtrack end end @@log.info("sorting tracks") tracks = sort_tracks(tracks) deduplicate_parts(tracks) jam_track.jam_track_tracks = tracks jam_track.jam_track_files = addt_files saved = jam_track.save if !saved finish('invalid_audio', jam_track.errors.inspect) return false end return synchronize_audio_files(jam_track, skip_audio_upload) end def reassign_instrument_parts(jam_track) tracks = [] addt_files = [] jam_track.jam_track_tracks.each do |track| return if !assign_instrument_parts(track.original_filename, tracks, addt_files, true) end @@log.info("sorting #{tracks.length} tracks") tracks = sort_tracks(tracks) deduplicate_parts(tracks) changed = false tracks.each do |track| if track.changed? changed = true puts "CHANGE: #{track.changes.inspect}" track.skip_inst_part_uniq = true track.save! end end if changed # if we messed up any instrument/parts by making a dup, this will catch it tracks.each do |track| track.skip_inst_part_uniq = false track.save! end end end def assign_instrument_parts(wav_file, tracks, addt_files, reassign = false) if !reassign track = JamTrackTrack.new track.original_filename = wav_file track.original_audio_s3_path = wav_file file = JamTrackFile.new file.original_filename = wav_file file.original_audio_s3_path = wav_file else matches = JamTrackTrack.where(original_filename: wav_file) if matches.count > 1 raise "multiple jam track tracks encountered with #{wav_file} as original_filename" elsif matches.count == 0 raise "unable to locate jam track wit h#{wav_file} as original_filename" end track = matches[0] track.original_audio_s3_path = wav_file file = nil end parsed_wav = parse_file(wav_file) unknowns = 0 if parsed_wav[:master] track.track_type = 'Master' track.part = 'Master Mix' track.instrument_id = 'computer' tracks << track @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") elsif parsed_wav[:type] == :track if !parsed_wav[:instrument] || !parsed_wav[:part] @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") unknowns += 1 else @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") end track.instrument_id = parsed_wav[:instrument] || 'other' track.track_type = 'Track' track.part = parsed_wav[:part]; tracks << track elsif parsed_wav[:type] == :clicktxt file.file_type = 'ClickTxt' addt_files << file elsif parsed_wav[:type] == :clickwav if file file.file_type = 'ClickWav' addt_files << file end # and also add a JamTrackTrack for this click track track.track_type = 'Click' track.part = 'Clicktrack' track.instrument_id = 'computer' track.position = 10000 tracks << track elsif parsed_wav[:type] == :precount file.file_type = 'Precount' file.precount_num = parsed_wav[:precount_num] addt_files << file else finish("unknown_file_type", "unknown file type #{wave_file}") return false end return true end def synchronize_audio_files(jam_track, skip_audio_upload) begin Dir.mktmpdir do |tmp_dir| # download each jam track here, and then do processing to determine: # what's the longest stem # and to then pad the rest of the tracks to make them all match in length jam_track.jam_track_tracks.each do |track| basename = File.basename(track.original_audio_s3_path) wav_file = File.join(tmp_dir, basename) # bring the original wav file down from S3 to local file system JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file) track.wav_file = wav_file end same_lengthening(jam_track, tmp_dir) jam_track.jam_track_tracks.each do |track| synchronize_audio_track(jam_track, tmp_dir, skip_audio_upload, track) end end generate_jmep(jam_track) rescue Exception => e finish("sync_audio_exception", e.to_s) return false end return true end # make all stems be the same length def same_lengthening(jam_track, tmp_dir) longest_duration = nil jam_track.jam_track_tracks.each do |track| duration_command = "soxi -D \"#{track.wav_file}\"" output = `#{duration_command}` result_code = $?.to_i if result_code == 0 duration = output.to_f.round track.tmp_duration = duration if longest_duration.nil? longest_duration = duration else if duration > longest_duration longest_duration = duration end end else @@log.warn("unable to determine duration for jam_track track #{jam_track.name} #{jam_track_track.instrument} #{jam_track_track.part}. output #{output}") end end @@log.info("duration determined to be #{longest_duration}") jam_track.duration = longest_duration jam_track.jam_track_tracks.each do |track| if track.tmp_duration < longest_duration # need to pad with silence to make all match in length amount = longest_duration - track.tmp_duration @@log.info("track #{track.instrument_id}:#{track.part} needs to be lengthened by #{amount}") output = cmd("soxi -c \"#{track.wav_file}\"", "padded_silence") channels = output.to_i output = cmd("soxi -r \"#{track.wav_file}\"", "get_sample_rate") sample_rate = output.to_i padding_file = create_silence(tmp_dir, "padded_silence#{track.id}", amount, sample_rate, channels) output_file = File.join(tmp_dir, "with_padding_#{track.id}.wav") cmd("sox \"#{track.wav_file}\" \"#{padding_file}\" \"#{output_file}\"", "same_lengthening") track.wav_file = output_file end end end def cmd(cmd, type) @@log.debug("executing #{cmd}") output = `#{cmd}` result_code = $?.to_i if result_code == 0 output else @error_reason = type + "_fail" @error_detail = "#{cmd}, #{output}" raise "command `#{cmd}` failed. #{type}, #{output}" end end def synchronize_audio_track(jam_track, tmp_dir, skip_audio_upload, track) basename = File.basename(track.original_audio_s3_path) # make a 44100 version, and a 48000 version ogg_44100_filename = File.basename(basename, ".wav") + "-44100.ogg" ogg_48000_filename = File.basename(basename, ".wav") + "-48000.ogg" # make a 44100 version, and a 48000 version mp3_48000_filename = File.basename(basename, ".wav") + "-48000.mp3" aac_48000_filename = File.basename(basename, ".wav") + "-48000.aac" ogg_44100_s3_path = track.filename(ogg_44100_filename) ogg_48000_s3_path = track.filename(ogg_48000_filename) mp3_48000_s3_path = track.filename(mp3_48000_filename) aac_48000_s3_path = track.filename(aac_48000_filename) track.skip_uploader = true if skip_audio_upload track["url_44"] = ogg_44100_s3_path track["md5_44"] = 'md5' track["length_44"] = 1 track["url_48"] = ogg_48000_s3_path track["md5_48"] = 'md5' track["length_48"] = 1 track["url_mp3_48"] = mp3_48000_filename track["md5_mp3_48"] = 'md5' track["length_mp3_48"] = 1 track["url_aac_48"] = aac_48000_filename track["md5_aac_48"] = 'md5' track["length_aac_48"] = 1 # we can't fake the preview as easily because we don't know the MD5 of the current item #track["preview_md5"] = 'md5' #track["preview_mp3_md5"] = 'md5' #track["preview_url"] = track.preview_filename('md5', 'ogg') #track["preview_length"] = 1 #track["preview_mp3_url"] = track.preview_filename('md5', 'mp3') #track["preview_mp3_length"] = 1 #track["preview_start_time"] = 0 else wav_file = track.wav_file sample_rate = `soxi -r "#{wav_file}"`.strip ogg_44100 = File.join(tmp_dir, ogg_44100_filename) ogg_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.ogg") if sample_rate == "44100" `oggenc "#{wav_file}" -q 6 -o "#{ogg_44100}"` else `oggenc "#{wav_file}" --resample 44100 -q 6 -o "#{ogg_44100}"` end if sample_rate == "48000" `oggenc "#{wav_file}" -q 6 -o "#{ogg_48000}"` else `oggenc "#{wav_file}" --resample 48000 -q 6 -o "#{ogg_48000}"` end # upload the new ogg files to s3 @@log.debug("uploading 44100 to #{ogg_44100_s3_path}") jamkazam_s3_manager.upload(ogg_44100_s3_path, ogg_44100) @@log.debug("uploading 48000 to #{ogg_48000_s3_path}") jamkazam_s3_manager.upload(ogg_48000_s3_path, ogg_48000) ogg_44100_digest = ::Digest::MD5.file(ogg_44100) # and finally update the JamTrackTrack with the new info track["url_44"] = ogg_44100_s3_path track["md5_44"] = ogg_44100_digest.hexdigest track["length_44"] = File.new(ogg_44100).size track["url_48"] = ogg_48000_s3_path track["md5_48"] = ::Digest::MD5.file(ogg_48000).hexdigest track["length_48"] = File.new(ogg_48000).size # now create mp3 and aac files mp3_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.mp3") aac_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.aac") `ffmpeg -i "#{wav_file}" -ar 48000 -ab 192k "#{mp3_48000}"` `ffmpeg -i "#{wav_file}" -c:a libfdk_aac -b:a 192k "#{aac_48000}"` # upload the new ogg files to s3 @@log.debug("uploading mp3 48000 to #{mp3_48000_s3_path}") jamkazam_s3_manager.upload(mp3_48000_s3_path, mp3_48000) @@log.debug("uploading aac 48000 to #{aac_48000_s3_path}") jamkazam_s3_manager.upload(aac_48000_s3_path, aac_48000) mp3_48000_digest = ::Digest::MD5.file(mp3_48000) # and finally update the JamTrackTrack with the new info track["url_mp3_48"] = mp3_48000_s3_path track["md5_mp3_48"] = mp3_48000_digest.hexdigest track["length_mp3_48"] = File.new(mp3_48000).size track["url_aac_48"] = aac_48000_s3_path track["md5_aac_48"] = ::Digest::MD5.file(aac_48000).hexdigest track["length_aac_48"] = File.new(aac_48000).size jam_track.save! # convert entire master ogg file to mp3, and push both to public destination if track.track_type == 'Master' preview_succeeded = synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_44100_digest) if !preview_succeeded return false end elsif track.track_type == 'Track' || track.track_type == 'Click' synchronize_track_preview(track, tmp_dir, ogg_44100) end end track.save! end def generate_mp3_aac_stem(jam_track, tmp_dir, skip_audio_upload) jam_track.jam_track_tracks.each do |track| if track.original_audio_s3_path.nil? @@log.error("jam_track #{jam_track.name} has empty stem. stem: #{track.id}") next end puts "track.original_audio_s3_path #{track.original_audio_s3_path}" basename = File.basename(track.original_audio_s3_path) s3_dirname = File.dirname(track.original_audio_s3_path) # make a 44100 version, and a 48000 version mp3_48000_filename = File.basename(basename, ".wav") + "-48000.mp3" aac_48000_filename = File.basename(basename, ".wav") + "-48000.aac" mp3_48000_s3_path = track.filename(mp3_48000_filename) aac_48000_s3_path = track.filename(aac_48000_filename) puts "mp3_48000_s3_path #{mp3_48000_s3_path}" track.skip_uploader = true if skip_audio_upload track["url_mp3_48"] = mp3_48000_filename track["md5_mp3_48"] = 'md5' track["length_mp3_48"] = 1 track["url_aac_48"] = aac_48000_filename track["md5_aac_48"] = 'md5' track["length_aac_48"] = 1 # we can't fake the preview as easily because we don't know the MD5 of the current item #track["preview_md5"] = 'md5' #track["preview_mp3_md5"] = 'md5' #track["preview_url"] = track.preview_filename('md5', 'ogg') #track["preview_length"] = 1 #track["preview_mp3_url"] = track.preview_filename('md5', 'mp3') #track["preview_mp3_length"] = 1 #track["preview_start_time"] = 0 else wav_file = File.join(tmp_dir, basename) # the wave file might already be on the system... # don't bother with the same track twice next if track["url_mp3_48"] && track["url_aac_48"] # bring the original wav file down from S3 to local file system JamTrackImporter::song_storage_manager.download(track.original_audio_s3_path, wav_file) unless File.exists?(wav_file) mp3_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.mp3") aac_48000 = File.join(tmp_dir, File.basename(basename, ".wav") + "-48000.aac") `ffmpeg -i "#{wav_file}" -ar 48000 -ab 192k "#{mp3_48000}"` `ffmpeg -i "#{wav_file}" -c:a libfdk_aac -b:a 192k "#{aac_48000}"` # upload the new ogg files to s3 @@log.debug("uploading mp3 48000 to #{mp3_48000_s3_path}") jamkazam_s3_manager.upload(mp3_48000_s3_path, mp3_48000) @@log.debug("uploading aac 48000 to #{aac_48000_s3_path}") jamkazam_s3_manager.upload(aac_48000_s3_path, aac_48000) mp3_48000_digest = ::Digest::MD5.file(mp3_48000) # and finally update the JamTrackTrack with the new info track["url_mp3_48"] = mp3_48000_s3_path track["md5_mp3_48"] = mp3_48000_digest.hexdigest track["length_mp3_48"] = File.new(mp3_48000).size track["url_aac_48"] = aac_48000_s3_path track["md5_aac_48"] = ::Digest::MD5.file(aac_48000).hexdigest track["length_aac_48"] = File.new(aac_48000).size track.save end end end def synchronize_duration(jam_track, ogg_44100) duration_command = "soxi -D \"#{ogg_44100}\"" output = `#{duration_command}` result_code = $?.to_i if result_code == 0 duration = output.to_f.round jam_track.duration = duration else @@log.warn("unable to determine duration for jam_track #{jam_track.name}. output #{output}") end true end def synchronize_track_preview(track, tmp_dir, ogg_44100) preview_start_time = determine_start_time(ogg_44100, tmp_dir, track.original_filename) # this is in seconds; convert to integer milliseconds preview_start_time = (preview_start_time * 1000).to_i preview_start_time = 0 if preview_start_time < 0 track.preview_start_time = preview_start_time track.process_preview(ogg_44100, tmp_dir) if track.preview_start_time if track.preview_generate_error @@log.warn(track.preview_generate_error) end end def synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_digest) begin aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac') convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\"" @@log.debug("converting to aac using: " + convert_aac_cmd) convert_output = `#{convert_aac_cmd}` aac_digest = ::Digest::MD5.file(aac_44100) track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest # upload 44100 aac to public location @@log.debug("uploading aac preview to #{track.preview_filename('aac')}") public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest) track.skip_uploader = true original_aac_preview_url = track["preview_aac_url"] # and finally update the JamTrackTrack with the new info track["preview_aac_url"] = track.preview_filename(aac_md5, 'aac') track["preview_aac_length"] = File.new(aac_44100).size track["preview_start_time"] = 0 if !track.save finish("save_master_preview", track.errors.to_s) return false end # if all that worked, now delete old previews, if present begin public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"] rescue puts "UNABLE TO CLEANUP OLD PREVIEW URL" end rescue Exception => e finish("sync_master_preview_exception", e.to_s) return false end return true end def synchronize_master_preview(track, tmp_dir, ogg_44100, ogg_digest) begin mp3_44100 = File.join(tmp_dir, 'output-preview-44100.mp3') convert_mp3_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -ab 192k \"#{mp3_44100}\"" @@log.debug("converting to mp3 using: " + convert_mp3_cmd) convert_output = `#{convert_mp3_cmd}` mp3_digest = ::Digest::MD5.file(mp3_44100) aac_44100 = File.join(tmp_dir, 'output-preview-44100.aac') convert_aac_cmd = "#{APP_CONFIG.ffmpeg_path} -i \"#{ogg_44100}\" -c:a libfdk_aac -b:a 192k \"#{aac_44100}\"" @@log.debug("converting to aac using: " + convert_aac_cmd) convert_output = `#{convert_aac_cmd}` aac_digest = ::Digest::MD5.file(aac_44100) track["preview_md5"] = ogg_md5 = ogg_digest.hexdigest track["preview_mp3_md5"] = mp3_md5 = mp3_digest.hexdigest track["preview_aac_md5"] = aac_md5 = aac_digest.hexdigest # upload 44100 ogg, mp3, aac to public location as well @@log.debug("uploading ogg preview to #{track.preview_filename('ogg')}") public_jamkazam_s3_manager.upload(track.preview_filename(ogg_digest.hexdigest, 'ogg'), ogg_44100, content_type: 'audio/ogg', content_md5: ogg_digest.base64digest) @@log.debug("uploading mp3 preview to #{track.preview_filename('mp3')}") public_jamkazam_s3_manager.upload(track.preview_filename(mp3_digest.hexdigest, 'mp3'), mp3_44100, content_type: 'audio/mpeg', content_md5: mp3_digest.base64digest) @@log.debug("uploading aac preview to #{track.preview_filename('aac')}") public_jamkazam_s3_manager.upload(track.preview_filename(aac_digest.hexdigest, 'aac'), aac_44100, content_type: 'audio/aac', content_md5: aac_digest.base64digest) track.skip_uploader = true original_ogg_preview_url = track["preview_url"] original_mp3_preview_url = track["preview_mp3_url"] original_aac_preview_url = track["preview_aac_url"] # and finally update the JamTrackTrack with the new info track["preview_url"] = track.preview_filename(ogg_md5, 'ogg') track["preview_length"] = File.new(ogg_44100).size # and finally update the JamTrackTrack with the new info track["preview_mp3_url"] = track.preview_filename(mp3_md5, 'mp3') track["preview_mp3_length"] = File.new(mp3_44100).size track["preview_aac_url"] = track.preview_filename(aac_md5, 'mp3') track["preview_aac_length"] = File.new(aac_44100).size track["preview_start_time"] = 0 if !track.save finish("save_master_preview", track.errors.to_s) return false end # if all that worked, now delete old previews, if present begin public_jamkazam_s3_manager.delete(original_ogg_preview_url) if original_ogg_preview_url && original_ogg_preview_url != track["preview_url"] public_jamkazam_s3_manager.delete(original_mp3_preview_url) if original_mp3_preview_url && original_mp3_preview_url != track["preview_mp3_url"] public_jamkazam_s3_manager.delete(original_aac_preview_url) if original_aac_preview_url && original_aac_preview_url != track["preview_aac_url"] rescue puts "UNABLE TO CLEANUP OLD PREVIEW URL" end rescue Exception => e finish("sync_master_preview_exception", e.to_s) return false end return true end def fetch_all_files(s3_path) JamTrackImporter::song_storage_manager.list_files(s3_path) end def fetch_important_files(s3_path) files = fetch_all_files(s3_path) files.select { |file| file.end_with?('.wav') || file.end_with?('.txt') } end def synchronize(jam_track, metadata, metalocation, options) # metalocation should be audio/original artist/song name/meta.yml metadata ||= {} parsed_metalocation = parse_metalocation(metalocation) return unless parsed_metalocation original_artist = parsed_metalocation[1] name = parsed_metalocation[2] if is_paris_storage? bpm = parsed_metalocation[-1] bpm.downcase! if bpm.end_with?('bpm') bpm = bpm[0..-4].to_f end metadata[:bpm] = bpm end success = synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options) return unless success audio_path = metalocation[0...-"/meta.yml".length] synchronized_audio = synchronize_audio(jam_track, metadata, audio_path, options[:skip_audio_upload]) return unless synchronized_audio created_plan = synchronize_recurly(jam_track) if created_plan finish("success", nil) end # do a last check on any problems with the jamtrack jam_track.sync_onboarding_exceptions end def synchronize_recurly(jam_track) begin recurly = RecurlyClient.new # no longer create JamTrack plans: VRFS-3028 # recurly.create_jam_track_plan(jam_track) unless recurly.find_jam_track_plan(jam_track) rescue RecurlyClientError => x finish('recurly_create_plan', x.errors.to_s) return false end true end class << self attr_accessor :storage_format attr_accessor :tency_mapping attr_accessor :tency_metadata attr_accessor :paris_mapping attr_accessor :paris_metadata attr_accessor :summaries attr_accessor :marks_approved def report_summaries @@log.debug("SUMMARIES DUMP") @@log.debug("--------------") @summaries.each do |k, v| if k == :no_instrument_detail @@log.debug("#{k}: #{v}") elsif k == :no_precount_detail v.each do |precount_detail| @@log.debug("precount: #{precount_detail}") end elsif k == :unique_artists v.each do |artist| @@log.debug("artist: #{artist}") end else @@log.debug("#{k}: #{v}") end end end def song_storage_manager if is_tency_storage? tency_s3_manager elsif is_paris_storage? paris_s3_manager elsif is_tim_tracks_storage? tim_tracks_s3_manager elsif is_drumma_storage? drumma_s3_manager elsif is_helbing_storage? helbing_s3_manager else s3_manager end end def summaries @summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}, no_precount_num: 0, no_precount_detail: [], unique_artists: SortedSet.new, multiple_masters: 0, total: 0} end def drumma_s3_manager @drumma_s3_manager ||= S3Manager.new('jamkazam-drumma', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def tency_s3_manager @tency_s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def paris_s3_manager @paris_s3_manager ||= S3Manager.new('jamkazam-paris', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def tim_tracks_s3_manager @tim_tracks_s3_manager ||= S3Manager.new('jamkazam-timtracks', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def helbing_s3_manager @tim_tracks_s3_manager ||= S3Manager.new('jamkazam-helbing', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def s3_manager @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def private_s3_manager @private_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def extract_paris_song_id(metalocation) first_path = metalocation.index('/') return nil unless first_path metalocation = metalocation[(first_path + 1)..-1] suffix = '/meta.yml' metalocation = metalocation[0...-suffix.length] first_dash = metalocation.index('-') return nil if first_dash.nil? id = metalocation[0...first_dash].strip return nil unless id.start_with?('S') # all start with S return nil if id[1..-1].to_i == 0 # and number after that id end def extract_tency_song_id(metalocation) # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml first_path = metalocation.index('/') return nil unless first_path metalocation = metalocation[(first_path + 1)..-1] suffix = '/meta.yml' metalocation = metalocation[0...-suffix.length] last_dash = metalocation.rindex('-') return nil if last_dash.nil? id = metalocation[(last_dash+1)..-1].strip return nil if id.to_i == 0 id end def is_default_storage? assert_storage_set @storage_format == 'default' end def is_drumma_storage? assert_storage_set @storage_format == 'Drumma' end def is_tency_storage? assert_storage_set @storage_format == 'Tency' end def is_paris_storage? assert_storage_set @storage_format == 'Paris' end def is_tim_tracks_storage? assert_storage_set @storage_format == 'TimTracks' end def is_helbing_storage? assert_storage_set @storage_format == 'Helbing' end def assert_storage_set raise "no storage_format set" if @storage_format.nil? end def iterate_tim_tracks_song_storage(&blk) count = 0 song_storage_manager.list_directories('mapped').each do |song| @@log.debug("searching through song directory '#{song}'") metalocation = "#{song}meta.yml" metadata = load_metalocation(metalocation) blk.call(metadata, metalocation) count += 1 #break if count > 100 end end def iterate_paris_song_storage(&blk) count = 0 song_storage_manager.list_directories('mapped').each do |song| @@log.debug("searching through song directory '#{song}'") #next if song != 'mapped/S1555-Ashlee Simpson-L-O-V-E-96bpm/' metalocation = "#{song}meta.yml" metadata = load_metalocation(metalocation) if metadata.nil? # we don't do a paris song unless it has metadata next end blk.call(metadata, metalocation) count += 1 #break if count > 1000 end end def iterate_tency_song_storage(&blk) count = 0 song_storage_manager.list_directories('mapped').each do |song| @@log.debug("searching through song directory '#{song}'") metalocation = "#{song}meta.yml" metadata = load_metalocation(metalocation) blk.call(metadata, metalocation) count += 1 #break if count > 100 end end def iterate_default_song_storage(&blk) song_storage_manager.list_directories('audio').each do |original_artist| @@log.debug("searching through artist directory '#{original_artist}'") songs = song_storage_manager.list_directories(original_artist) songs.each do |song| @@log.debug("searching through song directory' #{song}'") metalocation = "#{song}meta.yml" metadata = load_metalocation(metalocation) blk.call(metadata, metalocation) end end end def iterate_drumma_song_storage(&blk) song_storage_manager.list_directories.each do |song| @@log.debug("searching through song directory '#{song}'") metalocation = "#{song}meta.yml" metadata = load_metalocation(metalocation) blk.call(metadata, metalocation) end end def iterate_helbing_song_storage(&blk) count = 0 song_storage_manager.list_directories('mapped').each do |song| @@log.debug("searching through song directory '#{song}'") metalocation = "#{song}meta.yml" metadata = load_metalocation(metalocation) blk.call(metadata, metalocation) count += 1 #break if count > 100 end end def iterate_song_storage(&blk) if is_tency_storage? iterate_tency_song_storage do |metadata, metalocation| blk.call(metadata, metalocation) end elsif is_paris_storage? iterate_paris_song_storage do |metadata, metalocation| blk.call(metadata, metalocation) end elsif is_tim_tracks_storage? iterate_tim_tracks_song_storage do |metadata, metalocation| blk.call(metadata, metalocation) end elsif is_drumma_storage? iterate_drumma_song_storage do |metadata, metalocation| blk.call(metadata, metalocation) end elsif is_helbing_storage? iterate_helbing_song_storage do |metadata, metalocation| blk.call(metadata, metalocation) end else iterate_default_song_storage do |metadata, metalocation| blk.call(metadata, metalocation) end end end def dry_run iterate_song_storage do |metadata, metalocation| jam_track_importer = JamTrackImporter.new(@storage_format) jam_track_importer.metadata = metadata JamTrackImporter.summaries[:total] += 1 jam_track_importer.dry_run(metadata, metalocation) end report_summaries end # figure out which songs are in S3 that do not exist in the 2k spreadsheet (mapping.csv), and which songs are in the 2k spreadsheet that are not in S3 def tency_delta in_s3 = {} in_mapping = {} load_tency_mappings JamTrackImporter.tency_metadata.each do |song_id, metadata| in_mapping[song_id] = {artist: metadata[:original_artist], song: metadata[:name]} end iterate_song_storage do |metadata, metalocation| importer = JamTrackImporter.new(@storage_format) importer.metadata = metadata song_id = JamTrackImporter.extract_tency_song_id(metalocation) parsed_metalocation = importer.parse_metalocation(metalocation) next if song_id.nil? next if parsed_metalocation.nil? original_artist = parsed_metalocation[1] meta_name = parsed_metalocation[2] in_s3[song_id] = {artist: original_artist, song: meta_name} end in_s3_keys = Set.new(in_s3.keys) in_mapping_keys = Set.new(in_mapping.keys) only_in_mapping = in_mapping_keys - in_s3_keys only_in_s3 = in_s3_keys - in_mapping_keys CSV.open("only_in_s3.csv", "wb") do |csv| only_in_s3.each do |song_id| csv << [song_id, in_s3[song_id][:artist], in_s3[song_id][:song]] end end CSV.open("only_in_2k_selection.csv", "wb") do |csv| only_in_mapping.each do |song_id| csv << [song_id, in_mapping[song_id][:artist], in_mapping[song_id][:song]] end end end def add_tency_metadata JamTrackLicensor.find_by_name('Tency Music').jam_tracks.each do |jam_track| jam_track_importer = JamTrackImporter.new(@storage_format) jam_track_importer.add_licensor_metadata('Tency Music', jam_track.metalocation) break end end def create_masters iterate_song_storage do |metadata, metalocation| next if metadata.nil? jam_track_importer = JamTrackImporter.new(@storage_format) jam_track_importer.metadata = metadata jam_track_importer.create_master(metadata, metalocation) end end def create_master(path) metalocation = "#{path}/meta.yml" metadata = load_metalocation(metalocation) jam_track_importer = JamTrackImporter.new(@storage_format) jam_track_importer.create_master(metadata, metalocation) end def dry_run_original s3_manager.list_directories('audio').each do |original_artist| @@log.debug("searching through artist directory '#{original_artist}'") songs = s3_manager.list_directories(original_artist) songs.each do |song| @@log.debug("searching through song directory' #{song}'") metalocation = "#{song}meta.yml" metadata = load_metalocation(metalocation) jam_track_importer = JamTrackImporter.new jam_track_importer.metadata = metadata jam_track_importer.dry_run(metadata, metalocation) end end end def synchronize_preview(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name error_occurred = false error_msg = nil jam_track.jam_track_tracks.each do |track| next if track.track_type == 'Master' if track.preview_start_time track.generate_preview if track.preview_generate_error error_occurred = true error_msg = track.preview_generate_error else end end end if error_occurred importer.finish('preview_error', error_msg) else importer.finish('success', nil) end importer end # hunts for the most recent .aac, .mp3, or .ogg file def synchronize_preview_dev(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name importer.synchronize_preview_dev(jam_track) importer.finish('success', nil) importer end def synchronize_jamtrack_aac_preview(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name track = jam_track.master_track if track Dir.mktmpdir do |tmp_dir| ogg_44100 = File.join(tmp_dir, 'input.ogg') private_s3_manager.download(track.url_by_sample_rate(44), ogg_44100) ogg_44100_digest = ::Digest::MD5.file(ogg_44100) if importer.synchronize_aac_preview(track, tmp_dir, ogg_44100, ogg_44100_digest) importer.finish("success", nil) end end else importer.finish('no_master_track', nil) end importer end def synchronize_jamtrack_master_preview(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name master_track = jam_track.master_track if master_track Dir.mktmpdir do |tmp_dir| ogg_44100 = File.join(tmp_dir, 'input.ogg') private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100) ogg_44100_digest = ::Digest::MD5.file(ogg_44100) if importer.synchronize_master_preview(master_track, tmp_dir, ogg_44100, ogg_44100_digest) importer.finish("success", nil) end end else importer.finish('no_master_track', nil) end importer end def synchronize_previews_dev importers = [] JamTrack.all.each do |jam_track| importers << synchronize_preview_dev(jam_track) end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" || importer.reason == "no_preview_start_time" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to import.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def import_click_track(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name importer.import_click_track(jam_track) importer end def generate_jmep(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name importer.generate_jmep(jam_track) importer end def import_click_tracks importers = [] JamTrack.all.each do |jam_track| #jam_track = JamTrack.find('126') importers << import_click_track(jam_track) end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to generate jmep.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def generate_jmeps importers = [] if is_tency_storage? licensor = JamTrackLicensor.find_by_name!('Tency Music') elsif is_paris_storage? licensor = JamTrackLicensor.find_by_name!('Paris Music') end JamTrack.where(licensor_id: licensor).each do |jam_track| importers << generate_jmep(jam_track) end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to generate jmep.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def synchronize_previews importers = [] JamTrack.all.each do |jam_track| importers << synchronize_preview(jam_track) end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" || importer.reason == "no_preview_start_time" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to import.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def synchronize_jamtrack_aac_previews importers = [] JamTrack.all.each do |jam_track| importers << synchronize_jamtrack_aac_preview(jam_track) end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to import.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def synchronize_jamtrack_master_previews importers = [] JamTrack.all.each do |jam_track| importers << synchronize_jamtrack_master_preview(jam_track) end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to import.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def synchronize_duration(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name master_track = jam_track.master_track if master_track Dir.mktmpdir do |tmp_dir| ogg_44100 = File.join(tmp_dir, 'input.ogg') private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100) if importer.synchronize_duration(jam_track, ogg_44100) jam_track.save! importer.finish("success", nil) end end else importer.finish('no_duration', nil) end importer end def synchronize_durations importers = [] JamTrack.all.each do |jam_track| importers << synchronize_duration(jam_track) end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to import.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def download_master(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name Dir.mkdir('tmp') unless Dir.exists?('tmp') Dir.mkdir('tmp/jam_track_masters') unless Dir.exists?('tmp/jam_track_masters') master_track = jam_track.master_track if master_track ogg_44100 = File.join('tmp/jam_track_masters', "#{jam_track.original_artist} - #{jam_track.name}.ogg") private_s3_manager.download(master_track.url_by_sample_rate(44), ogg_44100) end importer end def generate_mp3_aac_stem(jam_track) importer = JamTrackImporter.new importer.name = jam_track.name Dir.mktmpdir do |tmp_dir| audio_path = jam_track.metalocation[0...-"/meta.yml".length] importer.associate_tracks_with_original_stems(jam_track, audio_path) importer.generate_mp3_aac_stem(jam_track, tmp_dir, false) end importer end def download_masters importers = [] JamTrack.all.each do |jam_track| importers << download_master(jam_track) end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to download.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def remove_s3_special_chars(filename) filename.tr('/&@:,$=+?;\^`><{}[]#%~|', '') end def generate_mp3_aac_stems(format) importers = [] jam_tracks = [] tency = JamTrackLicensor.find_by_name('Tency Music') @@log.info("processing storage #{@storage_format}") if is_tency_storage? tency = JamTrackLicensor.find_by_name!('Tency Music') jam_tracks = JamTrack.where(licensor_id: tency.id) elsif is_paris_storage? paris = JamTrackLicensor.find_by_name!('Paris Music') jam_tracks = JamTrack.where(licensor_id: paris.id) elsif is_default_storage? # XXX IF WE ADD ANOTHER STORAGE, UPDATE THE WHERE TO EXCLUDE IT AS WELL jam_tracks = JamTrack.where('licensor_id is null OR licensor_id != ?', tency.id) else raise 'unknown storage format!' end jam_tracks.each do |jam_track| if ENV['NODE_COUNT'] node_count = ENV['NODE_COUNT'].to_i node_number = ENV['NODE_NUMBER'].to_i raise "NO NODE_COUNT" if node_count == 0 jam_track_id = jam_track.id.to_i jam_track_id = jam_track_id + node_number if jam_track_id == 0 @@log.warn("skipping #{jam_track_id} because non-numeric ID") next elsif jam_track_id % node_count == 0 @@log.warn("starting JamTrack #{jam_track.id} (#{jam_track_id})") importers << generate_mp3_aac_stem(jam_track) else @@log.warn("skipping #{jam_track_id}") next end else importers << generate_mp3_aac_stem(jam_track) end end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to download.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def generate_slugs JamTrack.all.each do |jam_track| jam_track.generate_slug jam_track.save! end end def onboarding_exceptions JamTrack.all.each do |jam_track| jam_track.onboarding_exceptions end end def synchronize_all(options) importers = [] count = 0 iterate_song_storage do |metadata, metalocation| next if metadata.nil? && (is_tency_storage? || is_paris_storage?) importer = synchronize_from_meta(metalocation, options) importers << importer if importer.reason != 'jam_track_exists' && importer.reason != "other_processing" count+=1 end if count > 500 #break end end @@log.info("SUMMARY") @@log.info("-------") importers.each do |importer| if importer if importer.reason == "success" || importer.reason == "jam_track_exists" || importer.reason == "other_processing" @@log.info("#{importer.name} #{importer.reason}") else @@log.error("#{importer.name} failed to import.") @@log.error("#{importer.name} reason=#{importer.reason}") @@log.error("#{importer.name} detail=#{importer.detail}") end else @@log.error("NULL IMPORTER") end end end def jam_track_dry_run(metalocation) # see if we can find a JamTrack with this metalocation jam_track = JamTrack.find_by_metalocation(metalocation) meta = load_metalocation(metalocation) if jam_track @@log.debug("jamtrack #{jam_track.name} located by metalocation") jam_track.dry_run(meta, metalocation) else jam_track = JamTrack.new jam_track.dry_run(meta, metalocation) end end def paris_genre_dump load_paris_mappings genres = {} @paris_metadata.each do |id, value| genre1 = value[:genre1] genre2 = value[:genre2] genre3 = value[:genre3] genres[genre1.downcase.strip] = genre1.downcase.strip if genre1 genres[genre2.downcase.strip] = genre2.downcase.strip if genre2 genres[genre3.downcase.strip] = genre3.downcase.strip if genre3 end all_genres = Genre.select(:id).all.map(&:id) all_genres = Set.new(all_genres) genres.each do |genre, value| found = all_genres.include? genre puts "#{genre}" unless found end end def create_importer_from_existing(jam_track) importer = JamTrackImporter.new(@storage_format) importer.name = jam_track.name importer.metadata = load_metalocation(jam_track.metalocation) importer end def resync_instruments(licensor) load_paris_mappings if @paris_mapping.nil? JamTrack.where(licensor_id: licensor.id).each do |jam_track| if @paris_metadata[jam_track.vendor_id].nil? next end puts "RESYNCING JAMTRACK #{jam_track.id}" JamTrackTrack.where(jam_track_id: jam_track.id).order(:position).each do |track| puts "BEFORE TRACK #{track.instrument_id} #{track.part}" end importer = create_importer_from_existing(jam_track) importer.reassign_instrument_parts(jam_track) #puts ">>>>>>>>> HIT KEY TO CONTINUE <<<<<<<<<<" #STDIN.gets end end def fix_artist_song_name (licensor) load_paris_mappings if @paris_mapping.nil? JamTrack.where(licensor_id: licensor.id).each do |jam_track| metadata = @paris_metadata[jam_track.vendor_id] if metadata.nil? puts "OH NO! A Paris Song that does not belong! #{jam_track.id} #{jam_track.vendor_id}" next end puts "STARTING JAM_TRACK #{jam_track.id} #{jam_track.original_artist} #{jam_track.name}" jam_track.generate_slug if jam_track.changed? puts "SLUG CHANGED! #{jam_track.changes.inspect}" end if !jam_track.save puts "dup slug!!!!: #{jam_track.id} #{jam_track.name} #{jam_track.original_artist}" end if jam_track.changed? jam_track.reload end jam_track.original_artist = metadata[:original_artist] jam_track.name = metadata[:name] if jam_track.changed? puts "ARTIST/NAME CHANGE: #{jam_track.changes.inspect}" end if !jam_track.save puts "unable to save new artist/song!" end end end def fix_slugs(licensor) JamTrack.where(licensor_id: licensor.id).each do |jam_track| if jam_track.slug.end_with?('-') puts "removing trailing dash" jam_track.slug = jam_track.slug[0...-1] if !jam_track.save puts "dup slug!!!!: #{jam_track.id} #{jam_track.name} #{jam_track.original_artist}" end end end end def missing_masters(licensor) count = 0 JamTrack.where(licensor_id: licensor.id).each do |jam_track| if jam_track.master_track.nil? puts "MISSING #{jam_track.metalocation}" count += 1 end end puts "missing master count: #{count}" end def tency_genre_dump load_tency_mappings genres = {} @tency_metadata.each do |id, value| genre1 = value[:genre1] genre2 = value[:genre2] genre3 = value[:genre3] genre4 = value[:genre4] genre5 = value[:genre5] genres[genre1.downcase.strip] = genre1.downcase.strip if genre1 genres[genre2.downcase.strip] = genre2.downcase.strip if genre2 genres[genre3.downcase.strip] = genre3.downcase.strip if genre3 genres[genre4.downcase.strip] = genre4.downcase.strip if genre4 genres[genre5.downcase.strip] = genre5.downcase.strip if genre5 end all_genres = Genre.select(:id).all.map(&:id) all_genres = Set.new(all_genres) genres.each do |genre, value| found = all_genres.include? genre puts "#{genre}" unless found end end def load_marks_approved @marks_approved = {} Dir.mktmpdir do |tmp_dir| mapping_file = 'marks_approved.csv' mapping_csv = CSV.read(mapping_file, headers: true, return_headers: false) mapping_csv.each do |line| approved = line[3] track_url = line[2] comments = line[7] approved.strip! if approved track_url.strip! if track_url comments.strip! if comments if approved == 'MJ' prefix = 'https://www.jamkazam.com/landing/jamtracks/'.length slug = track_url[prefix..-1] puts "MARKS APPROVED #{slug} #{comments}" @marks_approved[slug] = comments end end end end def load_paris_mappings Dir.mktmpdir do |tmp_dir| mapping_file = File.join(tmp_dir, 'mapping.csv') metadata_file = File.join(tmp_dir, 'metadata.csv') # this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv if ENV['PARIS_ALREADY_DOWNLOADED'] == '1' mapping_file = 'paris_mapping.csv' metadata_file = 'paris_metadata.csv' else paris_s3_manager.download('mapping/mapping.csv', mapping_file) paris_s3_manager.download('mapping/metadata.csv', metadata_file) end mapping_csv = CSV.read(mapping_file) metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false) @paris_mapping = {} @paris_metadata = {} # convert both to hashes mapping_csv.each do |line| instrument = line[1] instrument.strip! if instrument part = line[2] part.strip! if part @paris_mapping[line[0].strip.downcase] = {instrument: instrument, part: part} end metadata_csv.each do |line| paris_artist = line[2] # Paris artist in metadata file is often all caps artist = paris_artist.split(' ').collect do |item| if item == 'DJ' 'DJ' else item.titleize end end.join(' ') @paris_metadata[line[1].strip] = {id: line[1].strip, original_artist: artist, name: line[3], genre1: line[4], genre2: line[5], genre3: line[6]} end @paris_metadata.each do |id, value| genres = [] genre1 = value[:genre1] genre2 = value[:genre2] genre3 = value[:genre3] genres << genre1.downcase.strip if genre1 genres << genre2.downcase.strip if genre2 genres << genre3.downcase.strip if genre3 value[:genres] = genres end end end def load_tency_mappings Dir.mktmpdir do |tmp_dir| mapping_file = File.join(tmp_dir, 'mapping.csv') metadata_file = File.join(tmp_dir, 'metadata.csv') # this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv if ENV['TENCY_ALREADY_DOWNLOADED'] == '1' mapping_file = 'mapping.csv' metadata_file = 'metadata.csv' else tency_s3_manager.download('mapping/mapping.csv', mapping_file) tency_s3_manager.download('mapping/metadata.csv', metadata_file) end mapping_csv = CSV.read(mapping_file) metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false) @tency_mapping = {} @tency_metadata = {} # convert both to hashes mapping_csv.each do |line| @tency_mapping[line[0].strip] = {instrument: line[1], part: line[2], count: line[3], trust: line[4]} end metadata_csv.each do |line| @tency_metadata[line[0].strip] = {id: line[0].strip, original_artist: line[1], name: line[2], additional_info: line[3], year: line[4], language: line[5], isrc: line[10], genre1: line[11], genre2: line[12], genre3: line[13], genre4: line[14], genre5: line[15]} end @tency_metadata.each do |id, value| genres = [] genre1 = value[:genre1] genre2 = value[:genre2] genre3 = value[:genre3] genre4 = value[:genre4] genre5 = value[:genre5] genres << genre1.downcase.strip if genre1 genres << genre2.downcase.strip if genre2 genres << genre3.downcase.strip if genre3 genres << genre4.downcase.strip if genre4 genres << genre5.downcase.strip if genre5 value[:genres] = genres end end end def load_metalocation(metalocation) if is_tency_storage? load_tency_mappings if @tency_mapping.nil? song_id = extract_tency_song_id(metalocation) if song_id.nil? puts "missing_song_id #{metalocation}" return nil end tency_data = @tency_metadata[song_id] if tency_data.nil? @@log.warn("missing tency metadata '#{song_id}'") end return tency_data elsif is_paris_storage? load_paris_mappings if @paris_mapping.nil? song_id = extract_paris_song_id(metalocation) if song_id.nil? puts "missing_song_id #{metalocation}" return nil end paris_data = @paris_metadata[song_id] if paris_data.nil? @@log.warn("missing paris metadata '#{song_id}'") end return paris_data elsif is_drumma_storage? data = {} begin data = drumma_s3_manager.read_all(metalocation) rescue AWS::S3::Errors::NoSuchKey return {} end meta = YAML.load(data) meta[:genres] = ['r&b'] if !meta[:genres] meta else begin data = s3_manager.read_all(metalocation) meta = YAML.load(data) if is_tim_tracks_storage? meta[:genres] = ['acapella'] elsif is_helbing_storage? meta[:genres] = ['jazz'] end meta rescue AWS::S3::Errors::NoSuchKey return nil end end end def create_from_metalocation(meta, metalocation, options = {skip_audio_upload: false}) jam_track = JamTrack.new sync_from_metadata(jam_track, meta, metalocation, options) end def update_from_metalocation(jam_track, meta, metalocation, options) sync_from_metadata(jam_track, meta, metalocation, options) end def sync_from_metadata(jam_track, meta, metalocation, options) jam_track_importer = JamTrackImporter.new(@storage_format) jam_track_importer.metadata = meta JamTrack.connection.execute('SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED') JamTrack.transaction do #begin jam_track_importer.synchronize(jam_track, meta, metalocation, options) #rescue Exception => e # jam_track_importer.finish("unhandled_exception", e.to_s) #end if jam_track_importer.reason != "success" raise ActiveRecord::Rollback end end jam_track_importer end def synchronize_from_meta(metalocation, options) # see if we can find a JamTrack with this metalocation jam_track = JamTrack.find_by_metalocation(metalocation) meta = load_metalocation(metalocation) if meta.nil? && is_paris_storage? raise "no tency song matching this metalocation #{metalocation}" end jam_track_importer = nil if jam_track @@log.debug("jamtrack #{jam_track.name} located by metalocation") jam_track_importer = update_from_metalocation(jam_track, meta, metalocation, options) else jam_track_importer = create_from_metalocation(meta, metalocation, options) end if jam_track_importer.reason == "success" @@log.info("#{jam_track_importer.name} successfully imported") else @@log.error("#{jam_track_importer.name} failed to import.") @@log.error("#{jam_track_importer.name} reason=#{jam_track_importer.reason}") @@log.error("#{jam_track_importer.name} detail=#{jam_track_importer.detail}") end jam_track_importer end end end end