From cde055ca0dacbddafa01ec024440eb68d50187a7 Mon Sep 17 00:00:00 2001 From: Seth Call Date: Wed, 5 Aug 2015 10:45:41 -0500 Subject: [PATCH] * put up build that can generate master mixes for Tency --- admin/config/initializers/jam_tracks.rb | 1 - db/manifest | 3 +- db/up/jam_track_onboarding_enhancements.sql | 65 ++ ruby/lib/jam_ruby.rb | 2 + .../lib/jam_ruby/import/tency_stem_mapping.rb | 244 +++++- ruby/lib/jam_ruby/jam_track_importer.rb | 772 ++++++++++++++++-- ruby/lib/jam_ruby/models/genre.rb | 3 +- ruby/lib/jam_ruby/models/genre_jam_track.rb | 8 + ruby/lib/jam_ruby/models/jam_track.rb | 23 +- ruby/lib/jam_ruby/models/jam_track_file.rb | 78 ++ ruby/spec/factories.rb | 2 +- ruby/spec/jam_ruby/jam_track_importer_spec.rb | 102 ++- ruby/spec/jam_ruby/models/jam_track_spec.rb | 21 + web/app/views/api_jam_tracks/show.rabl | 2 +- .../views/api_jam_tracks/show_for_client.rabl | 6 +- web/lib/tasks/jam_tracks.rake | 17 +- web/spec/factories.rb | 2 +- 17 files changed, 1234 insertions(+), 117 deletions(-) create mode 100644 db/up/jam_track_onboarding_enhancements.sql create mode 100644 ruby/lib/jam_ruby/models/genre_jam_track.rb create mode 100644 ruby/lib/jam_ruby/models/jam_track_file.rb diff --git a/admin/config/initializers/jam_tracks.rb b/admin/config/initializers/jam_tracks.rb index cd02eccee..33efd995c 100644 --- a/admin/config/initializers/jam_tracks.rb +++ b/admin/config/initializers/jam_tracks.rb @@ -16,7 +16,6 @@ class JamRuby::JamTrack end def jmep_json_generate - self.genre_id = nil if self.genre_id == '' self.licensor_id = nil if self.licensor_id == '' self.jmep_json = nil if self.jmep_json == '' self.time_signature = nil if self.time_signature == '' diff --git a/db/manifest b/db/manifest index bd6ca8ce6..f63abf28a 100755 --- a/db/manifest +++ b/db/manifest @@ -296,4 +296,5 @@ add_description_to_perf_samples.sql alter_genre_player_unique_constraint.sql musician_search.sql enhance_band_profile.sql -alter_band_profile_rate_defaults.sql \ No newline at end of file +alter_band_profile_rate_defaults.sql +jam_track_onboarding_enhancements.sql \ No newline at end of file diff --git a/db/up/jam_track_onboarding_enhancements.sql b/db/up/jam_track_onboarding_enhancements.sql new file mode 100644 index 000000000..60d20eca9 --- /dev/null +++ b/db/up/jam_track_onboarding_enhancements.sql @@ -0,0 +1,65 @@ +UPDATE instruments SET id = 'double bass', description = 'Double Bass' WHERE id = 'upright bass'; +INSERT INTO instruments (id, description, popularity) VALUES ('steel guitar', 'Steel Guitar', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('orchestra', 'Orchestra', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('glockenspiel', 'Glockenspiel', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('dobro', 'Dobro', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('harp', 'Harp', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('vocoder', 'Vocoder', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('flugelhorn', 'Flugelhorn', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('timpani', 'Timpani', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('bassoon', 'Bassoon', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('charango', 'Charango', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('theremin', 'Theremin', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('sitar', 'Sitar', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('piccolo', 'Piccolo', 1); +INSERT INTO instruments (id, description, popularity) VALUES ('bagpipes', 'Bagpipes', 1); +ALTER TABLE jam_tracks ADD COLUMN onboarding_exceptions JSON; +ALTER TABLE jam_tracks ADD COLUMN original_filename VARCHAR; +ALTER TABLE jam_tracks ADD COLUMN additional_info VARCHAR; +ALTER TABLE jam_tracks ADD COLUMN language VARCHAR NOT NULL DEFAULT 'eng'; +ALTER TABLE jam_tracks ADD COLUMN year INTEGER; +ALTER TABLE jam_tracks ADD COLUMN vendor_id VARCHAR; + +INSERT INTO jam_track_licensors (name, description) VALUES ('Tency Music', 'Tency Music is a music production company specialized in re-recordings.'); + +CREATE TABLE genres_jam_tracks ( + id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(), + jam_track_id VARCHAR(64) NOT NULL REFERENCES jam_tracks(id) ON DELETE CASCADE, + genre_id VARCHAR(64) NOT NULL REFERENCES genres(id) ON DELETE CASCADE, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO genres_jam_tracks (jam_track_id, genre_id) ((SELECT jam_tracks.id, jam_tracks.genre_id FROM jam_tracks)); +ALTER TABLE jam_tracks DROP COLUMN genre_id; + +-- holds precount, click.wav, click.txt +CREATE TABLE jam_track_files ( + id VARCHAR(64) PRIMARY KEY DEFAULT uuid_generate_v4(), + jam_track_id VARCHAR(64) REFERENCES jam_tracks(id) ON DELETE CASCADE, + file_type VARCHAR NOT NULL, + original_filename VARCHAR NOT NULL, + precount_num INTEGER, + url VARCHAR, + md5 VARCHAR, + length bigint, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +INSERT INTO genres (id, description) VALUES ('soft rock', 'Soft Rock'); +INSERT INTO genres (id, description) VALUES ('rap', 'Rap'); +INSERT INTO genres (id, description) VALUES ('tv & movie soundtrack', 'TV & Movie Soundtrack'); +INSERT INTO genres (id, description) VALUES ('holiday', 'Holiday'); +INSERT INTO genres (id, description) VALUES ('kids', 'Kids'); +INSERT INTO genres (id, description) VALUES ('disco', 'Disco'); +INSERT INTO genres (id, description) VALUES ('soul', 'Soul'); +INSERT INTO genres (id, description) VALUES ('hard rock', 'Hard Rock'); +INSERT INTO genres (id, description) VALUES ('funk', 'Funk'); +INSERT INTO genres (id, description) VALUES ('dance', 'Dance'); +INSERT INTO genres (id, description) VALUES ('creole', 'Creole'); +INSERT INTO genres (id, description) VALUES ('traditional', 'Traditional'); +INSERT INTO genres (id, description) VALUES ('oldies', 'Oldies'); +INSERT INTO genres (id, description) VALUES ('world', 'World'); +INSERT INTO genres (id, description) VALUES ('musical', 'Musical'); +INSERT INTO genres (id, description) VALUES ('celtic', 'Celtic'); diff --git a/ruby/lib/jam_ruby.rb b/ruby/lib/jam_ruby.rb index 7b9ddc338..6b2d70a20 100755 --- a/ruby/lib/jam_ruby.rb +++ b/ruby/lib/jam_ruby.rb @@ -206,6 +206,8 @@ require "jam_ruby/models/jam_track" require "jam_ruby/models/jam_track_track" require "jam_ruby/models/jam_track_right" require "jam_ruby/models/jam_track_tap_in" +require "jam_ruby/models/jam_track_file" +require "jam_ruby/models/genre_jam_track" require "jam_ruby/app/mailers/async_mailer" require "jam_ruby/app/mailers/batch_mailer" require "jam_ruby/app/mailers/progress_mailer" diff --git a/ruby/lib/jam_ruby/import/tency_stem_mapping.rb b/ruby/lib/jam_ruby/import/tency_stem_mapping.rb index 389b424d2..769496562 100644 --- a/ruby/lib/jam_ruby/import/tency_stem_mapping.rb +++ b/ruby/lib/jam_ruby/import/tency_stem_mapping.rb @@ -12,12 +12,226 @@ module JamRuby def initialize @originals_folder = "/Volumes/sethcall/Dropbox/seth@jamkazam.com/JamTracks - Tency Music - Original Folder for Normalization Map" @mapping_folder = "/Volumes/sethcall/Dropbox/seth@jamkazam.com/JamTracks - Tency Music" - @originals_songs = {} + @original_songs = {} @mapping_songs = {} + @mappings = {} end def create_map tency_originals + tency_maps + + dump + end + + def create_mapping_map + tency_maps + + dump_map + end + + def hydrate + @original_songs = YAML.load_file('original_songs.yml') + @mapping_songs = YAML.load_file('mapping_songs.yml') + end + + def parse_sanitized_filename(filename) + instrument = nil + part = nil + + basename = File.basename(filename) + stem = basename.index('Stem') + + if stem + stripped = basename[(stem + 'Stem'.length)..-5] # takes of 'stem' and '.wav' + stripped.strip! + dash = stripped.index('-') + + if dash == 0 + stripped = stripped[1..-1].strip! + # now we should have something like "Vocal - Lead" (instrument - part) + instrument, part = stripped.split('-') + instrument.strip! if instrument + part.strip! if part + else + "no or misplaced dash for #{filename}" + end + + else + raise "no stem for #{filename}" + end + + [instrument, part] + end + + # For all the tracks that I have labeled manually as + # Instrument = Upright Bass and Part = Upright Bass, + # can you please change both the Instrument and Part to Double Bass instead? + # + def check_mappings + missing_instrument = 0 + missing_part = 0 + part_names = [] + + hydrate + @mapping_songs.each do |cache_id, data| + mapped_filename = data[:filename] + @@log.debug("parsing #{mapped_filename}") + instrument, part = parse_sanitized_filename(mapped_filename) + @@log.debug("parsed #{instrument} (#{part})") + missing_instrument = missing_instrument + 1 unless instrument + missing_part = missing_part + 1 unless part + part_names << mapped_filename unless part + end + + @@log.info("SUMMARY") + @@log.info("-------") + @@log.info("missing instruments:#{missing_instrument} missing parts: #{missing_part}") + @@log.info("files with no parts: #{part_names}") + + # files with no parts: + # ["Huey Lewis And The News - Heart And Soul - 31957/Heart And Soul Stem - Synth 2.wav", + # "ZZ Top - Tush - 20852/Tush Stem - Clicktrack.wav", + # "Crosby Stills And Nash - Teach Your Children - 15440/Teach Your Children Stem - Bass Guitar.wav", + # /Brad Paisley - She's Everything - 19886/She's Everything Stem - Clicktrack.wav", + # "Toby Keith - Beer For My Horses - 7221/Beer For My Horses Stem - Lap Steel.wav", + # Toby Keith - Beer For My Horses - 7221/Beer For My Horses Stem - Acoustic Guitar.wav" + + end + + def track_mapping(basename, instr_part) + instrument = instr_part[:instrument] + part = instr_part[:part] + + basename.downcase! + + info = @mappings[basename] + + unless info + info = {matches:[]} + @mappings[basename] = info + end + + info[:matches] << instr_part + end + + def correlate + mapped = 0 + unmapped = 0 + unmapped_details = [] + no_instrument = [] + common_unknown_instruments = {} + + hydrate + @mapping_songs.each do |cache_id, data| + # go through each track hand-mapped, and find it's matching song if any. + + mapped_filename = data[:filename] + found_original = @original_songs[cache_id] + if found_original + # mapping made + + original_filename = found_original[:filename] + original_basename = File.basename(original_filename).downcase + + mapped = mapped + 1 + + instrument, part = parse_sanitized_filename(mapped_filename) + instr_part = JamTrackImporter.determine_instrument(instrument, part) + + instr_part[:instrument] + + if instr_part[:instrument] + + # track the mapping of this one + track_mapping(original_basename, instr_part) + + else + @@log.error("unable to determine instrument for #{File.basename(mapped_filename)}") + no_instrument << ({filename: File.basename(mapped_filename), instrument: instrument, part: part}) + common_unknown_instruments["#{instrument}-(#{part})"] = 1 + end + + else + unmapped = unmapped + 1 + unmapped_details << {filename: mapped_filename} + end + end + + puts("SUMMARY") + puts("-------") + puts("MAPPED:#{mapped} UNMAPPED:#{unmapped}") + unmapped_details.each do |unmapped_detail| + puts "UNMAPPED FILE: #{File.basename(unmapped_detail[:filename])}" + end + puts("UNKNOWN INSTRUMENT: #{no_instrument.length}") + no_instrument.each do |item| + puts("UNKNOWN INSTRUMENT: #{item[:filename]}") + end + common_unknown_instruments.each do |key, value| + puts("#{key}") + end + @mappings.each do |basename, mapping| + matches = mapping[:matches] + counts = matches.each_with_object(Hash.new(0)) { |word,counts| counts[word] += 1 } + ordered_matches = counts.sort_by {|k, v| -v} + output = "" + ordered_matches.each do |match| + detail = match[0] + count = match[1] + output << "#{detail[:instrument]}(#{detail[:part]})/#{count}, " + end + + puts "map detail: #{basename}: #{output}" + + mapping[:ordered] = ordered_matches + mapping[:detail] = output + end + CSV.open("mapping.csv", "wb") do |csv| + @mappings.each do |basename, mapping| + item = mapping[:ordered] + + trust_worthy = item.length == 1 + unless trust_worthy + # if the 1st item is at least 4 'counts' more than the next item, we can consider it trust_worthy + if item[0][1] - 4 > item[1][1] + trust_worthy = true + end + end + csv << [ basename, item[0][0][:instrument], item[0][0][:part], item[0][1], trust_worthy ] + end + end + CSV.open("determinate-single-matches.csv", "wb") do |csv| + @mappings.each do |basename, mapping| + if mapping[:ordered].length == 1 && mapping[:ordered][0][1] == 1 + item = mapping[:ordered] + csv << [ basename, item[0][0][:instrument], item[0][0][:part], item[0][1] ] + end + end + end + CSV.open("determinate-multi-matches.csv", "wb") do |csv| + @mappings.each do |basename, mapping| + if mapping[:ordered].length == 1 && mapping[:ordered][0][1] > 1 + item = mapping[:ordered] + csv << [ basename, item[0][0][:instrument], item[0][0][:part], item[0][1] ] + end + end + end + CSV.open("ambiguous-matches.csv", "wb") do |csv| + @mappings.each do |basename, mapping| + if mapping[:ordered].length > 1 + csv << [ basename, mapping[:detail] ] + end + end + end + end + + def dump + File.open('original_songs.yml', 'w') {|f| f.write(YAML.dump(@original_songs)) } + File.open('mapping_songs.yml', 'w') {|f| f.write(YAML.dump(@mapping_songs)) } + end + def dump_map + File.open('mapping_songs.yml', 'w') {|f| f.write(YAML.dump(@mapping_songs)) } end def md5(filepath) @@ -36,43 +250,47 @@ module JamRuby @@log.debug "" end end + def tency_originals songs = Pathname.new(@originals_folder).children.select { |c| c.directory? } songs.each do |filename| id = parse_id(filename.basename.to_s ) - tracks = Pathname.new(filename).children.select {|c| c.file? } + files = Pathname.new(filename).children.select {|c| c.file? } # also look into any 1st level folders we might find dirs = Pathname.new(filename).children.select {|c| c.directory? } dirs.each do |dir| more_tracks = Pathname.new(dir).children.select {|c| c.file? } - tracks = tracks + more_tracks + files = files + more_tracks end - puts "tracks #{tracks.inspect}" - break + files.each do |file| + @@log.debug("processing original track #{file.to_s}") + md5 = md5(file.to_s) + song = {md5:md5, filename:file.to_s, id:id} + @original_songs[cache_id(id, md5)] = song + end end end def tency_maps songs = Pathname.new(@mapping_folder).children.select { |c| c.directory? } - songs.each do |filename| - id = parse_id_mapped(filename.basename.to_s ) - @@log.debug "processing song #{filename}" + songs.each do |song_filename| + id = parse_id_mapped(song_filename.basename.to_s ) + @@log.debug "processing song #{song_filename.to_s}" - tracks = Pathname.new(filename).children.select {|c| c.file? } + tracks = Pathname.new(song_filename).children.select {|c| c.file? } tracks.each do |track| - if track.include? "Stem" + if track.to_s.include? "Stem" + @@log.debug("processing mapped track #{track.to_s}") md5 = md5(track.to_s) - song = {md5:md5, filename:filename} + song = {md5:md5, filename:track.to_s} @mapping_songs[cache_id(id, md5)] = song end end - - break end end diff --git a/ruby/lib/jam_ruby/jam_track_importer.rb b/ruby/lib/jam_ruby/jam_track_importer.rb index dcf12fa17..c7c39d25e 100644 --- a/ruby/lib/jam_ruby/jam_track_importer.rb +++ b/ruby/lib/jam_ruby/jam_track_importer.rb @@ -14,6 +14,7 @@ module JamRuby attr_accessor :name attr_accessor :reason attr_accessor :detail + attr_accessor :storage_format def jamkazam_s3_manager @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) @@ -23,11 +24,109 @@ module JamRuby @public_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_public, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end + def initialize(storage_format = 'default') + @storage_format = storage_format + end + def finish(reason, detail) self.reason = reason self.detail = detail end + + # this method was created due to Tency-sourced data having no master track + def create_master(metadata, metalocation) + + parsed_metalocation = parse_metalocation(metalocation) + + if parsed_metalocation.nil? + finish("invalid_metalocation", metalocation) + return + end + + original_artist = parsed_metalocation[1] + meta_name = parsed_metalocation[2] + + self.name = metadata[:name] || meta_name + + + audio_path = metalocation[0...-"/meta.yml".length] + + all_files = fetch_important_files(audio_path) + + audio_files = [] + master_found = false + all_files.each do |file| + + parsed_wav = parse_file(file) + if parsed_wav[:master] + master_found = true + elsif parsed_wav[:type] == :track + + audio_files << file + end + end + + if master_found + @@log.debug("master exists... skipping #{self.name} ") + finish('success', nil) + else + + tracks = [] + + # XXX REMOVE - so i don't have to re-downloading 500mb every time I try to test this code + # tmp_dir = Dir.mktmpdir + Dir.mktmpdir do |tmp_dir| + @@log.debug("downloading all audio files in #{tmp_dir}") + audio_files.each do |s3_track| + track = File.join(tmp_dir, File.basename(s3_track)) + tracks << track + JamTrackImporter.song_storage_manager.download(s3_track, track) + end + + temp_file = File.join(tmp_dir, "temp.wav") + output_filename = "#{self.name} Master Mix.wav" + output_file = File.join(tmp_dir, output_filename) + command = "sox -m " + tracks.each do |track| + command << " \"#{track}\"" + end + command << " \"#{temp_file}\"" + + @@log.debug("mixing with cmd: " + command) + sox_output = `#{command}` + result_code = $?.to_i + + if result_code != 0 + @@log.error("unable to generate master mix") + finish("sox_master_mix_failure", sox_output) + else + + # now normalize the audio + + command = "sox --norm \"#{temp_file}\" \"#{output_file}\"" + @@log.debug("normalizing with cmd: " + command) + sox_output = `#{command}` + result_code = $?.to_i + if result_code != 0 + @@log.error("unable to normalize master mix") + finish("sox_master_mix_failure", sox_output) + else + + # now we need to upload the output back up + s3_target = audio_path + '/' + output_filename + @@log.debug("uploading #{output_file} to #{s3_target}") + JamTrackImporter.song_storage_manager.upload(s3_target, output_file ) + finish('success', nil) + end + + end + end + + end + + end + def dry_run(metadata, metalocation) metadata ||= {} @@ -42,31 +141,85 @@ module JamRuby return unless success - dry_run_audio(metadata, "audio/#{original_artist}/#{name}") + audio_path = metalocation[0...-"/meta.yml".length] + + dry_run_audio(metadata, audio_path) finish("success", nil) end + def is_tency_storage? + assert_storage_set + @storage_format == 'Tency' + end + + def assert_storage_set + raise "no storage_format set" if @storage_format.nil? + end + + def parse_metalocation(metalocation) + # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml + if is_tency_storage? - bits = metalocation.split('/') + suffix = '/meta.yml' - if bits.length != 4 - finish("invalid_metalocation", "metalocation not valid #{metalocation}") - return nil + unless metalocation.end_with? suffix + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + metalocation = metalocation[0...-suffix.length] + + first_path = metalocation.index('/') + if first_path.nil? + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + metalocation = metalocation[(first_path + 1)..-1] + + bits = ['audio'] + # example: Sister Hazel - All For You - 10385 + first_dash = metalocation.index('-') + if first_dash + artist = metalocation[0...first_dash].strip + bits << artist + else + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + last_dash = metalocation.rindex('-') + if last_dash + song = metalocation[(first_dash+1)...last_dash].strip + bits << song + else + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + bits << 'meta.yml' + bits + else + bits = metalocation.split('/') + + if bits.length != 4 + finish("invalid_metalocation", "metalocation not valid #{metalocation}") + return nil + end + + if bits[0] != "audio" + finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}") + return nil + end + + if bits[3] != 'meta.yml' + finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}") + return nil + end + + bits end - - if bits[0] != "audio" - finish("invalid_metalocation", "first bit is not 'audio' #{metalocation}") - return nil - end - - if bits[3] != 'meta.yml' - finish('invalid_metalocation', "last bit is not 'meta.yml' #{metalocation}") - return nil - end - - bits end # if you change this, it will (at least without some work )break development usage of jamtracks @@ -91,6 +244,78 @@ module JamRuby true end + def determine_genres(metadata) + genres = [] + if metadata[:genres] + metadata[:genres].each do |genre| + if genre == 'hard/metal' + genres << Genre.find('hard rock') + genres << Genre.find('metal') + elsif genre == 'christmas' + genres << Genre.find('holiday') + elsif genre == 'alternative' + genres << Genre.find('alternative rock') + elsif genre == '80s' + # swallow + elsif genre == 'love' + # swallow + elsif genre == 'christian' || genre == 'gospel' + genres << Genre.find('religious') + elsif genre == 'punk/grunge' + genres << Genre.find('punk') + elsif genre == 'electro' + genres << Genre.find('electronic') + elsif genre == 'teen pop' + genres << Genre.find('pop') + elsif genre == "rock 'n roll" + genres << Genre.find('rock') + elsif genre == 'zouk/creole' + genres << Genre.find('creole') + elsif genre == 'world/folk' + genres << Genre.find('world') + genres << Genre.find('folk') + elsif genre == 'french pop' + # swallow + elsif genre == 'schlager' + #swallow + elsif genre == 'humour' + # swallow + elsif genre == 'oriental' + genres << genre.find('asian') + end + end + end + + genres + end + + def determine_language(metadata) + + found = ISO_639.find_by_code('eng') + + language = metadata[:language] + + if language + language.downcase! + + if language == 'instrumental' + return 'instrumental' + end + + if language.include? 'spanish' + found = ISO_639.find_by_code('spa') + elsif language.include? 'german' + found = ISO_639.find_by_code('ger') + elsif language.include? 'portuguese' + found = ISO_639.find_by_code('por') + elsif language.include? 'english' + found = ISO_639.find_by_code('eng') + end + end + + found[0] # 3 letter code + end + def synchronize_metadata(jam_track, metadata, metalocation, original_artist, name, options) metadata ||= {} @@ -104,7 +329,10 @@ module JamRuby jam_track.metalocation = metalocation jam_track.original_artist = metadata["original_artist"] || original_artist jam_track.name = self.name - jam_track.genre_id = 'rock' + jam_track.additional_info = metadata[:additional_info] + jam_track.year = metadata[:year] + jam_track.genres = determine_genres(metadata) + jam_track.language = determine_language(metadata) jam_track.plan_code = metadata["plan_code"] || gen_plan_code(jam_track.original_artist, jam_track.name) jam_track.price = 1.99 jam_track.reproduction_royalty_amount = 0 @@ -114,6 +342,11 @@ module JamRuby jam_track.sales_region = 'Worldwide' jam_track.recording_type = 'Cover' jam_track.description = "This is a JamTrack audio file for use exclusively with the JamKazam service. This JamTrack is a high quality cover of the #{jam_track.original_artist} song \"#{jam_track.name}\"." + + if is_tency_storage? + jam_track.vendor_id = metadata[:id] + jam_track.licensor = JamTrackLicensor.find_by_name('Tency Music') + end else if !options[:resync_audio] #@@log.debug("#{self.name} skipped because it already exists in database") @@ -169,12 +402,18 @@ module JamRuby else instrument = 'electric guitar' end - elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' + elsif potential_instrument == 'acoustic' + instrument = 'acoustic guitar' + elsif potential_instrument == 'acoutic guitar' + instrument = 'electric guitar' + elsif potential_instrument == 'electric gutiar' || potential_instrument == 'electric guitat' || potential_instrument == 'electric guitary' instrument = 'electric guitar' elsif potential_instrument == 'keys' instrument = 'keyboard' elsif potential_instrument == 'vocal' || potential_instrument == 'vocals' instrument = 'voice' + elsif potential_instrument == 'upright bass' + instrument = 'double bass' elsif potential_instrument == 'bass' instrument = 'bass guitar' elsif potential_instrument == 'drum' @@ -187,8 +426,9 @@ module JamRuby else part = 'Sound FX' end - - + elsif potential_instrument == 'computer scratches' + instrument = 'computer' + part = 'Scratches' elsif potential_instrument == "sax" instrument = 'saxophone' elsif potential_instrument == "vocal back up" @@ -215,18 +455,43 @@ module JamRuby elsif potential_instrument == 'fretless bass' instrument = 'bass guitar' part = 'Fretless' + elsif potential_instrument == 'lap steel' || potential_instrument == 'pedal steel' + instrument = 'Steel Guitar' elsif potential_instrument == 'clock percussion' instrument = 'computer' part = 'Clock' - elsif potential_instrument == 'horns' + elsif potential_instrument == 'horns' || potential_instrument == 'horn' instrument = 'other' part = 'Horns' - elsif potential_instrument == 'strings' + elsif potential_instrument == 'english horn' instrument = 'other' + part = 'English Horn' + elsif potential_instrument == 'bass clarinet' + instrument = 'other' + part = 'Bass Clarinet' + elsif potential_instrument == 'recorder' + instrument = 'other' + part = 'Recorder' + elsif potential_instrument == 'marimba' + instrument = 'keyboard' + part = 'Marimba' + elsif potential_instrument == 'strings' + instrument = 'orchestra' part = 'Strings' - elsif potential_instrument == 'orchestration' - instrument = 'computer' - part = 'Orchestration' + elsif potential_instrument == 'celesta' + instrument = 'keyboard' + elsif potential_instrument == 'balalaika' + instrument = 'other' + part = 'Balalaika' + elsif potential_instrument == 'tanpura' + instrument = 'other' + part = 'Tanpura' + elsif potential_instrument == 'quena' + instrument = 'other' + part = 'Quena' + elsif potential_instrument == 'bouzouki' + instrument = 'other' + part = 'Bouzouki' elsif potential_instrument == 'claps' || potential_instrument == 'hand claps' instrument = 'computer' part = 'Claps' @@ -248,20 +513,35 @@ module JamRuby end - def parse_wav(file) + def parse_file(file) bits = file.split('/') filename = bits[bits.length - 1] # remove all but just the filename filename_no_ext = filename[0..-5] comparable_filename = filename_no_ext.downcase # remove .wav + type = nil master = false instrument = nil part = nil + precount_num = nil - if comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") + if comparable_filename == "click" || comparable_filename.include?("clicktrack") + if filename.end_with?('.txt') + type = :clicktxt + else + type = :clickwav + end + elsif comparable_filename.include? "precount" + type = :precount + index = comparable_filename.index('precount') + precount = comparable_filename[(index + 'precount'.length)..-1].strip + precount_num = precount.to_i unless precount.to_i == 0 + elsif comparable_filename.include?("master mix") || comparable_filename.include?("mastered mix") master = true + type = :master else + type = :track stem_location = comparable_filename.index('stem -') unless stem_location stem_location = comparable_filename.index('stems -') @@ -296,72 +576,160 @@ module JamRuby result = determine_instrument(possible_instrument, possible_part) instrument = result[:instrument] part = result[:part] + else + if is_tency_storage? + # we can check to see if we can find mapping info for this filename + mapping = JamTrackImporter.tency_mapping[filename.downcase] + + if mapping && mapping[:trust] + instrument = mapping[:instrument] + part = mapping[:part] + end + end end + end - {filename: filename, master: master, instrument: instrument, part: part} + {filename: filename, master: master, instrument: instrument, part: part, type: type, precount_num: precount_num} end def dry_run_audio(metadata, s3_path) - all_files = fetch_wav_files(s3_path) + all_files = fetch_important_files(s3_path) all_files.each do |file| - if file.end_with?('.wav') - parsed_wav = parse_wav(file) - if parsed_wav[:master] - @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") - else - if !parsed_wav[:instrument] || !parsed_wav[:part] - @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") - else - @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + + # ignore click/precount + parsed_wav = parse_file(file) + if parsed_wav[:master] + @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") + elsif parsed_wav[:type] == :track + + JamTrackImporter.summaries[:total_tracks] += 1 + + if parsed_wav[:instrument].nil? + detail = JamTrackImporter.summaries[:no_instrument_detail] + file_detail = detail[parsed_wav[:filename].downcase] + if file_detail.nil? + detail[parsed_wav[:filename].downcase] = 0 end + detail[parsed_wav[:filename].downcase] += 1 + + JamTrackImporter.summaries[:no_instrument] += 1 + end + + JamTrackImporter.summaries[:no_part] += 1 if parsed_wav[:part].nil? + + if !parsed_wav[:instrument] || !parsed_wav[:part] + @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + else + @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + end + elsif parsed_wav[:type] == :clickwav + + elsif parsed_wav[:type] == :clicktxt + + elsif parsed_wav[:type] == :precount + + else + JamTrackImporter.summaries[:unknown_filetype] += 1 + end + end + end + + + def set_custom_weight(track) + + slop = 800 + + instrument_weight = nil + # if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end + + if track.persisted? + instrument_weight = track.position + else + if track.instrument_id == 'voice' + + if track.part && track.part.start_with?('Lead') + instrument_weight = 100 + elsif track.part && track.part.start_with?('Backing') + instrument_weight = 110 + else + instrument_weight = 120 + end + + elsif track.instrument_id == 'drums' + + if track.part && track.part == 'Drums' + instrument_weight = 150 + elsif track.part && track.part == 'Percussion' + instrument_weight = 160 + else + instrument_weight = 170 + end + + elsif track.instrument_id == 'bass guitar' && track.part && track.part == 'Bass' + instrument_weight = 180 + + elsif track.instrument_id == 'piano' && track.part && track.part == 'Piano' + instrument_weight = 250 + + elsif track.instrument_id == 'keyboard' + + if track.part && track.part.start_with?('Synth') + instrument_weight = 260 + elsif track.part && track.part.start_with?('Pads') + instrument_weight = 270 + else + instrument_weight = 280 + end + + elsif track.instrument_id == 'acoustic guitar' + if track.part && track.part.start_with?('Lead') + instrument_weight = 300 + elsif track.part && track.part.start_with?('Rhythm') + instrument_weight = 310 + else + instrument_weight = 320 + end + elsif track.instrument_id == 'electric guitar' + if track.part && track.part.start_with?('Lead') + instrument_weight = 400 + elsif track.part && track.part.start_with?('Solo') + instrument_weight = 410 + elsif track.part && track.part.start_with?('Rhythm') + instrument_weight = 420 + else + instrument_weight = 440 end else - @@log.debug("#{self.name} ignoring non-wav file #{file}") + instrument_weight = slop + end + + if track.track_type == 'Master' + instrument_weight = 1000 end end + + instrument_weight end def sort_tracks(tracks) - def set_custom_weight(track) - weight = 5 - # if there are any persisted tracks, do not sort from scratch; just stick new stuff at the end - - if track.persisted? - weight = track.position - else - case track.instrument_id - when 'electric guitar' - weight = 100 - when 'acoustic guitar' - weight = 200 - when 'drums' - weight = 300 - when 'keys' - weight = 400 - when 'computer' - weight = 600 - else - weight = 500 - end - if track.track_type == 'Master' - weight = 1000 - end - end - - - weight - end - sorted_tracks = tracks.sort do |a, b| a_weight = set_custom_weight(a) b_weight = set_custom_weight(b) - a_weight <=> b_weight + if a_weight != b_weight + a_weight <=> b_weight + elsif a.instrument_id != b.instrument_id + a.instrument_id <=> b.instrument_id + else + a_part = a.part + b_part = b.part + a_part <=> b_part + end end # default to 1, but if there are any persisted tracks, this will get manipulated to be +1 the highest persisted track @@ -389,9 +757,10 @@ module JamRuby attempt_to_match_existing_tracks = true # find all wav files in the JamTracks s3 bucket - wav_files = fetch_wav_files(s3_path) + wav_files = fetch_important_files(s3_path) tracks = [] + addt_files = [] wav_files.each do |wav_file| @@ -421,28 +790,46 @@ module JamRuby @@log.debug("no existing track found; creating a new one") track = JamTrackTrack.new - track.original_audio_s3_path = wav_file + track.original_filename = wav_file - parsed_wav = parse_wav(wav_file) + file = JamTrackFile.new + file.original_filename = wav_file + parsed_wav = parse_file(wav_file) + + unknowns = 0 if parsed_wav[:master] track.track_type = 'Master' track.part = 'Master Mix' track.instrument_id = 'computer' + tracks << track @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") - else + elsif parsed_wav[:type] == :track + if !parsed_wav[:instrument] || !parsed_wav[:part] @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + unknowns += 1 else @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") end + track.instrument_id = parsed_wav[:instrument] || 'other' track.track_type = 'Track' - track.part = parsed_wav[:part] || 'Other' + track.part = parsed_wav[:part] || "Other #{unknowns}" + tracks << track + elsif parsed_wav[:type] == :clicktxt + file.file_type = 'ClickTxt' + elsif parsed_wav[:type] == :clickwav + file.file_type = 'ClickWav' + elsif parsed_wav[:type] == :precount + file.file_type = 'Precount' + file.precount_num = parsed_wav[:precount_num] + else + finish("unknown_file_type", "unknown file type #{wave_file}") + return false end - tracks << track end jam_track.jam_track_tracks.each do |jam_track_track| @@ -453,10 +840,18 @@ module JamRuby end end + jam_track.jam_track_files.each do |jam_track_file| + unless addt_files.include?(jam_track_file) + @@log.info("destroying removed JamTrackFile #{jam_track_file.inspect}") + jam_track_file.destroy # should also delete s3 files associated with this jamtrack + end + end + @@log.info("sorting tracks") tracks = sort_tracks(tracks) jam_track.jam_track_tracks = tracks + jam_track.jam_track_files = addt_files saved = jam_track.save @@ -642,12 +1037,12 @@ module JamRuby end def fetch_all_files(s3_path) - JamTrackImporter::s3_manager.list_files(s3_path) + JamTrackImporter::song_storage_manager.list_files(s3_path) end - def fetch_wav_files(s3_path) + def fetch_important_files(s3_path) files = fetch_all_files(s3_path) - files.select { |file| file.end_with?('.wav') } + files.select { |file| file.end_with?('.wav') || file.end_with?('.txt') } end def synchronize(jam_track, metadata, metalocation, options) @@ -667,7 +1062,9 @@ module JamRuby return unless success - synchronized_audio = synchronize_audio(jam_track, metadata, "audio/#{original_artist}/#{name}", options[:skip_audio_upload]) + audio_path = metalocation[0...-"/meta.yml".length] + + synchronized_audio = synchronize_audio(jam_track, metadata, audio_path, options[:skip_audio_upload]) return unless synchronized_audio @@ -693,16 +1090,143 @@ module JamRuby class << self + attr_accessor :storage_format + attr_accessor :tency_mapping + attr_accessor :summaries + + def report_summaries + @@log.debug("SUMMARIES DUMP") + @@log.debug("--------------") + @summaries.each do |k, v| + + if k == :no_instrument_detail + @@log.debug("#{k}: #{v}") + else + @@log.debug("#{k}: #{v}") + end + end + end + + def song_storage_manager + if is_tency_storage? + tency_s3_manager + else + s3_manager + end + end + + def summaries + @summaries ||= {unknown_filetype: 0, no_instrument: 0, no_part: 0, total_tracks: 0, no_instrument_detail: {}} + end + + def tency_s3_manager + @tency_s3_manager ||= S3Manager.new('jamkazam-tency', APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) + end + def s3_manager @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket_jamtracks, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end def private_s3_manager - @s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) + @private_s3_manager ||= S3Manager.new(APP_CONFIG.aws_bucket, APP_CONFIG.aws_access_key_id, APP_CONFIG.aws_secret_access_key) end + def extract_tency_song_id(metalocation) + # metalocation = mapped/4 Non Blondes - What's Up - 6475/meta.yml + + first_path = metalocation.index('/') + return nil unless first_path + metalocation = metalocation[(first_path + 1)..-1] + + suffix = '/meta.yml' + metalocation = metalocation[0...-suffix.length] + + last_dash = metalocation.rindex('-') + return nil if last_dash.nil? + + id = metalocation[(last_dash+1)..-1].strip + + return nil if id.to_i == 0 + + id + end + + def is_tency_storage? + assert_storage_set + @storage_format == 'Tency' + end + + def assert_storage_set + raise "no storage_format set" if @storage_format.nil? + end + + def iterate_tency_song_storage(&blk) + count = 0 + song_storage_manager.list_directories('mapped').each do |song| + @@log.debug("searching through song directory '#{song}'") + + metalocation = "#{song}meta.yml" + + metadata = load_metalocation(metalocation) + + blk.call(metadata, metalocation) + + count += 1 + #break if count > 100 + + end + end + + def iterate_default_song_storage(&blk) + song_storage_manager.list_directories('audio').each do |original_artist| + @@log.debug("searching through artist directory '#{original_artist}'") + + songs = song_storage_manager.list_directories(original_artist) + songs.each do |song| + @@log.debug("searching through song directory' #{song}'") + + metalocation = "#{song}meta.yml" + + metadata = load_metalocation(metalocation) + + blk.call(metadata, metalocation) + end + end + end + + def iterate_song_storage(&blk) + if is_tency_storage? + iterate_tency_song_storage do |metadata, metalocation| + blk.call(metadata, metalocation) + end + else + iterate_default_song_storage do |metadata, metalocation| + blk.call(metadata, metalocation) + end + end + end def dry_run + iterate_song_storage do |metadata, metalocation| + jam_track_importer = JamTrackImporter.new(@storage_format) + + jam_track_importer.dry_run(metadata, metalocation) + end + + report_summaries + end + + def create_masters + iterate_song_storage do |metadata, metalocation| + jam_track_importer = JamTrackImporter.new(@storage_format) + + jam_track_importer.create_master(metadata, metalocation) + + break + end + end + + def dry_run_original s3_manager.list_directories('audio').each do |original_artist| @@log.debug("searching through artist directory '#{original_artist}'") @@ -959,12 +1483,86 @@ module JamRuby end end + def genre_dump + load_tency_mappings + + genres = {} + @tency_metadata.each do |id, value| + + genre1 = value[:genre1] + genre2 = value[:genre2] + genre3 = value[:genre3] + genre4 = value[:genre4] + genre5 = value[:genre5] + + genres[genre1.downcase.strip] = genre1.downcase.strip if genre1 + genres[genre2.downcase.strip] = genre2.downcase.strip if genre2 + genres[genre3.downcase.strip] = genre3.downcase.strip if genre3 + genres[genre4.downcase.strip] = genre4.downcase.strip if genre4 + genres[genre5.downcase.strip] = genre5.downcase.strip if genre5 + end + + all_genres = Genre.select(:id).all.map(&:id) + + all_genres = Set.new(all_genres) + genres.each do |genre, value| + found = all_genres.include? genre + + puts "#{genre}" unless found + end + end + + def load_tency_mappings + Dir.mktmpdir do |tmp_dir| + mapping_file = File.join(tmp_dir, 'mapping.csv') + metadata_file = File.join(tmp_dir, 'metadata.csv') + + # this is a developer option to skip the download and look in the CWD to grab mapping.csv and metadata.csv + if ENV['TENCY_ALREADY_DOWNLOADED'] == '1' + mapping_file = 'mapping.csv' + metadata_file = 'metadata.csv' + else + tency_s3_manager.download('mapping/mapping.csv', mapping_file) + tency_s3_manager.download('mapping/metadata.csv', metadata_file) + end + + mapping_csv = CSV.read(mapping_file) + metadata_csv = CSV.read(metadata_file, headers: true, return_headers: false) + + @tency_mapping = {} + @tency_metadata = {} + # convert both to hashes + mapping_csv.each do |line| + @tency_mapping[line[0]] = {instrument: line[1], part: line[2], count: line[3], trust: line[4]} + end + + metadata_csv.each do |line| + @tency_metadata[line[0]] = {id: line[0], original_artist: line[1], name: line[2], additional_info: line[3], year: line[4], language: line[5], isrc: line[10], genre1: line[11], genre2: line[12], genre3: line[13], genre4: line[14], genre5: line[15]} + end + end + end + def load_metalocation(metalocation) - begin - data = s3_manager.read_all(metalocation) - return YAML.load(data) - rescue AWS::S3::Errors::NoSuchKey - return nil + + if is_tency_storage? + load_tency_mappings if @tency_mapping.nil? + song_id = extract_tency_song_id(metalocation) + + if song_id.nil? + puts "missing_song_id #{metalocation}" + return nil + end + + tency_data = @tency_metadata[song_id] + + return tency_data + else + begin + data = s3_manager.read_all(metalocation) + return YAML.load(data) + rescue AWS::S3::Errors::NoSuchKey + return nil + end end end diff --git a/ruby/lib/jam_ruby/models/genre.rb b/ruby/lib/jam_ruby/models/genre.rb index 1b0cd9ada..91d80f755 100644 --- a/ruby/lib/jam_ruby/models/genre.rb +++ b/ruby/lib/jam_ruby/models/genre.rb @@ -16,7 +16,8 @@ module JamRuby has_and_belongs_to_many :recordings, :class_name => "JamRuby::Recording", :join_table => "recordings_genres" # jam tracks - has_many :jam_tracks, :class_name => "JamRuby::JamTrack" + has_many :genres_jam_tracks, :class_name => "JamRuby::GenreJamTrack", :foreign_key => "genre_id" + has_many :jam_tracks, :through => :genres_jam_tracks, :class_name => "JamRuby::JamTrack", :source => :genre def to_s description diff --git a/ruby/lib/jam_ruby/models/genre_jam_track.rb b/ruby/lib/jam_ruby/models/genre_jam_track.rb new file mode 100644 index 000000000..aa05e4fd8 --- /dev/null +++ b/ruby/lib/jam_ruby/models/genre_jam_track.rb @@ -0,0 +1,8 @@ +module JamRuby + class GenreJamTrack < ActiveRecord::Base + + self.table_name = 'genres_jam_tracks' + belongs_to :jam_track, class_name: 'JamRuby::JamTrack' + belongs_to :genre, class_name: 'JamRuby::Genre' + end +end diff --git a/ruby/lib/jam_ruby/models/jam_track.rb b/ruby/lib/jam_ruby/models/jam_track.rb index 8ab120c62..71b7aabed 100644 --- a/ruby/lib/jam_ruby/models/jam_track.rb +++ b/ruby/lib/jam_ruby/models/jam_track.rb @@ -14,7 +14,7 @@ module JamRuby attr_accessor :uploading_preview attr_accessible :name, :description, :bpm, :time_signature, :status, :recording_type, - :original_artist, :songwriter, :publisher, :licensor, :licensor_id, :pro, :genre, :genre_id, :sales_region, :price, + :original_artist, :songwriter, :publisher, :licensor, :licensor_id, :pro, :genres_jam_tracks_attributes, :sales_region, :price, :reproduction_royalty, :public_performance_royalty, :reproduction_royalty_amount, :licensor_royalty_amount, :pro_royalty_amount, :plan_code, :initial_play_silence, :jam_track_tracks_attributes, :jam_track_tap_ins_attributes, :version, :jmep_json, :jmep_text, :pro_ascap, :pro_bmi, :pro_sesac, :duration, as: :admin @@ -42,11 +42,14 @@ module JamRuby validates_format_of :reproduction_royalty_amount, with: /^\d+\.*\d{0,3}$/ validates_format_of :licensor_royalty_amount, with: /^\d+\.*\d{0,3}$/ - belongs_to :genre, class_name: "JamRuby::Genre" belongs_to :licensor , class_name: 'JamRuby::JamTrackLicensor', foreign_key: 'licensor_id' + has_many :genres_jam_tracks, :class_name => "JamRuby::GenreJamTrack", :foreign_key => "jam_track_id" + has_many :genres, :through => :genres_jam_tracks, :class_name => "JamRuby::Genre", :source => :genre + has_many :jam_track_tracks, :class_name => "JamRuby::JamTrackTrack", order: 'track_type ASC, position ASC, part ASC, instrument_id ASC' has_many :jam_track_tap_ins, :class_name => "JamRuby::JamTrackTapIn", order: 'offset_time ASC' + has_many :jam_track_files, :class_name => "JamRuby::JamTrackFile" has_many :jam_track_rights, :class_name => "JamRuby::JamTrackRight" #, inverse_of: 'jam_track', :foreign_key => "jam_track_id" # ' @@ -171,7 +174,7 @@ module JamRuby end if options[:group_artist] - query = query.select("original_artist, array_agg(jam_tracks.id) AS id, MIN(name) AS name, MIN(description) AS description, MIN(recording_type) AS recording_type, MIN(original_artist) AS original_artist, MIN(songwriter) AS songwriter, MIN(publisher) AS publisher, MIN(sales_region) AS sales_region, MIN(price) AS price, MIN(version) AS version, MIN(genre_id) AS genre_id") + query = query.select("original_artist, array_agg(jam_tracks.id) AS id, MIN(name) AS name, MIN(description) AS description, MIN(recording_type) AS recording_type, MIN(original_artist) AS original_artist, MIN(songwriter) AS songwriter, MIN(publisher) AS publisher, MIN(sales_region) AS sales_region, MIN(price) AS price, MIN(version) AS version") query = query.group("original_artist") query = query.order('jam_tracks.original_artist') else @@ -180,7 +183,12 @@ module JamRuby end query = query.where("jam_tracks.status = ?", 'Production') unless user.admin - query = query.where("jam_tracks.genre_id = '#{options[:genre]}'") unless options[:genre].blank? + + unless options[:genre].blank? + query = query.joins(:genres) + query = query.where('genre_id = ? ', options[:genre]) + end + query = query.where("jam_track_tracks.instrument_id = '#{options[:instrument]}' and jam_track_tracks.track_type != 'Master'") unless options[:instrument].blank? query = query.where("jam_tracks.sales_region = '#{options[:availability]}'") unless options[:availability].blank? @@ -231,7 +239,12 @@ module JamRuby query = query.order('jam_tracks.original_artist') query = query.where("jam_tracks.status = ?", 'Production') unless user.admin - query = query.where("jam_tracks.genre_id = '#{options[:genre]}'") unless options[:genre].blank? + + unless options[:genre].blank? + query = query.joins(:genres) + query = query.where('genre_id = ? ', options[:genre]) + end + query = query.where("jam_track_tracks.instrument_id = '#{options[:instrument]}'") unless options[:instrument].blank? query = query.where("jam_tracks.sales_region = '#{options[:availability]}'") unless options[:availability].blank? diff --git a/ruby/lib/jam_ruby/models/jam_track_file.rb b/ruby/lib/jam_ruby/models/jam_track_file.rb new file mode 100644 index 000000000..1f71e5300 --- /dev/null +++ b/ruby/lib/jam_ruby/models/jam_track_file.rb @@ -0,0 +1,78 @@ +module JamRuby + + # holds a click track or precount file + class JamTrackFile < ActiveRecord::Base + include JamRuby::S3ManagerMixin + + # there should only be one Master per JamTrack, but there can be N Track per JamTrack + FILE_TYPE = %w{ClickWav ClickTxt Precount} + + @@log = Logging.logger[JamTrackFile] + + before_destroy :delete_s3_files + + attr_accessible :jam_track_id, :file_type, :filename, as: :admin + attr_accessible :url, :md5, :length, as: :admin + + attr_accessor :original_audio_s3_path, :skip_uploader, :preview_generate_error + + before_destroy :delete_s3_files + + validates :track_type, inclusion: {in: FILE_TYPE } + + belongs_to :jam_track, class_name: "JamRuby::JamTrack" + + # create storage directory that will house this jam_track, as well as + def store_dir + "jam_track_files" + end + + # create name of the file + def filename(original_name) + "#{store_dir}/#{jam_track.original_artist}/#{jam_track.name}/#{original_name}" + end + + def manually_uploaded_filename + if click_wav? + filename('click.wav') + elsif click_txt? + filename('click.txt') + elsif precount? + filename('precount.wav') + else + raise 'unknown file type: ' + file_type + end + + end + + def click_wav? + track_type == 'ClickWav' + end + + def click_txt? + track_type == 'ClickTxt' + end + + def precount? + track_type == 'Precount' + end + + # creates a short-lived URL that has access to the object. + # the idea is that this is used when a user who has the rights to this tries to download this JamTrack + # we would verify their rights (can_download?), and generates a URL in response to the click so that they can download + # but the url is short lived enough so that it wouldn't be easily shared + def sign_url(expiration_time = 120) + s3_manager.sign_url(self[url], {:expires => expiration_time, :response_content_type => 'audio/wav', :secure => true}) + end + + def can_download?(user) + # I think we have to make a special case for 'previews', but maybe that's just up to the controller to not check can_download? + jam_track.owners.include?(user) + end + + + def delete_s3_files + s3_manager.delete(self[:url]) if self[:url] && s3_manager.exists?(self[:url]) + end + end +end diff --git a/ruby/spec/factories.rb b/ruby/spec/factories.rb index 90cd48c44..531d3cc46 100644 --- a/ruby/spec/factories.rb +++ b/ruby/spec/factories.rb @@ -740,7 +740,7 @@ FactoryGirl.define do licensor_royalty_amount 0.999 sequence(:plan_code) { |n| "jamtrack-#{n}" } - genre JamRuby::Genre.first + genres [JamRuby::Genre.first] association :licensor, factory: :jam_track_licensor factory :jam_track_with_tracks do diff --git a/ruby/spec/jam_ruby/jam_track_importer_spec.rb b/ruby/spec/jam_ruby/jam_track_importer_spec.rb index a2cf96620..fbdc3d9cc 100644 --- a/ruby/spec/jam_ruby/jam_track_importer_spec.rb +++ b/ruby/spec/jam_ruby/jam_track_importer_spec.rb @@ -22,11 +22,13 @@ describe JamTrackImporter do in_directory_with_file(metafile) before(:each) do + JamTrackImporter.storage_format = 'default' content_for_file(YAML.dump(sample_yml)) end it "no meta" do s3_metalocation = 'audio/Artist 1/Bogus Place/meta.yml' + JamTrackImporter.storage_format = 'default' JamTrackImporter.load_metalocation(s3_metalocation).should be_nil end @@ -38,9 +40,105 @@ describe JamTrackImporter do end end + describe "sort_tracks" do + let(:jam_track) { FactoryGirl.create(:jam_track) } + let(:importer) { JamTrackImporter.new() } + let(:vocal) {Instrument.find('voice')} + let(:drums) {Instrument.find('drums')} + let(:bass_guitar) {Instrument.find('bass guitar')} + let(:piano) {Instrument.find('piano')} + let(:keyboard) {Instrument.find('keyboard')} + let(:acoustic_guitar) {Instrument.find('acoustic guitar')} + let(:electric_guitar) {Instrument.find('electric guitar')} + let(:other) {Instrument.find('other')} + + it "the big sort" do + # specified in https://jamkazam.atlassian.net/browse/VRFS-3296 + vocal_lead = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Lead') + vocal_lead_female = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Lead Female') + vocal_lead_male = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Lead Male') + vocal_backing = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Backing') + vocal_random = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: vocal, part: 'Random') + drums_drums = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: drums, part: 'Drums') + drums_percussion = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: drums, part: 'Percussion') + drums_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: drums, part: 'A') + drums_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: drums, part: 'C') + bass_guitar_bass = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: bass_guitar, part: 'Bass') + bass_guitar_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: bass_guitar, part: 'some bass') + bass_guitar_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: bass_guitar, part: 'zome bass') + piano_piano = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: piano, part: 'Piano') + keyboard_synth_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'Synth 1') + keyboard_synth_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'Synth 2') + keyboard_pads = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'Pads') + keyboard_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'A') + keyboard_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: keyboard, part: 'Z') + acoust_guitar_lead = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Lead') + acoust_guitar_lead_x = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Lead X') + acoust_guitar_solo_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Solo 1') + acoust_guitar_solo_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Solo 2') + acoust_guitar_rhythm = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Rhythm') + acoust_guitar_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'A') + acoust_guitar_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: acoustic_guitar, part: 'Z') + elect_guitar_lead = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Lead') + elect_guitar_lead_x = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Lead X') + elect_guitar_solo_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Solo 1') + elect_guitar_solo_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Solo 2') + elect_guitar_rhythm = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Rhythm') + elect_guitar_random_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'A') + elect_guitar_random_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: electric_guitar, part: 'Z') + other_1 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: other, part: 'Other 1') + other_2 = FactoryGirl.build(:jam_track_track, jam_track: jam_track, instrument: other, part: 'Other 2') + + expected = [ + vocal_lead, + vocal_lead_female, + vocal_lead_male, + vocal_backing, + vocal_random, + drums_drums, + drums_percussion, + drums_random_1, + drums_random_2, + bass_guitar_bass, + piano_piano, + keyboard_synth_1, + keyboard_synth_2, + keyboard_pads, + keyboard_random_1, + keyboard_random_2, + acoust_guitar_lead, + acoust_guitar_lead_x, + acoust_guitar_rhythm, + acoust_guitar_random_1, + acoust_guitar_solo_1, + acoust_guitar_solo_2, + acoust_guitar_random_2, + elect_guitar_lead, + elect_guitar_lead_x, + elect_guitar_solo_1, + elect_guitar_solo_2, + elect_guitar_rhythm, + elect_guitar_random_1, + elect_guitar_random_2, + bass_guitar_random_1, + bass_guitar_random_2, + other_1, + other_2 + ] + shuffled = expected.shuffle + sorted_tracks = importer.sort_tracks(shuffled) + + importer.set_custom_weight(vocal_lead).should eq(100) + + expected.each_with_index do |expected_track, i| + sorted_tracks[i].should eq(expected_track) + end + end + end + describe "synchronize" do let(:jam_track) { JamTrack.new } - let(:importer) { JamTrackImporter.new } + let(:importer) { JamTrackImporter.new() } let(:minimum_meta) { nil } let(:metalocation) { 'audio/Artist 1/Song 1/meta.yml' } let(:options) {{ skip_audio_upload:true }} @@ -64,7 +162,7 @@ describe JamTrackImporter do describe "parse_wav" do it "Guitar" do - result = JamTrackImporter.new.parse_wav('blah/Ready for Love Stem - Guitar - Main.wav') + result = JamTrackImporter.new.parse_file('blah/Ready for Love Stem - Guitar - Main.wav') result[:instrument].should eq('electric guitar') result[:part].should eq('Main') end diff --git a/ruby/spec/jam_ruby/models/jam_track_spec.rb b/ruby/spec/jam_ruby/models/jam_track_spec.rb index d17019df7..3c17c1315 100644 --- a/ruby/spec/jam_ruby/models/jam_track_spec.rb +++ b/ruby/spec/jam_ruby/models/jam_track_spec.rb @@ -12,6 +12,7 @@ describe JamTrack do jam_track = FactoryGirl.create(:jam_track) jam_track.licensor.should_not be_nil jam_track.licensor.jam_tracks.should == [jam_track] + jam_track.genres.length.should eq(1) end describe 'plays' do @@ -98,6 +99,26 @@ describe JamTrack do query[1].should eq(jam_track1) end + it "queries on genre" do + jam_track1 = FactoryGirl.create(:jam_track_with_tracks, original_artist: 'artist', name: 'a') + jam_track2 = FactoryGirl.create(:jam_track_with_tracks, original_artist: 'artist', name: 'b') + jam_track1.genres = [Genre.find('rock')] + jam_track2.genres = [Genre.find('asian')] + jam_track1.save! + jam_track2.save! + + query, pager = JamTrack.index({genre: 'rock'}, user) + query.size.should == 1 + query[0].should eq(jam_track1) + + query, pager = JamTrack.index({genre: 'asian'}, user) + query.size.should == 1 + query[0].should eq(jam_track2) + + query, pager = JamTrack.index({genre: 'african'}, user) + query.size.should == 0 + end + it "supports showing purchased only" do jam_track1 = FactoryGirl.create(:jam_track_with_tracks, name: 'a') diff --git a/web/app/views/api_jam_tracks/show.rabl b/web/app/views/api_jam_tracks/show.rabl index 028180632..e05ca0cbf 100644 --- a/web/app/views/api_jam_tracks/show.rabl +++ b/web/app/views/api_jam_tracks/show.rabl @@ -3,7 +3,7 @@ object @jam_track attributes :id, :name, :description, :recording_type, :original_artist, :songwriter, :publisher, :sales_region, :price, :version, :duration node :genres do |item| - [item.genre.description] # XXX: need to return single genre; not array + item.genres.select(:description).map(&:description) end node :added_cart do |item| diff --git a/web/app/views/api_jam_tracks/show_for_client.rabl b/web/app/views/api_jam_tracks/show_for_client.rabl index bc212f46d..ce60ba00b 100644 --- a/web/app/views/api_jam_tracks/show_for_client.rabl +++ b/web/app/views/api_jam_tracks/show_for_client.rabl @@ -1,9 +1,9 @@ object @jam_track -attributes :id, :name, :description, :initial_play_silence, :original_artist, :version, :genre +attributes :id, :name, :description, :initial_play_silence, :original_artist, :version -node :genre do |jam_track| - jam_track.genre.present? ? jam_track.genre.id : nil +node :genres do |item| + item.genres.select(:description).map(&:description) end node :jmep do |jam_track| diff --git a/web/lib/tasks/jam_tracks.rake b/web/lib/tasks/jam_tracks.rake index 66c06aac2..65ec0febb 100644 --- a/web/lib/tasks/jam_tracks.rake +++ b/web/lib/tasks/jam_tracks.rake @@ -4,6 +4,16 @@ namespace :jam_tracks do JamTrackImporter.dry_run end + task tency_dry_run: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + JamTrackImporter.dry_run + end + + task tency_create_masters: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + JamTrackImporter.create_masters + end + task sync: :environment do |task, args| path = ENV['TRACK_PATH'] @@ -26,6 +36,11 @@ namespace :jam_tracks do JamTrackImporter.synchronize_from_meta("audio/#{path}/meta.yml", resync_audio:true, skip_audio_upload:false) end + task tency_genre_dump: :environment do |task, args| + JamTrackImporter.storage_format = 'Tency' + JamTrackImporter.genre_dump + end + task sync_all: :environment do |task, args| JamTrackImporter.synchronize_all(skip_audio_upload:false) end @@ -95,6 +110,6 @@ namespace :jam_tracks do task tency: :environment do |task, arg| mapper = TencyStemMapping.new - mapper.create_map + mapper.correlate end end diff --git a/web/spec/factories.rb b/web/spec/factories.rb index 52edb7820..48f2ef3af 100644 --- a/web/spec/factories.rb +++ b/web/spec/factories.rb @@ -731,7 +731,7 @@ FactoryGirl.define do make_track true end - genre JamRuby::Genre.first + genres [JamRuby::Genre.first] association :licensor, factory: :jam_track_licensor after(:create) do |jam_track, evaluator|