From ff6f133620d2fbbe11f3b76a9b8e1a8a9c9b08aa Mon Sep 17 00:00:00 2001 From: Seth Call Date: Fri, 18 Dec 2015 11:35:56 -0600 Subject: [PATCH] * import fixes for Paris --- ruby/lib/jam_ruby/jam_track_importer.rb | 204 +++++++++++++++++------- web/app/assets/javascripts/globals.js | 2 + web/app/assets/javascripts/utils.js | 1 + web/config/logging.rb | 2 +- web/lib/tasks/jam_tracks.rake | 18 ++- 5 files changed, 166 insertions(+), 61 deletions(-) diff --git a/ruby/lib/jam_ruby/jam_track_importer.rb b/ruby/lib/jam_ruby/jam_track_importer.rb index 396fa6c30..cdaf65d46 100644 --- a/ruby/lib/jam_ruby/jam_track_importer.rb +++ b/ruby/lib/jam_ruby/jam_track_importer.rb @@ -405,6 +405,7 @@ module JamRuby s3_target = audio_path + '/' + output_filename @@log.debug("uploading #{output_file} to #{s3_target}") JamTrackImporter.song_storage_manager.upload(s3_target, output_file) + finish('success', nil) end @@ -414,6 +415,8 @@ module JamRuby end def dry_run(metadata, metalocation) + # STDIN.gets + @@log.debug("dry_run: #{metadata.inspect}") metadata ||= {} @@ -1124,16 +1127,9 @@ module JamRuby possible_part = "#{bits[2]} #{bits[3]}" end - # this implies we've found an instrument and part in file name; try this out first - if possible_instrument && possible_part - result = determine_instrument(possible_instrument, possible_part) - instrument = result[:instrument] - part = result[:part] - end - # otherwise, try mapping - if instrument.nil? - mapping = JamTrackImporter.paris_mapping[possible_instrument] + if instrument.nil? && possible_instrument + mapping = JamTrackImporter.paris_mapping[possible_instrument.downcase] if mapping instrument = mapping[:instrument].downcase part = mapping[:part] @@ -1450,58 +1446,9 @@ module JamRuby @@log.debug("no existing track found; creating a new one") - track = JamTrackTrack.new - track.original_filename = wav_file - track.original_audio_s3_path = wav_file - - file = JamTrackFile.new - file.original_filename = wav_file - file.original_audio_s3_path = wav_file - - parsed_wav = parse_file(wav_file) - - unknowns = 0 - if parsed_wav[:master] - track.track_type = 'Master' - track.part = 'Master Mix' - track.instrument_id = 'computer' - tracks << track - @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") - elsif parsed_wav[:type] == :track - - if !parsed_wav[:instrument] || !parsed_wav[:part] - @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") - unknowns += 1 - else - @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") - end - - track.instrument_id = parsed_wav[:instrument] || 'other' - track.track_type = 'Track' - track.part = parsed_wav[:part] || "Other #{unknowns}" - tracks << track - elsif parsed_wav[:type] == :clicktxt - file.file_type = 'ClickTxt' - addt_files << file - elsif parsed_wav[:type] == :clickwav - file.file_type = 'ClickWav' - addt_files << file - - # and also add a JamTrackTrack for this click track - track.track_type = 'Click' - track.part = 'Clicktrack' - track.instrument_id = 'computer' - track.position = 10000 - tracks << track - elsif parsed_wav[:type] == :precount - file.file_type = 'Precount' - file.precount_num = parsed_wav[:precount_num] - addt_files << file - else - finish("unknown_file_type", "unknown file type #{wave_file}") + if !assign_instrument_parts(wav_file, tracks, addt_files) return false end - end jam_track.jam_track_tracks.each do |jam_track_track| @@ -1537,6 +1484,100 @@ module JamRuby return synchronize_audio_files(jam_track, skip_audio_upload) end + def reassign_instrument_parts(jam_track) + + tracks = [] + addt_files = [] + + jam_track.jam_track_tracks.each do |track| + return if !assign_instrument_parts(track.original_filename, tracks, addt_files, true) + end + + @@log.info("sorting #{tracks.length} tracks") + tracks = sort_tracks(tracks) + + deduplicate_parts(tracks) + + tracks.each do |track| + if track.changed? + puts "CHANGE: #{track.changes.inspect}" + end + if !track.save! + finish('invalid_audio', track.errors.inspect) + end + end + end + + def assign_instrument_parts(wav_file, tracks, addt_files, reassign = false) + if !reassign + track = JamTrackTrack.new + track.original_filename = wav_file + track.original_audio_s3_path = wav_file + + file = JamTrackFile.new + file.original_filename = wav_file + file.original_audio_s3_path = wav_file + else + matches = JamTrackTrack.where(original_filename: wav_file) + if matches.count > 1 + raise "multiple jam track tracks encountered with #{wav_file} as original_filename" + elsif matches.count == 0 + raise "unable to locate jam track wit h#{wav_file} as original_filename" + end + track = matches[0] + track.original_audio_s3_path = wav_file + file = nil + end + + parsed_wav = parse_file(wav_file) + + unknowns = 0 + if parsed_wav[:master] + track.track_type = 'Master' + track.part = 'Master Mix' + track.instrument_id = 'computer' + tracks << track + @@log.debug("#{self.name} master! filename: #{parsed_wav[:filename]}") + elsif parsed_wav[:type] == :track + + if !parsed_wav[:instrument] || !parsed_wav[:part] + @@log.warn("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + unknowns += 1 + else + @@log.debug("#{self.name} track! instrument: #{parsed_wav[:instrument] ? parsed_wav[:instrument] : 'N/A'}, part: #{parsed_wav[:part] ? parsed_wav[:part] : 'N/A'}, filename: #{parsed_wav[:filename]} ") + end + + track.instrument_id = parsed_wav[:instrument] || 'other' + track.track_type = 'Track' + track.part = parsed_wav[:part] || "Other #{unknowns}" + tracks << track + elsif parsed_wav[:type] == :clicktxt + file.file_type = 'ClickTxt' + addt_files << file + elsif parsed_wav[:type] == :clickwav + if file + file.file_type = 'ClickWav' + addt_files << file + end + + # and also add a JamTrackTrack for this click track + track.track_type = 'Click' + track.part = 'Clicktrack' + track.instrument_id = 'computer' + track.position = 10000 + tracks << track + elsif parsed_wav[:type] == :precount + file.file_type = 'Precount' + file.precount_num = parsed_wav[:precount_num] + addt_files << file + else + finish("unknown_file_type", "unknown file type #{wave_file}") + return false + end + + return true + end + def synchronize_audio_files(jam_track, skip_audio_upload) begin @@ -2905,6 +2946,51 @@ module JamRuby end end + + def create_importer_from_existing(jam_track) + importer = JamTrackImporter.new(@storage_format) + importer.name = jam_track.name + importer.metadata = load_metalocation(jam_track.metalocation) + importer + end + + def resync_instruments(licensor) + + load_paris_mappings if @paris_mapping.nil? + + JamTrack.where(licensor_id: licensor.id).each do |jam_track| + + puts "RESYNCING JAMTRACK #{jam_track.id}" + JamTrackTrack.where(jam_track_id: jam_track.id).order(:position).each do |track| + puts "BEFORE TRACK #{track.instrument_id} #{track.part}" + end + + importer = create_importer_from_existing(jam_track) + importer.reassign_instrument_parts(jam_track) + + #puts ">>>>>>>>> HIT KEY TO CONTINUE <<<<<<<<<<" + #STDIN.gets + end + + end + + + def missing_masters(licensor) + + count = 0 + JamTrack.where(licensor_id: licensor.id).each do |jam_track| + + + if jam_track.master_track.nil? + puts "MISSING #{jam_track.metalocation}" + count += 1 + end + + end + + puts "missing master count: #{count}" + end + def tency_genre_dump load_tency_mappings diff --git a/web/app/assets/javascripts/globals.js b/web/app/assets/javascripts/globals.js index 148b5ecc5..d6a7367f5 100644 --- a/web/app/assets/javascripts/globals.js +++ b/web/app/assets/javascripts/globals.js @@ -214,6 +214,7 @@ "Bass Guitar": { "client_id": 20, "server_id": "bass guitar" }, "Computer": { "client_id": 30, "server_id": "computer" }, "Drums": { "client_id": 40, "server_id": "drums" }, + "Percussion": { "client_id": 41, "server_id": "percussion" }, "Electric Guitar": { "client_id": 50, "server_id": "electric guitar" }, "Keyboard": { "client_id": 60, "server_id": "keyboard" }, "Piano": { "client_id": 61, "server_id": "piano" }, @@ -244,6 +245,7 @@ 20: { "server_id": "bass guitar" }, 30: { "server_id": "computer" }, 40: { "server_id": "drums" }, + 41: { "server_id": "percussion" }, 50: { "server_id": "electric guitar" }, 60: { "server_id": "keyboard" }, 61: { "server_id": "piano"} , diff --git a/web/app/assets/javascripts/utils.js b/web/app/assets/javascripts/utils.js index e4d626e0e..c9157ae67 100644 --- a/web/app/assets/javascripts/utils.js +++ b/web/app/assets/javascripts/utils.js @@ -68,6 +68,7 @@ "computer": "computer", "_default": "default", "drums": "drums", + "percussion": "drums", "electric guitar": "electric_guitar", "euphonium": "euphonium", "flute": "flute", diff --git a/web/config/logging.rb b/web/config/logging.rb index d6b9d73aa..f822dc339 100644 --- a/web/config/logging.rb +++ b/web/config/logging.rb @@ -95,7 +95,7 @@ Logging::Rails.configure do |config| Logging.logger['ActiveSupport::OrderedOptions'].level = :warn Logging.logger['InfluxDB'].level = :warn #Logging.logger['Rails'].level = :warn - #Logging.logger['ActiveRecord::Base'].level = :warn + Logging.logger['ActiveRecord::Base'].level = :warn # Under Phusion Passenger smart spawning, we need to reopen all IO streams # after workers have forked. diff --git a/web/lib/tasks/jam_tracks.rake b/web/lib/tasks/jam_tracks.rake index f19b7010f..d1ac8be6c 100644 --- a/web/lib/tasks/jam_tracks.rake +++ b/web/lib/tasks/jam_tracks.rake @@ -24,6 +24,11 @@ namespace :jam_tracks do JamTrackImporter.dry_run end + task paris_create_masters: :environment do |task, args| + JamTrackImporter.storage_format = 'Paris' + JamTrackImporter.create_masters + end + task timtracks_dry_run: :environment do |task, args| JamTrackImporter.storage_format = 'TimTracks' JamTrackImporter.dry_run @@ -39,7 +44,6 @@ namespace :jam_tracks do JamTrackImporter.add_tency_metadata end - task tency_create_master: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' @@ -85,11 +89,23 @@ namespace :jam_tracks do JamTrackImporter.synchronize_from_meta("audio/#{path}/meta.yml", resync_audio: true, skip_audio_upload: false) end + task resync_instruments: :environment do |task, args| + + JamTrackImporter.storage_format = 'Paris' + JamTrackImporter.resync_instruments(JamTrackLicensor.find_by_name!('Paris Music')) + end + + task dump_missing_masters: :environment do |task, args| + JamTrackImporter.storage_format = 'Paris' + JamTrackImporter.missing_masters(JamTrackLicensor.find_by_name!('Paris Music')) + end + task tency_genre_dump: :environment do |task, args| JamTrackImporter.storage_format = 'Tency' JamTrackImporter.tency_genre_dump end + task paris_genre_dump: :environment do |task, args| JamTrackImporter.storage_format = 'Paris'