require 'json' require 'resque' require 'resque-retry' require 'net/http' require 'digest/md5' module JamRuby class JamTrackMixdownPackager extend JamRuby::ResqueStats include JamRuby::S3ManagerMixin TAP_IN_PADDING = 2 MAX_PAN = 90 MIN_PAN = -90 KNOCK_SECONDS = 0.035 attr_accessor :mixdown_package_id, :settings, :mixdown_package, :mixdown, :step @queue = :jam_track_mixdown_packager def log @log || Logging.logger[JamTrackMixdownPackager] end def self.perform(mixdown_package_id, bitrate=48) jam_track_builder = JamTrackMixdownPackager.new() jam_track_builder.mixdown_package_id = mixdown_package_id jam_track_builder.run end def compute_steps @step = 0 number_downloads = @track_settings.length number_volume_adjustments = (@track_settings.select { |track| should_alter_volume? track }).length pitch_shift_steps = @mixdown.will_pitch_shift? ? 1 : 0 mix_steps = 1 package_steps = 1 number_downloads + number_volume_adjustments + pitch_shift_steps + mix_steps + package_steps end def run begin log.info("Mixdown job starting. mixdown_packager_id #{mixdown_package_id}") begin @mixdown_package = JamTrackMixdownPackage.find(mixdown_package_id) # bailout check if @mixdown_package.signed? log.debug("package is already signed. bailing") return end @mixdown = @mixdown_package.jam_track_mixdown @settings = @mixdown.settings process_jmep track_settings # compute the step count total_steps = compute_steps # track that it's started ( and avoid db validations ) signing_started_at = Time.now last_step_at = Time.now #JamTrackMixdownPackage.where(:id => @mixdown_package.id).update_all(:signing_started_at => signing_started_at, :should_retry => false, packaging_steps: total_steps, current_packaging_step: 0, last_step_at: last_step_at, :signing => true) # because we are skipping 'after_save', we have to keep the model current for the notification. A bit ugly... @mixdown_package.current_packaging_step = 0 @mixdown_package.packaging_steps = total_steps @mixdown_package.signing_started_at = signing_started_at @mixdown_package.signing = true @mixdown_package.should_retry = false @mixdown_package.last_step_at = last_step_at @mixdown_package.queued = false @mixdown_package.save SubscriptionMessage.mixdown_signing_job_change(@mixdown_package) package log.info "Signed mixdown package to #{@mixdown_package[:url]}" rescue Exception => e # record the error in the database post_error(e) #SubscriptionMessage.mixdown_signing_job_change(@mixdown_package) # and let the job fail, alerting ops too raise end end end def should_alter_volume? track # short cut is possible if vol = 1.0 and pan = 0 vol = track[:vol] pan = track[:pan] vol != 1.0 || pan != 0 end def process_jmep @start_points = [] @initial_padding = 0.0 @tap_in_initial_silence = 0 speed = @settings['speed'] || 0 @speed_factor = 1.0 + (-speed.to_f / 100.0) @inverse_speed_factor = 1 - (-speed.to_f / 100) log.info("speed factor #{@speed_factor}") jmep = @mixdown.jam_track.jmep_json if jmep.nil? log.debug("no jmep") return end events = jmep["Events"] return if events.nil? || events.length == 0 metronome = nil events.each do |event| if event.has_key?("metronome") metronome = event["metronome"] break end end if metronome.nil? || metronome.length == 0 log.debug("no metronome events for jmep", jmep) return end @start_points = metronome.select { |x| puts x.inspect; x["action"] == "start" } log.debug("found #{@start_points.length} metronome start points") start_point = @start_points[0] if start_point start_time = parse_time(start_point["ts"]) if start_time < 2.0 padding = start_time - 2.0 @initial_padding = padding.abs @initial_tap_in = start_time end end if @speed_factor != 1.0 metronome.length.times do |count| # we expect to find metronome start/stop grouped if count % 2 == 0 start = metronome[count] stop = metronome[count + 1] if start["action"] != "start" || stop["action"] != "stop" # bail out log.error("found de-coupled metronome events #{start.to_json} | #{stop.to_json}") next end bpm = start["bpm"].to_f stop_time = parse_time(stop['ts']) ticks = stop['ticks'].to_i new_bpm = bpm * @inverse_speed_factor new_stop_time = stop_time * @speed_factor new_start_time = new_stop_time - (60.0/new_bpm * ticks) log.info("original bpm:#{bpm} start: #{parse_time(start["ts"])} stop: #{stop_time}") log.info("updated bpm:#{new_bpm} start: #{new_start_time} stop: #{new_stop_time}") stop["ts"] = new_stop_time start["ts"] = new_start_time start["bpm"] = new_bpm stop["bpm"] = new_bpm @tap_in_initial_silence = (@initial_tap_in + @initial_padding) * @speed_factor end end end @start_points = metronome.select { |x| puts x.inspect; x["action"] == "start" } end # format like: "-0:00:02:820" def parse_time(ts) if ts.is_a?(Float) return ts end time = 0.0 negative = false if ts.start_with?('-') negative = true end # parse time_format bits = ts.split(':').reverse bit_position = 0 bits.each do |bit| if bit_position == 0 # milliseconds milliseconds = bit.to_f time += milliseconds/1000 elsif bit_position == 1 # seconds time += bit.to_f elsif bit_position == 2 # minutes time += 60 * bit.to_f elsif bit_position == 3 # hours # not bothering end bit_position += 1 end if negative time = 0.0 - time end time end def path_to_resources File.join(File.dirname(File.expand_path(__FILE__)), '../../../lib/jam_ruby/app/assets/sounds') end def knock_file if long_sample_rate == 44100 knock = File.join(path_to_resources, 'knock44.wav') else knock = File.join(path_to_resources, 'knock48.wav') end knock end def create_silence(tmp_dir, segment_count, duration) file = File.join(tmp_dir, "#{segment_count}.wav") # -c 2 means stereo cmd("sox -n -r #{long_sample_rate} -c 2 #{file} trim 0.0 #{duration}", "silence") file end def create_tapin_track(tmp_dir) return nil if @start_points.length == 0 segment_count = 0 #initial_silence = @initial_tap_in + @initial_padding initial_silence = @tap_in_initial_silence #log.info("tapin data: initial_tap_in: #{@initial_tap_in}, initial_padding: #{@initial_padding}, initial_silence: #{initial_silence}") time_points = [] files = [] if initial_silence > 0 files << create_silence(tmp_dir, segment_count, initial_silence) time_points << {type: :silence, ts: initial_silence} segment_count += 1 end time_cursor = nil @start_points.each do |start_point| tap_time = parse_time(start_point["ts"]) if !time_cursor.nil? between_silence = tap_time - time_cursor files << create_silence(tmp_dir, segment_count, between_silence) time_points << {type: :silence, ts: between_silence} end time_cursor = tap_time bpm = start_point["bpm"].to_f tick_silence = 60.0/bpm - KNOCK_SECONDS ticks = start_point["ticks"].to_i ticks.times do |tick| files << knock_file files << create_silence(tmp_dir, segment_count, tick_silence) time_points << {type: :knock, ts: KNOCK_SECONDS} time_points << {type: :silence, ts: tick_silence} time_cursor + 60.0/bpm segment_count += 1 end end log.info("time points for tap-in: #{time_points.inspect}") # do we need to pad with time? not sure sequence_cmd = "sox " files.each do |file| sequence_cmd << "\"#{file}\" " end count_in = File.join(tmp_dir, "count-in.wav") sequence_cmd << "\"#{count_in}\"" cmd(sequence_cmd, "count_in") @count_in_file = count_in count_in end # creates a list of tracks to actually mix def track_settings altered_tracks = @settings["tracks"] || [] @track_settings = [] #void slider2Pan(int i, float *f); stems = @mixdown.jam_track.stem_tracks @track_count = stems.length @include_count_in = @settings["count-in"] && @start_points.length > 0 && @mixdown_package.encrypt_type.nil? # temp # @include_count_in = true if @include_count_in @track_count += 1 end stems.each do |stem| vol = 1.0 pan = 0 match = false skipped = false # is this stem in the altered_tracks list? altered_tracks.each do |alteration| if alteration["id"] == stem.id if alteration["mute"] || alteration["vol"] == 0 log.debug("leaving out track because muted or 0 volume #{alteration.inspect}") skipped = true next else vol = alteration["vol"] || vol pan = alteration["pan"] || pan end @track_settings << {stem: stem, vol: vol, pan: pan} match = true break end end # if we didn't deliberately skip this one, and if there was no 'match' (meaning user did not specify), then we leave this in unchanged if !skipped && !match @track_settings << {stem: stem, vol: vol, pan: pan} end end if @include_count_in @track_settings << {count_in: true, vol: 1.0, pan: 0} end @track_settings end def slider_to_pan(pan) # transpose MIN_PAN to MAX_PAN to # 0-1.0 range #assumes abs(MIN_PAN) == abs(MAX_PAN) # k = f(i) = (i)/(2*MAX_PAN) + 0.5 # so f(MIN_PAN) = -0.5 + 0.5 = 0 k = ((pan * (1.0))/ (2.0 * MAX_PAN)) + 0.5 l, r = 0 if k == 0 l = 0.0 r = 1.0 else l = Math.sqrt(k) r = Math.sqrt(1-k) end [l, r] end def package log.info("Settings: #{@settings.to_json}") Dir.mktmpdir do |tmp_dir| # download all files @track_settings.each do |track| if track[:count_in] file = create_tapin_track(tmp_dir) bump_step(@mixdown_package) else jam_track_track = track[:stem] file = File.join(tmp_dir, jam_track_track.id + '.ogg') bump_step(@mixdown_package) # download each track needed s3_manager.download(jam_track_track.url_by_sample_rate(@mixdown_package.sample_rate), file) end track[:file] = file end audio_process tmp_dir end end def audio_process(tmp_dir) # use sox remix to apply mute, volume, pan settings # step 1: apply pan and volume per track. mute and vol of 0 has already been handled, by virtue of those tracks not being present in @track_settings # step 2: mix all tracks into single track, dividing by constant number of jam tracks, which is same as done by client backend # step 3: apply pitch and speed (if applicable) # step 4: encrypt with jkz (if applicable) apply_vol_and_pan tmp_dir create_silence_padding tmp_dir mix tmp_dir pitch_speed tmp_dir final_packaging tmp_dir end # output is :volumed_file in each track in @track_settings def apply_vol_and_pan(tmp_dir) @track_settings.each do |track| jam_track_track = track[:stem] count_in = track[:count_in] file = track[:file] unless should_alter_volume? track track[:volumed_file] = file else pan_l, pan_r = slider_to_pan(track[:pan]) vol = track[:vol] # short channel_l = pan_l * vol channel_r = pan_r * vol bump_step(@mixdown_package) # sox claps.wav claps-remixed.wav remix 1v1.0 2v1.0 if count_in volumed_file = File.join(tmp_dir, 'count-in' + '-volumed.ogg') else volumed_file = File.join(tmp_dir, jam_track_track.id + '-volumed.ogg') end cmd("sox \"#{file}\" \"#{volumed_file}\" remix 1v#{channel_r} 2v#{channel_l}", 'vol_pan') track[:volumed_file] = volumed_file end end end def create_silence_padding(tmp_dir) if @initial_padding > 0 && @include_count_in @padding_file = File.join(tmp_dir, "initial_padding.ogg") # -c 2 means stereo cmd("sox -n -r #{long_sample_rate} -c 2 #{@padding_file} trim 0.0 #{@initial_padding}", "initial_padding") @track_settings.each do |track| next if track[:count_in] input = track[:volumed_file] output = input[0..-5] + '-padded.ogg' padd_cmd = "sox '#{@padding_file}' '#{input}' '#{output}'" cmd(padd_cmd, "pad_track_with_silence") track[:volumed_file] = output end end end # output is @mix_file def mix(tmp_dir) bump_step(@mixdown_package) @mix_file = File.join(tmp_dir, "mix.ogg") pitch = @settings['pitch'] || 0 speed = @settings['speed'] || 0 real_count = @track_settings.count real_count -= 1 if @include_count_in # if there is only one track to mix, we need to skip mixing (sox will barf if you try to mix one file), but still divide by number of tracks if real_count <= 1 mix_divide = 1.0/@track_count cmd = "sox -v #{mix_divide} \"#{@track_settings[0][:volumed_file]}\" \"#{@mix_file}\"" cmd(cmd, 'volume_adjust') else # sox -m will divide by number of inputs by default. But we purposefully leave out tracks that are mute/no volume (to save downloading/processing time in this job) # so we need to tell sox to divide by how many tracks there are as a constant, because this is how the client works today #sox -m -v 1/n file1 -v 1/n file2 out cmd = "sox -m" mix_divide = 1.0/@track_count @track_settings.each do |track| # if pitch/shifted, we lay the tap-in after pitch/speed shift # next if (pitch != 0 || speed != 0) && track[:count_in] next if track[:count_in] volumed_file = track[:volumed_file] cmd << " -v #{mix_divide} \"#{volumed_file}\"" end cmd << " \"#{@mix_file}\"" cmd(cmd, 'mix_adjust') end end def long_sample_rate sample_rate = 48000 if @mixdown_package.sample_rate != 48 sample_rate = 44100 end sample_rate end # output is @speed_mix_file def pitch_speed tmp_dir # # usage # This app will take an ogg, wav, or mp3 file (for the uploads) as its input and output an ogg file. # Usage: # sbsms path-to-input.ogg path-to-output.ogg TimeStrech PitchShift # input is @mix_file, created by mix() # output is @speed_mix_file pitch = @settings['pitch'] || 0 speed = @settings['speed'] || 0 # if pitch and speed are 0, we do nothing here if pitch == 0 && speed == 0 @speed_mix_file = @mix_file else bump_step(@mixdown_package) @speed_mix_file = File.join(tmp_dir, "speed_mix_file.ogg") # usage: sbsms infile<.wav|.aif|.mp3|.ogg> outfile<.ogg> rate[0.01:100] halfsteps[-48:48] outSampleRateInHz sample_rate = long_sample_rate # rate comes in as a percent (like 5, -5 for 5%, -5%). We need to change that to 1.05/ sbsms_speed = speed/100.0 sbsms_speed = 1.0 + sbsms_speed sbsms_pitch = pitch cmd("sbsms \"#{@mix_file}\" \"#{@speed_mix_file}\" #{sbsms_speed} #{sbsms_pitch} #{sample_rate}", 'speed_pitch_shift') end if @include_count_in # lay the tap-ins over the recording layered = File.join(tmp_dir, "layered_speed_mix.ogg") cmd("sox -m '#{@count_in_file}' '#{@speed_mix_file}' '#{layered}'", "layer_tap_in") @speed_mix_file = layered end end def final_packaging tmp_dir bump_step(@mixdown_package) url = nil private_key = nil md5 = nil length = 0 output = nil if @mixdown_package.encrypt_type output, private_key = encrypt_jkz tmp_dir else # create output file to correct output format output = convert tmp_dir end # upload output to S3 s3_url = "#{@mixdown_package.store_dir}/#{@mixdown_package.filename}" s3_manager.upload(s3_url, output) length = File.size(output) computed_md5 = Digest::MD5.new File.open(output, 'rb').each { |line| computed_md5.update(line) } md5 = computed_md5.to_s @mixdown_package.finish_sign(s3_url, private_key, length, md5.to_s) end # returns output destination, converting if necessary def convert(tmp_dir) # if the file already ends with the desired file type, call it a win if @speed_mix_file.end_with?(@mixdown_package.file_type) @speed_mix_file else # otherwise we need to convert from lastly created file to correct output = File.join(tmp_dir, "output.#{@mixdown_package.file_type}") cmd("#{APP_CONFIG.normalize_ogg_path} --bitrate 192 \"#{@speed_mix_file}\"", 'normalize') if @mixdown_package.file_type == JamTrackMixdownPackage::FILE_TYPE_AAC cmd("ffmpeg -i \"#{@speed_mix_file}\" -c:a libfdk_aac -b:a 128k \"#{output}\"", 'convert_aac') elsif @mixdown_package.file_type == JamTrackMixdownPackage::FILE_TYPE_MP3 cmd("ffmpeg -i \"#{@speed_mix_file}\" -ab 192k \"#{output}\"", 'convert_mp3') else raise 'unknown file_type' end output end end def encrypt_jkz(tmp_dir) py_root = APP_CONFIG.jamtracks_dir step = 0 private_key = nil # we need to make the id of the custom mix be the name of the file (ID.ogg) custom_mix_name = File.join(tmp_dir, "#{@mixdown.id}.ogg") FileUtils.mv(@speed_mix_file, custom_mix_name) jam_file_opts = "" jam_file_opts << " -i #{Shellwords.escape("#{custom_mix_name}+mixdown")}" sku = @mixdown_package.id title = @mixdown.name output = File.join(tmp_dir, "#{title.parameterize}.jkz") py_file = File.join(py_root, "jkcreate.py") version = @mixdown_package.version right = @mixdown.jam_track.right_for_user(@mixdown.user) if @mixdown_package.sample_rate == 48 private_key = right.private_key_48 else private_key = right.private_key_44 end unless private_key @error_reason = 'no_private_key' @error_detail = 'user needs to generate JamTrack for given sample rate' raise @error_reason end private_key_file = File.join(tmp_dir, 'skey.pem') File.open(private_key_file, 'w') { |f| f.write(private_key) } log.debug("PRIVATE KEY") log.debug(private_key) log.info "Executing python source in #{py_file}, outputting to #{tmp_dir} (#{output})" cli = "python #{py_file} -D -k #{sku} -p #{Shellwords.escape(tmp_dir)}/pkey.pem -s #{Shellwords.escape(tmp_dir)}/skey.pem #{jam_file_opts} -o #{Shellwords.escape(output)} -t #{Shellwords.escape(title)} -V #{Shellwords.escape(version)}" Open3.popen3(cli) do |stdin, stdout, stderr, wait_thr| pid = wait_thr.pid exit_status = wait_thr.value err = stderr.read(1000) out = stdout.read(1000) #puts "stdout: #{out}, stderr: #{err}" raise ArgumentError, "Error calling python script: #{err}" if err.present? raise ArgumentError, "Error calling python script: #{out}" if out && (out.index("No track files specified") || out.index("Cannot find file")) private_key = File.read(private_key_file) end return output, private_key end def cmd(cmd, type) log.debug("executing #{cmd}") output = `#{cmd}` result_code = $?.to_i if result_code == 0 output else @error_reason = type + "_fail" @error_detail = "#{cmd}, #{output}" raise "command `#{cmd}` failed." end end # increment the step, which causes a notification to be sent to the client so it can keep the UI fresh as the packaging step goes on def bump_step(mixdown_package) step = @step last_step_at = Time.now mixdown_package.current_packaging_step = step mixdown_package.last_step_at = last_step_at JamTrackMixdownPackage.where(:id => mixdown_package.id).update_all(last_step_at: last_step_at, current_packaging_step: step) SubscriptionMessage.mixdown_signing_job_change(mixdown_package) @step = step + 1 end # set @error_reason before you raise an exception, and it will be sent back as the error reason # otherwise, the error_reason will be unhandled-job-exception def post_error(e) begin # if error_reason is null, assume this is an unhandled error unless @error_reason @error_reason = "unhandled-job-exception" @error_detail = e.to_s end @mixdown_package.finish_errored(@error_reason, @error_detail) log.error "mixdown package failed. reason=#{@error_reason}\n#detail=#{@error_detail}" rescue Exception => e log.error "unable to post back to the database the error #{e}" end end end end