jam-cloud/web/lib/tasks/jam_tracks.rake

Rails.logger = Logger.new(STDOUT)

namespace :jam_tracks do

  task import_click_tracks: :environment do |task, args|
    JamTrackImporter.storage_format = 'Helbing'
    JamTrackImporter.import_click_tracks
  end

  task generate_jmep: :environment do |task, args|
    JamTrackImporter.storage_format = 'Tency'
    JamTrackImporter.load_marks_approved
    JamTrackImporter.generate_jmeps
  end

  task generate_jmep_paris: :environment do |task, args|
    JamTrackImporter.storage_format = 'Paris'
    JamTrackImporter.load_marks_approved
    JamTrackImporter.generate_jmeps
  end

  task dry_run: :environment do |task, args|
    if ENV['STORAGE']
      JamTrackImporter.storage_format = ENV['STORAGE']
    end
    JamTrackImporter.dry_run
  end

  # valid values for STORAGE
  # Helbing
  task dry_run: :environment do |task, args|
    if ENV['STORAGE']
      JamTrackImporter.storage_format = ENV['STORAGE']
    end
    JamTrackImporter.dry_run
  end

  task create_clevie: :environment do |task, args|
    licensor = JamTrackLicensor.new()
    licensor.name = 'Steely & Clevie'
    licensor.slug = 'steely-and-clevie'
    licensor.save!
  end
  task tency_dry_run: :environment do |task, args|
    JamTrackImporter.storage_format = 'Tency'
    JamTrackImporter.dry_run
  end

  task paris_dry_run: :environment do |task, args|
    JamTrackImporter.storage_format = 'Paris'
    JamTrackImporter.dry_run
  end

  task drumma_dry_run: :environment do |task, args|
    JamTrackImporter.storage_format = 'Drumma'
    JamTrackImporter.dry_run
  end

  task clevie_dry_run: :environment do |task, args|
    JamTrackImporter.storage_format = 'Clevie'
    JamTrackImporter.dry_run
  end

  task paris_create_masters: :environment do |task, args|
    JamTrackImporter.storage_format = 'Paris'
    JamTrackImporter.create_masters
  end

  task timtracks_dry_run: :environment do |task, args|
    JamTrackImporter.storage_format = 'TimTracks'
    JamTrackImporter.dry_run
  end

  task tency_create_masters: :environment do |task, args|
    JamTrackImporter.storage_format = 'Tency'
    JamTrackImporter.create_masters
  end
  task drumma_create_masters: :environment do |task, args|
    JamTrackImporter.storage_format = 'Drumma'
    JamTrackImporter.create_masters
  end

  task tency_licensor_yaml: :environment do |task, args|
    JamTrackImporter.storage_format = 'Tency'
    JamTrackImporter.add_tency_metadata
  end

  task tency_create_master: :environment do |task, args|
    JamTrackImporter.storage_format = 'Tency'

    path = ENV['TRACK_PATH']

    if !path
      puts "TRACK_PATH must be set to something like audio/AC DC/Back in Black  or mapped/50 Cent - In Da Club - 12401"
      exit(1)
    end

    JamTrackImporter.create_master(path)
  end

  task clevie_create_master: :environment do |task, args|
    JamTrackImporter.storage_format = 'Clevie'

    path = ENV['TRACK_PATH']

    if !path
      puts "TRACK_PATH must be set to something like audio/AC DC/Back in Black  or mapped/50 Cent - In Da Club - 12401"
      exit(1)
    end

    JamTrackImporter.create_master(path)
  end

  task tency_delta: :environment do |task, args|
    JamTrackImporter.storage_format = 'Tency'
    JamTrackImporter.tency_delta
  end

  task sync: :environment do |task, args|
    path = ENV['TRACK_PATH']

    if !path
      puts "TRACK_PATH must be set to something like audio/AC DC/Back in Black  or mapped/50 Cent - In Da Club - 12401"
      exit(1)
    end

    if ENV['STORAGE_FORMAT']
      JamTrackImporter.storage_format = ENV['STORAGE_FORMAT']
    elsif path.start_with?('mapped')
      JamTrackImporter.storage_format = 'Tency'
    end

    JamTrackImporter.synchronize_from_meta("#{path}/meta.yml", skip_audio_upload: false)
  end

  task resync_audio: :environment do |task, args|
    path = ENV['TRACK_PATH']

    if !path
      puts "TRACK_PATH must be set to something like AD DC/Back in Black"
      exit(1)
    end

    if path.start_with?('mapped')
      JamTrackImporter.storage_format = 'Paris'
    end

    JamTrackImporter.synchronize_from_meta("#{path}/meta.yml", resync_audio: true, skip_audio_upload: false)
  end

  task resync_instruments: :environment do |task, args|
    JamTrackImporter.storage_format = ENV['STORAGE_FORMAT']
    JamTrackImporter.resync_instruments(JamTrackLicensor.find_by_name!('Stockton Helbing'))
  end


  task fix_slugs: :environment do |task, args|
    JamTrackImporter.storage_format = 'Paris'
    JamTrackImporter.fix_slugs(JamTrackLicensor.find_by_name!('Paris Music'))
  end

  task fix_paris_artists: :environment do |task, args|
    JamTrackImporter.storage_format = 'Paris'
    JamTrackImporter.fix_artist_song_name(JamTrackLicensor.find_by_name!('Paris Music'))
  end

  task dump_missing_masters: :environment do |task, args|
    JamTrackImporter.storage_format = 'Paris'
    JamTrackImporter.missing_masters(JamTrackLicensor.find_by_name!('Paris Music'))
  end

  task tency_genre_dump: :environment do |task, args|
    JamTrackImporter.storage_format = 'Tency'
    JamTrackImporter.tency_genre_dump
  end


  task paris_genre_dump: :environment do |task, args|

    JamTrackImporter.storage_format = 'Paris'

    File.open("/Users/seth/workspace/backups/paris/bucket_contents.txt", "r").each_line do |line|

      mapped_index = line.index('mapped')
      if mapped_index
        path = line[(mapped_index + 'mapped/'.length)..-1]
        bits = path.split('/')
        if bits[-1].strip.end_with?('.wav')
          # got a wave file. let's peek

          if bits.length == 2
            metalocation = "mapped/#{bits[0]}/#{bits[1]}/meta.yml"
            importer = JamTrackImporter.new
            importer.storage_format = 'Paris'

            meta = importer.parse_metalocation(metalocation)
            if meta.length != 6
              raise "UNknOWN META! #{meta.inspect}"
            end

            song_id = JamTrackImporter.extract_paris_song_id(metalocation)
            raise "unknown song id in #{metalocation}" if song_id.nil?
          else
            raise "UNKNOWN!!!"
          end

        end


      end
    end


    JamTrackImporter.paris_genre_dump
  end

  task sync_paris: :environment do |task, args|
    JamTrackImporter.storage_format = 'Paris'
    JamTrackImporter.synchronize_all(skip_audio_upload: false)
  end

  task sync_tency: :environment do |task, args|
    JamTrackImporter.storage_format = 'Tency'
    JamTrackImporter.synchronize_all(skip_audio_upload: false)
  end

  task sync_tim_tracks: :environment do |task, args|
    JamTrackImporter.storage_format = 'TimTracks'
    JamTrackImporter.synchronize_all(skip_audio_upload: false)
  end

  task sync_helbing: :environment do |task, args|
    JamTrackImporter.storage_format = 'Helbing'
    JamTrackImporter.synchronize_all(skip_audio_upload: false)
  end

  task sync_clevie: :environment do |task, args|
    JamTrackImporter.storage_format = 'Clevie'
    JamTrackImporter.synchronize_all(skip_audio_upload: false)
  end

  task tency_dups: :environment do |task, args|

  end
  task onboarding_exceptions: :environment do |task, args|
    JamTrackImporter.onboarding_exceptions
  end

  task generate_slugs: :environment do |task, arg|
    JamTrackImporter.generate_slugs
  end

  task generate_mp3_aac_stems: :environment do |task, arg|
    JamTrackImporter.storage_format = 'default'
    JamTrackImporter.generate_mp3_aac_stems(JamTrackImporter.storage_format)

    JamTrackImporter.storage_format = 'Tency'
    JamTrackImporter.generate_mp3_aac_stems(JamTrackImporter.storage_format)
  end

  task sync_all: :environment do |task, args|
    JamTrackImporter.synchronize_all(skip_audio_upload: false)
  end

  task sync_all_dev: :environment do |task, args|
    JamTrackImporter.storage_format = 'default'
    JamTrackImporter.synchronize_all(skip_audio_upload: true)
  end

  task sync_previews_all: :environment do |task, arg|
    JamTrackImporter.synchronize_previews
  end

  task sync_master_preview_all: :environment do |task, args|
    importer = JamTrackImporter.synchronize_jamtrack_master_previews
  end

  task sync_master_aac: :environment do |task, args|
    JamTrackImporter.synchronize_jamtrack_aac_previews
  end

  task sync_click_mp3_to_wav: :environment do |task, args|
    JamTrackImporter.storage_format = 'Helbing'
    JamTrackImporter.convert_click_track_to_wavs
  end

  # popuplate preview info without uploading/processing audio files (use what's in S3)
  task sync_previews_dev: :environment do |task, args|
    JamTrackImporter.synchronize_previews_dev
  end

  # syncs just one master track for a give JamTrack
  task sync_master_preview: :environment do |task, args|
    plan_code = ENV['PLAN_CODE']
    if !plan_code
      puts "PLAN_CODE must be set to something like jamtrack-acdc-backinblack"
      exit(1)
    end

    jam_track = JamTrack.find_by_plan_code!(plan_code)

    importer = JamTrackImporter.synchronize_jamtrack_master_preview(jam_track)

    if importer.reason.nil? || importer.reason == "success" || importer.reason == "jam_track_exists"
      puts("#{importer.name} #{importer.reason}")
    else
      puts("#{importer.name} failed to import.")
      puts("#{importer.name} reason=#{importer.reason}")
      puts("#{importer.name} detail=#{importer.detail}")
    end
  end

  task sync_duration_all: :environment do |task, args|
    importer = JamTrackImporter.synchronize_durations
  end

  # syncs just one master track for a give JamTrack
  task sync_duration: :environment do |task, args|
    plan_code = ENV['PLAN_CODE']
    if !plan_code
      puts "PLAN_CODE must be set to something like jamtrack-acdc-backinblack"
      exit(1)
    end

    jam_track = JamTrack.find_by_plan_code!(plan_code)

    importer = JamTrackImporter.synchronize_duration(jam_track)

    if importer.reason.nil? || importer.reason == "success" || importer.reason == "jam_track_exists"
      puts("#{importer.name} #{importer.reason}")
    else
      puts("#{importer.name} failed to import.")
      puts("#{importer.name} reason=#{importer.reason}")
      puts("#{importer.name} detail=#{importer.detail}")
    end
  end

  task download_masters: :environment do |task, arg|
    JamTrackImporter.download_masters
  end


  task tency: :environment do |task, arg|
    mapper = TencyStemMapping.new
    mapper.correlate
  end

  task touch: :environment do |task, arg|
    JamTrack.all.each do |jam_track|
      jam_track.jmep_json_generate
      jam_track.save!
    end
  end

  task generate_private_key: :environment do |task, arg|
    JamTrackRight.all.each do |right|
      if right.private_key_44.nil? || right.private_key_48.nil?

        if right.private_key_44.nil? && right.private_key_48
          right.private_key_44 = right.private_key_48
          puts "COPY 48 > 44"
        elsif right.private_key_48.nil? && right.private_key_44
          right.private_key_48 = right.private_key_44
          puts "COPY 44 > 48"
        elsif right.private_key_48.nil? && right.private_key_44.nil?
          rsa_key = OpenSSL::PKey::RSA.new(1024)
          key = rsa_key.to_pem()
          right.private_key_44 = key
          right.private_key_48 = key
          puts "GEN 44 + 48"
        end

        right.save
      else
        puts "OK 44 + 48"
      end
    end
  end

  task gen_jamtrack_files: :environment do |task, arg|

    instruments = Instrument.all
    jam_tracks = JamTrack.where(status: 'Production').order('original_artist, name')

    Dir.mktmpdir do |tmp_dir|
      csv_file = File.join(tmp_dir, "production.csv")
      pdf_file = File.join(tmp_dir, "production.pdf")


      CSV.open(csv_file, "wb") do |csv|
        header = ['Artist', 'Name', 'Link', 'Click Track', 'Count-In', 'Added to Catalog']

        instruments.each do |instrument|
          header << instrument.description
        end

        csv << header
        jam_tracks.each do |jam_track|
          row = []
          row << jam_track.original_artist
          row << jam_track.name
          row << Rails.application.config.external_root_url + Rails.application.routes.url_helpers.individual_jamtrack_path(jam_track.slug)
          row << (jam_track.click_track.nil? ? 'no' : 'yes')
          row << (jam_track.has_count_in? ? 'yes' : 'no')
          row << jam_track.created_at.to_date.to_s

          tracks = jam_track.jam_track_tracks

          instruments.each do |instrument|

            match = false
            tracks.each do |track|
              if track.track_type == 'Track' && track.instrument_id == instrument.id
                match = true
                break
              end
            end

            row << (match ? 'yes' : 'no')
          end
          csv << row
        end
      end

      Prawn::Document.generate(pdf_file) do

        text "Current JamTracks Catalog by JamKazam", align: :center, size:18, inline_format: true
        text "Last Updated: #{Time.now.strftime('%m/%d/%Y')}", align: :center
        move_down 10
        text "This PDF was downloaded from <link href=\"https://s3.amazonaws.com/jamkazam-public/public/lists/all-jamkazam-jamtracks.pdf\"><color rgb='0000FF'>here</color></link>.", inline_format: true, font_size:12
        move_down 10
        text "The CSV version can be downloaded from <link href=\"https://s3.amazonaws.com/jamkazam-public/public/lists/all-jamkazam-jamtracks.csv\"><color rgb='0000FF'>here</color></link>. It also contains instrument info.", inline_format: true, font_size:12
        move_down 20
        pdf_data= []

        header = ['Artist', 'Song Name', 'Site Link', 'Click Track', 'Count In', 'Added to Catalog']

        columns = header.length

        pdf_data << header

        jam_tracks.each do |jam_track|
          table_row = []
          table_row << jam_track.original_artist
          table_row << jam_track.name
          table_row << "<link href=\"#{Rails.application.config.external_root_url + Rails.application.routes.url_helpers.individual_jamtrack_path(jam_track.slug)}\"><color rgb='0000FF'>link </color></link>"
          table_row << (jam_track.click_track.nil? ? 'no' : 'yes')
          table_row << (jam_track.has_count_in? ? 'yes' : 'no')
          table_row << jam_track.created_at.to_date.to_s

          pdf_data << table_row
        end


        table(pdf_data, :cell_style => {inline_format: true}, header: true, row_colors: ["D0D0D0", "F0F0F0"])
      end


      s3_manager = S3Manager.new(Rails.application.config.aws_bucket_public, Rails.application.config.aws_access_key_id, Rails.application.config.aws_secret_access_key)

      s3_manager.upload('public/lists/all-jamkazam-jamtracks.csv', csv_file, content_type: 'text/csv')
      s3_manager.upload('public/lists/all-jamkazam-jamtracks.pdf', pdf_file, content_type: 'application/pdf')

      FileUtils.mv(csv_file, 'tmp/test.csv')
      FileUtils.mv(pdf_file, 'tmp/test.pdf')
    end
  end

  task gen_jamtrack_manifest: :environment do |task, arg|


    # DOWNLOAD_AUDIO =
    max = ENV['MAX_JAMTRACKS'].to_i
    if max == 0
      max = nil
    end
    puts "MAX JAMTRACKS #{max}"

    jam_tracks = JamTrack.includes([:jam_track_tracks,
                                    {genres_jam_tracks: :genres},
                                    {jam_track_tracks: :instrument},
                                    :genres]).where(status: 'Production').order('original_artist, name')
    private_bucket = Rails.application.config.aws_bucket
    s3_manager = S3Manager.new(private_bucket, Rails.application.config.aws_access_key_id, Rails.application.config.aws_secret_access_key)

    tmp_dir = Dir.mktmpdir
    FileUtils.mkdir_p tmp_dir

    puts "tmp_dir=#{tmp_dir}"

    csv_file = File.join(tmp_dir, "manifest.csv")
    top_folder = File.join(tmp_dir, "audio")
    FileUtils.mkdir_p(top_folder)

    CSV.open(csv_file, "wb") do |csv|
      header = ['JamTrackId', 'TrackId', 'Artist', 'Song', 'Instrument', 'Part', 'Type', 'Genre', 'LocalOgg', 'LocalMeta', 's3_path_url_44', 's3_path_url_48', 'AudioExists']
      csv << header

      jam_tracks.each do |jam_track|
        song = jam_track.name
        jam_track.jam_track_tracks.each do |jam_track_track|
          instrument = jam_track_track.instrument_id
          part = jam_track_track.part ? jam_track_track.part : ''


          # construct the meta file for this track:
          meta = {}
          meta[:jam_track_id] = jam_track.id
          meta[:track_id] = jam_track_track.id
          meta[:artist] = jam_track.original_artist
          meta[:song] = jam_track.name
          meta[:instrument] = instrument
          meta[:part] = part
          meta[:type] = jam_track_track.track_type
          genre = jam_track.genres.first
          meta[:genre] = genre ? genre.description : ''
          meta[:s3_path_url_44] = "s3://#{private_bucket}/" + jam_track_track.url_44
          meta[:s3_path_url_48] = "s3://#{private_bucket}/" + jam_track_track.url_48

          meta_json = File.join(tmp_dir, "meta_#{jam_track.id}_#{jam_track_track.id}.json")
          File.open(meta_json, "w") do |f|
            f.write(JSON.pretty_generate(meta))
          end

          # find the first a-z, 0-9 character and use that
          first_character = song[0].downcase
          folder = File.join(top_folder, first_character)
          if File.exist?(folder) == false
            FileUtils.mkdir_p(folder)
          end
          # folder structure is:
          # audio/a/song-instrument-part-type.ogg
          # audio/a/song-instrument-part-type.ogg.meta
          # run 'part' through a sanitizer to make it filesystem safe
          part = part.gsub(/[^0-9A-Za-z]/, '_')
          base_name = File.join(folder, "#{jam_track.id}_#{jam_track_track.id}_#{instrument}_#{part}_#{jam_track_track.track_type}".downcase)
          ogg_file = "#{base_name}.ogg"

          exists = s3_manager.exists?(jam_track_track.url_48)


          row = []
          row << jam_track.id
          row << jam_track_track.id
          row << jam_track.original_artist
          row << jam_track.name
          row << instrument
          row << part
          row << jam_track_track.track_type
          genre = jam_track.genres.first
          row << (genre ? genre.description : '')
          row << ogg_file.gsub(tmp_dir, '')
          row << ogg_file.gsub(tmp_dir, '') + ".meta"
          row << "s3://#{private_bucket}/" + jam_track_track.url_44
          row << "s3://#{private_bucket}/" + jam_track_track.url_48
          row << exists

          csv << row


          FileUtils.cp(meta_json, "#{ogg_file}.meta")
          # cleanup meta
          FileUtils.rm(meta_json)
        end

        if max > 0
          max -= 1
          if max == 0
            puts "Max of jamtracks reached"
            break
          end
        end
      end
    end


    # dump the 1st 10 lines to stdout for quick verification
    File.open(csv_file, "r") do|f|
      10.times do |i|
        puts f.readline
      end
    end

    s3_manager.upload('jam_track_manifests/manifest.csv', csv_file, content_type: 'text/csv')

    folder_to_zip = top_folder
    output_tar_path = Dir.mktmpdir

    output_tar_file = "#{output_tar_path}/archive.tar.gz"

    safe_output = Shellwords.escape(output_tar_file)

    # -c = create
    # -z = compress with gzip
    # -f = to a file
    # -P = (Optional, but useful) Preserve absolute paths.
    #      By default, 'tar' strips the leading '/' for security.
    #      A common pattern is to cd into the parent dir first.

    # Safer way: cd to the directory to get relative paths
    parent_dir = File.dirname(folder_to_zip)
    folder_name = File.basename(folder_to_zip)
    safe_parent = Shellwords.escape(parent_dir)
    safe_folder_name = Shellwords.escape(folder_name)

    # This command is safer as it creates the archive with relative paths
    command = "tar -czf #{safe_output} -C #{safe_parent} #{safe_folder_name}"
    success = system(command)

    if success
      puts "Successfully created tar.gz file."
    else
      puts "Failed to create tar.gz file."
    end


    s3_manager.upload('jam_track_manifests/all-tracks.tar.gz', safe_output, content_type: 'application/gzip')

    puts "tar.gz output=#{safe_output}"

    puts "tmp_dir=#{tmp_dir}"
  end
end