jam-cloud/ruby/lib/jam_ruby/models/geo_ip_locations.rb

module JamRuby
  class GeoIpLocations < ActiveRecord::Base

    # index names created on the copied table used during import.
    # they do not exist except during import
    GEOIPLOCATIONS_INDEX_NAME = 'geoiplocations_geog_gix'
    COPIED_GEOIPLOCATIONS_INDEX_NAME = 'geoiplocations_copied_geog_gix'

    PRIMARY_KEY_NAME = 'geoiplocations_pkey'
    COPIED_PRIMARY_KEY_NAME = 'geoiplocations_copied_pkey'

    @@log = Logging.logger[GeoIpLocations]

    self.table_name = 'geoiplocations'
    self.primary_key = 'locid'

    has_many :blocks, class_name: 'JamRuby::GeoIpBlocks', inverse_of: 'location', foreign_key: 'locid'

    # Returns a hash with location information. Fields are nil if they can't be figured.
    # This is a class method because it doesn't need to be in a transaction.
    def self.lookup(ip_address)

      city = state = country = nil
      locid = ispid = 0

      unless ip_address.nil? || ip_address !~ /^\d+\.\d+\.\d+\.\d+$/

        addr = ip_address_to_int(ip_address)

        block = GeoIpBlocks.lookup(addr)
        if block
          locid = block.locid

          location = GeoIpLocations.find_by_locid(locid)
          if location
            # todo translate countrycode to country, region(code) to region
            # MSC: it seems fine to store countrycode; the UI can translate countrycode to country display name. same for region
            country = location.countrycode
            state = location.region
            city = location.city
          end
        end

        isp = JamIsp.lookup(addr)
        if isp
          ispid = isp.coid
        end
      end

      {city: city, state: state, country: country, addr: addr, locidispid: locid*1000000+ispid}
    end

    def self.createx(locid, countrycode, region, city, postalcode, latitude, longitude, metrocode, areacode)
      c = connection.raw_connection
      c.exec_params("insert into #{self.table_name} (locid, countrycode, region, city, postalcode, latitude, longitude, metrocode, areacode, geog) values($1, $2, $3, $4, $5, $6, $7, $8, $9, ST_SetSRID(ST_MakePoint($7, $6), 4326)::geography)",
                    [locid, countrycode, region, city, postalcode, latitude, longitude, metrocode, areacode])
    end

    def self.i(s)
      return 'NULL' if s.nil? or s.blank?
      return s.to_i
    end

    def self.where_latlng(relation, params, current_user=nil)
      # this is only valid to call when relation is about bands
      distance = params[:distance].to_i
      if distance > 0
        latlng = nil
        location_city = params[:city]
        location_state = params[:state]
        location_country = params[:country]
        remote_ip = params[:remote_ip]

        if location_city and location_state and location_country
          geo = self.where(city: location_city, region: location_state, countrycode: location_country).limit(1).first

          if geo and geo.latitude and geo.longitude and (geo.latitude != 0 or geo.longitude != 0)
            # it isn't reasonable for both to be 0...
            latlng = [geo.latitude, geo.longitude]
          end
        elsif current_user and current_user.locidispid and current_user.locidispid != 0
          location = GeoIpLocations.find_by_locid(current_user.locidispid/1000000)
          if location and location.latitude and location.longitude and (location.latitude != 0 or location.longitude != 0)
            # it isn't reasonable for both to be 0...
            latlng = [location.latitude, location.longitude]
          end
        elsif remote_ip
          geo = GeoIpBlocks.ip_lookup(remote_ip)
          geo = geo.location if geo

          if geo and geo.latitude and geo.longitude and (geo.latitude != 0 or geo.longitude != 0)
            # it isn't reasonable for both to be 0...
            latlng = [geo.latitude, geo.longitude]
          end
        end

        if latlng
          relation = relation.where(['latitude IS NOT NULL AND longitude IS NOT NULL']).within(distance, origin: latlng)
        end
      end
      relation
    end


    def self.import_from_max_mind(options)

      file = options[:file]
      use_copy = options[:use_copy]

      # File Geo-134
      # Format:
      # locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode

      start = Time.now

      copied_table_name = Database.copy_table(self.table_name)
      city_copied_table_name = Database.copy_table(City.table_name)

      if use_copy
        Database.copy(copied_table_name, file)
      else
        File.open(file, 'r:ISO-8859-1') do |io|
          s = io.gets.strip # eat the copyright line. gah, why do they have that in their file??
          unless s.eql? 'Copyright (c) 2012 MaxMind LLC.  All Rights Reserved.'
            puts s
            puts 'Copyright (c) 2012 MaxMind LLC.  All Rights Reserved.'
            raise 'file does not start with expected copyright (line 1): Copyright (c) 2012 MaxMind LLC.  All Rights Reserved.'
          end

          s = io.gets.strip # eat the headers line
          unless s.eql? 'locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode'
            puts s
            puts 'locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode'
            raise 'file does not start with expected header (line 2): locId,country,region,city,postalCode,latitude,longitude,metroCode,areaCode'
          end

          saved_level = ActiveRecord::Base.logger ? ActiveRecord::Base.logger.level : 0
          count = 0

          stmt = "INSERT INTO #{copied_table_name} (locid, countrycode, region, city, postalcode, latitude, longitude, metrocode, areacode) VALUES"

          vals = ''
          sep = ''
          i = 0
          n = 20

          csv = ::CSV.new(io, {encoding: 'ISO-8859-1', headers: false})
          csv.each do |row|
            raise "file does not have expected number of columns (9): #{row.length}" unless row.length == 9

            locid = row[0]
            countrycode = row[1]
            region = row[2]
            city = row[3]
            postalcode = row[4]
            latitude = row[5]
            longitude = row[6]
            metrocode = row[7]
            areacode = row[8]

            vals = vals+sep+"(#{locid}, '#{countrycode}', '#{region}', #{quote_value(city)}, '#{postalcode}', #{latitude}, #{longitude}, #{i(metrocode)}, '#{areacode}')"
            sep = ','
            i += 1

            if count == 0 or i >= n then
              self.connection.execute stmt+vals
              count += i
              vals = ''
              sep = ''
              i = 0

              if ActiveRecord::Base.logger and ActiveRecord::Base.logger.level > 1 then
                ActiveRecord::Base.logger.debug "... logging inserts into #{copied_table_name} suspended ..."
                ActiveRecord::Base.logger.level = 1
              end

              if ActiveRecord::Base.logger and count%10000 < n then
                ActiveRecord::Base.logger.level = saved_level
                ActiveRecord::Base.logger.debug "... inserted #{count} into #{copied_table_name} ..."
                ActiveRecord::Base.logger.level = 1
              end
            end
          end

          if i > 0 then
            self.connection.execute stmt+vals
            count += i
          end

          if ActiveRecord::Base.logger then
            ActiveRecord::Base.logger.level = saved_level
            ActiveRecord::Base.logger.debug "loaded #{count} records into #{copied_table_name}"
          end
        end
      end

      # create primary key index -- this will be renamed later in the import process
      GeoIpLocations.connection.execute("CREATE UNIQUE INDEX #{COPIED_PRIMARY_KEY_NAME} ON #{copied_table_name} USING btree (locid)").check
      GeoIpLocations.connection.execute("ALTER TABLE #{copied_table_name} ADD CONSTRAINT #{COPIED_PRIMARY_KEY_NAME} PRIMARY KEY USING INDEX #{COPIED_PRIMARY_KEY_NAME}").check


      sts = self.connection.execute "ALTER TABLE #{copied_table_name} DROP COLUMN geog;"
      ActiveRecord::Base.logger.debug "DROP COLUMN geog returned sts #{sts.cmd_status}" if ActiveRecord::Base.logger
      # sts.check [we don't care]

      sts = self.connection.execute "ALTER TABLE #{copied_table_name} ADD COLUMN geog geography(point, 4326);"
      ActiveRecord::Base.logger.debug "ADD COLUMN geog returned sts #{sts.cmd_status}" if ActiveRecord::Base.logger
      sts.check

      sts = self.connection.execute "UPDATE #{copied_table_name} SET geog = ST_SetSRID(ST_MakePoint(longitude, latitude), 4326)::geography;"
      ActiveRecord::Base.logger.debug "SET geog returned sts #{sts.cmd_tuples}" if ActiveRecord::Base.logger
      sts.check

      sts = self.connection.execute "CREATE INDEX #{COPIED_GEOIPLOCATIONS_INDEX_NAME} ON #{copied_table_name} USING GIST (geog);"
      ActiveRecord::Base.logger.debug "CREATE INDEX #{COPIED_GEOIPLOCATIONS_INDEX_NAME} returned sts #{sts.cmd_status}" if ActiveRecord::Base.logger
      sts.check

      sts = self.connection.execute "INSERT INTO #{city_copied_table_name} (city, region, countrycode) SELECT DISTINCT city, region, countrycode FROM #{copied_table_name} WHERE length(city) > 0 AND length(countrycode) > 0;"
      ActiveRecord::Base.logger.debug "INSERT INTO #{city_copied_table_name} returned sts #{sts.cmd_status}" if ActiveRecord::Base.logger
      sts.check

      elapsed = Time.now - start
      @@log.debug("#{copied_table_name} import took #{elapsed} seconds")
    end

    def self.after_maxmind_import
      # handle geoiplocations
      self.connection.execute("DROP TABLE #{self.table_name}").check
      self.connection.execute("ALTER INDEX #{COPIED_PRIMARY_KEY_NAME} RENAME TO #{PRIMARY_KEY_NAME}").check
      self.connection.execute("ALTER INDEX #{COPIED_GEOIPLOCATIONS_INDEX_NAME} RENAME TO #{GEOIPLOCATIONS_INDEX_NAME}").check
      self.connection.execute("ALTER TABLE #{self.table_name}_copied RENAME TO #{self.table_name}").check

      # handle cities
      self.connection.execute("DROP TABLE #{City.table_name}").check
      self.connection.execute("ALTER TABLE #{City.table_name}_copied RENAME TO #{City.table_name}").check
    end
  end
end