class Redis::Dump

  1. lib/redis/dump.rb
Superclass: Object

Constants

VALID_TYPES = ['string', 'set', 'list', 'zset', 'hash', 'none'].freeze  

Attributes

chunk_size [RW]
debug [RW]
encoder [RW]
host [RW]
parser [RW]
password [RW]
port [RW]
safe [RW]
with_base64 [RW]
with_optimizations [RW]
dbs [RW]
redis_connections [R]
uri [RW]

Public Class methods

check_utf8= (check)
[show source] 
# File lib/redis/dump.rb, line 34
def check_utf8=(check)
  if check == false
    @parser = Yajl::Parser.new(:check_utf8 => false)
  end
end
ld (msg)
[show source] 
# File lib/redis/dump.rb, line 28
def ld(msg)
  STDERR.puts "#%.4f: %s" % [Time.now.utc.to_f, msg] if debug
end
le (msg)
[show source] 
# File lib/redis/dump.rb, line 25
def le(msg)
  STDERR.puts "#%.4f: %s" % [Time.now.utc.to_f, msg]
end
memory_usage ()
[show source] 
# File lib/redis/dump.rb, line 31
def memory_usage
  %x`ps -o rss= -p #{Process.pid}`.to_i # in kb
end
new (dbs=nil, uri="redis://#{Redis::Dump.host}:#{Redis::Dump.port}")
[show source] 
# File lib/redis/dump.rb, line 42
def initialize(dbs=nil, uri="redis://#{Redis::Dump.host}:#{Redis::Dump.port}")
  @redis_connections = {}
  @uri = uri
  unless dbs.nil?
    @dbs = Range === dbs ? dbs : (dbs..dbs)
    @dbs = (@dbs.first.to_i..@dbs.last.to_i) # enforce integers
    @dbs.to_a.each { |db| redis(db) } # open_all_connections
  end
end

Public Instance methods

connect (this_uri)
[show source] 
# File lib/redis/dump.rb, line 54
def connect(this_uri)
  self.class.ld 'CONNECT: ' << this_uri
  opts = {
    :url => this_uri
  }
  opts[:password] = Redis::Dump.password if Redis::Dump.password
  Redis.new **opts
end
dump (filter=nil)
[show source] 
# File lib/redis/dump.rb, line 69
def dump filter=nil
  filter ||= '*'
  entries = []
  each_database do |redis|
    chunk_entries = []
    dump_keys = redis.keys(filter)
    dump_keys_size = dump_keys.size
    Redis::Dump.ld "Memory after loading keys: #{Redis::Dump.memory_usage}kb"
    dump_keys.each_with_index do |key,idx|
      entry, idxplus = key, idx+1
      if block_given?
        chunk_entries << entry
        process_chunk idx, dump_keys_size do |count|
          Redis::Dump.ld " dumping #{chunk_entries.size} (#{count}) from #{redis.connection[:id]}"
          output_buffer = []
          chunk_entries = chunk_entries.select do |key|
            type = Redis::Dump.type(redis, key)
            if self.class.with_optimizations && type == 'string'
              true
            else
              output_buffer.push self.class.encoder.encode(Redis::Dump.dump(redis, key, type))
              false
            end
          end
          unless output_buffer.empty?
            yield output_buffer
          end
          unless chunk_entries.empty?
            yield Redis::Dump.dump_strings(redis, chunk_entries) { |obj| self.class.encoder.encode(obj) }
          end
          output_buffer.clear
          chunk_entries.clear
        end
      else
        entries << self.class.encoder.encode(Redis::Dump.dump(redis, entry))
      end
    end
  end
  entries
end
each_database ()
[show source] 
# File lib/redis/dump.rb, line 63
def each_database
  @redis_connections.keys.sort.each do |redis_uri|
    yield redis_connections[redis_uri]
  end
end
load (string_or_stream, &each_record)
[show source] 
# File lib/redis/dump.rb, line 143
def load(string_or_stream, &each_record)
  count = 0
  Redis::Dump.ld " LOAD SOURCE: #{string_or_stream}"
  Redis::Dump.parser.parse string_or_stream do |obj|
    unless @dbs.member?(obj["db"].to_i)
      Redis::Dump.ld "db out of range: #{obj["db"]}"
      next
    end
    this_redis = redis(obj["db"])
    Redis::Dump.ld "load[#{this_redis.hash}, #{obj}]"
    if each_record.nil?
      if Redis::Dump.safe && this_redis.exists(obj['key'])
        Redis::Dump.ld " record exists (no change)"
        next
      end
      begin
        val, type = obj['value'], obj['type']
        if Redis::Dump.with_base64 && type === 'string'
          val = Base64.decode64 val
        end
        ret = Redis::Dump.set_value this_redis, obj['key'], type, val, obj['ttl']
      rescue => ex
        Redis::Dump.le '(key: %s) %s' % [obj['key'], ex.message]
      end
    else
      each_record.call obj
    end
    count += 1
  end
  count
end
redis (db)
[show source] 
# File lib/redis/dump.rb, line 51
def redis(db)
  redis_connections["#{uri}/#{db}"] ||= connect("#{uri}/#{db}")
end
report (filter='*')
[show source] 
# File lib/redis/dump.rb, line 116
def report filter='*'
  values = []
  total_size, dbs = 0, {}
  each_database do |redis|
    chunk_entries = []
    dump_keys = redis.keys(filter)
    dump_keys_size = dump_keys.size
    dump_keys.each_with_index do |key,idx|
      entry, idxplus = Redis::Dump.report(redis, key), idx+1
      chunk_entries << entry
      process_chunk idx, dump_keys_size do |count|
        Redis::Dump.ld " reporting on #{chunk_entries.size} (#{idxplus}) from #{redis.connection[:id]}"
        chunk_entries.each do |e|
          puts record if obj.global.verbose >= 1
          dbs[e['db']] ||= 0
          dbs[e['db']] += e['size']
          total_size += e['size']
        end
        chunk_entries.clear
      end
    end
  end
  puts dbs.keys.sort.collect { |db| "  db#{db}: #{dbs[db].to_bytes}" }
  puts "total: #{total_size.to_bytes}"
  values
end