mirrors
/
persistent-shell-history
mirror of https://github.com/vbatts/persistent-shell-history.git
1
0
Fork 0
Code Issues Releases Wiki Activity

BIG TIME CHANGES. 

Using Marshal, instead of YAML. Reduces my library of +28,000 commands
from 168Mb to 2.1Mb. Also reduces the run time from 4.5 minutes to
5secs.

Included a check for whether the existing [old] format is used, and
migrate it to the new version.
This commit is contained in:
Vincent Batts 2012-09-21 13:17:38 -04:00
parent 789ea644ac
commit 3617b25466
6 changed files with 207 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
bin/bash_history.rb
View File

@ -1,11 +1,12 @@
#!/usr/bin/env ruby
require 'persistent-shell-history/binary-history-store'
require 'persistent-shell-history/old-history-store'
require 'optparse'
options = {}
bh_options = {}
OptionParser.new do |opts|
bh_options = {:archive_file => File.expand_path("~/.bash_history.db")}
opts = OptionParser.new do |opts|
opts.on('--inspect','inspect the data') do |o|
options[:inspect] = o
end
@ -15,50 +16,70 @@ OptionParser.new do |opts|
opts.on('-d','--db FILE','use database FILE instead of the default (~/.bash_history.db)') do |o|
bh_options[:archive_file] = o
end
opts.on('--migrate','check-for and migrate, only') do |o|
options[:migrate] = o
end
opts.on('-l','--list','list history') do |o|
options[:list] = o
end
opts.on('--fix','fix times') do |o|
options[:fix] = o
end
opts.on('--format FORMAT','specify a different strftime format. (default is "%F %T")') do |o|
bh_options[:time_format] = o
end
opts.on('-f','--find PAT','find a command with pattern PAT') do |o|
options[:find] = o
end
end.parse!(ARGV)
end
begin
opts.parse!(ARGV)
rescue => ex
puts ex
puts opts
end
def migrate(old_bashhistorystore)
require 'fileutils'
# migrate
temp_db = GDBM.new("#{old_bashhistorystore.archive}.#{$$}")
old_bashhistorystore.keys.each {|key|
h = old_bashhistorystore[key]
h[:time] = h[:time].map {|t| t.to_i }
temp_db[key] = Marshal.dump(h)
}
old_bashhistorystore.db.close()
temp_db.close()
ts = "#{Time.now.year}#{Time.now.month}#{Time.now.day}#{Time.now.hour}#{Time.now.min}"
old_filename = "#{old_bashhistorystore.archive}.old.#{ts}"
FileUtils.mv(old_bashhistorystore.archive, old_filename)
puts "archived [#{old_bashhistorystore.archive}] to [#{old_filename}] ..."
FileUtils.mv("#{old_bashhistorystore.archive}.#{$$}", old_bashhistorystore.archive)
end
# First check the database, for whether it is the old format,
# if so, convert it, and reopen it.
bh = Persistent::Shell::OldHistoryStore.new(bh_options)
bh.load()
migrate(bh) if bh.is_oldformat?
bh.db.close unless bh.db.closed?
exit(0) if options[:migrate]
# re-open the history storage
bh = Persistent::Shell::BinaryHistoryStore.new(bh_options)
# load the new bash_history into the database
bh.load() unless (options[:inspect] or options[:find] or options[:list])
if options[:inspect]
p bh
p "storing #{bh.keys.count} commands"
end
if options[:fix]
count = 0
bh.db.each_pair do |k,v|
yv = bh._yl(v)
if yv[:time].nil?
yv[:time] = [Time.at(0)]
bh.db[k] = bh._yd(yv)
count += 1
elsif not yv[:time].kind_of? Array
yv[:time] = [yv[:time]]
bh.db[k] = bh._yd(yv)
count += 1
end
end
puts "fixed [#{count}] times values"
end
if options[:find]
bh.find(options[:find]).sort_by {|x| x[:time] }.each do |val|
puts bh._f(val)
puts bh.fmt(val)
end
elsif options[:list]
bh.values_by_time.each do |val|
puts bh._f(val)
puts bh.fmt(val)
end
end

125
lib/persistent-shell-history/binary-history-store.rb Normal file
View File

@ -0,0 +1,125 @@
require 'digest/md5'
require 'gdbm'
require 'yaml'
require 'persistent-shell-history/abstract-history-store'
require 'persistent-shell-history/history'
require 'persistent-shell-history/command'
module Persistent
module Shell
class BinaryHistoryStore < AbstractHistoryStore
OPTIONS = {
:file => File.expand_path("~/.bash_history"),
:archive_file => File.expand_path("~/.bash_history.db"),
:time_format => "%F %T",
}
def initialize(opts = {})
@options = OPTIONS.merge(opts)
end
def archive; @options[:archive_file]; end
def archive=(arg); @options[:archive_file] = arg; end
def hist_file; @options[:file]; end
def hist_file=(arg); @options[:file] = arg; end
def time_format; @options[:time_format]; end
def time_format=(tf); @options[:time_format] = tf; end
def db
@db ||= GDBM.new(@options[:archive_file])
end
def [](key); _ml(db[key]); end
def keys; db.keys; end
def values; db.map {|k,v| _ml(v) }; end
def values_by_time
return db.map {|k,v|
data = _ml(v)
data[:time].map {|t|
data.merge(:time => t)
}
}.flatten.sort_by {|x|
x[:time]
}
end
def commands; values.map {|v| v[:cmd] }; end
def _md(data); Marshal.dump(data); end
def _ml(data); Marshal.load(data); end
def _md5(data); Digest::MD5.hexdigest(data); end
# display a formatted time commend
def fmt(cmd); " %s %s" % [Time.at(cmd[:time]).strftime(@options[:time_format]), cmd[:cmd]]; end
def find(pat)
return values.select {|v|
v if v[:cmd] =~ /#{pat}/
}.map {|v|
v[:time].map {|t|
v.merge(:time => t)
}
}.flatten
end
def load(filename = @options[:file])
open(filename) do |f|
f.each_line do |line|
if line =~ /^#(.*)$/
l = f.readline.chomp
key = _md5(l)
if db.has_key?(key)
times = _ml(db[key])[:time]
if times.kind_of? Array
times.push($1.to_i)
else
times = [times]
end
db[key] = _md({:cmd => l, :time => times.uniq })
else
db[key] = _md({:cmd => l, :time => [$1.to_i] })
end
else
key = _md5(line.chomp)
if db.has_key?(key)
times = _ml(db[key])[:time]
if times.kind_of? Array
times.push($1.to_i)
else
times = [times]
end
db[key] = _md({:cmd => l, :time => times.uniq })
else
db[key] = _md({:cmd => line.chomp, :time => [0] })
end
end
end
end
end
# returns a Persistent::Shell::History object from the current GDBM database.
# intended for marshalling to other history-stores
def to_history
history = History.new
values.each do |value|
value[:time].each do |t|
history << Command.new(value[:cmd], t.to_i)
end
end
return history
end
# create an output that looks like a regular ~/.bash_history file
def render(file)
File.open(file,'w+') do |f|
values.each do |v|
f.write("#" + v[:time].to_i.to_s + "\n") if v[:time] and not (v[:time].to_i == 0)
f.write(v[:cmd] + "\n")
end
end
end
end # class BinaryHistoryStore
end # Shell
end # Persistent

5
lib/persistent-shell-history/command.rb
View File

@ -1,5 +1,7 @@
require 'digest/md5'
require 'json'
module Persistent
module Shell
class Command < Struct.new(:cmd, :time)
@ -9,6 +11,9 @@ module Persistent
def to_h
{ :cmd => cmd, :time => time, }
end
def to_json(*a)
to_h.to_json(*a)
end
end
end
end

3
lib/persistent-shell-history/history.rb
View File

@ -1,5 +1,6 @@
require 'persistent-shell-history/command'
require 'json'
module Persistent
module Shell
@ -11,6 +12,8 @@ module Persistent
def commands; @cmds; end
def commands=(cmds); @cmds = cmds; end
def <<(arg); @cmds << arg; end
def to_a; commands.map {|c| c.to_h }; end
def to_json(*a); commands.to_json(*a); end
end
class BashHistory < History
def initialize(filename = '~/.bash_history')

30
lib/persistent-shell-history/old-history-store.rb
View File

@ -10,21 +10,43 @@ require 'persistent-shell-history/command'
module Persistent
module Shell
class OldHistoryStore < AbstractHistoryStore
SCHEMA_VERSION = "1"
OPTIONS = {
:file => File.expand_path("~/.bash_history"),
:archive_file => File.expand_path("~/.bash_history.db"),
:time_format => "%F %T",
}
def initialize(opts = {})
@options = OPTIONS.merge(opts)
#load() unless db.has_key? "schema_version"
end
def archive; @options[:archive_file]; end
def archive=(arg); @options[:archive_file] = arg; end
def hist_file; @options[:file]; end
def hist_file=(arg); @options[:file] = arg; end
# check the archive, whether it's the old format.
# If so, it needs to be converted to the BinaryHistoryStore
def is_oldformat?
db.keys[0..5].each {|key|
begin
YAML.load(db[key])
rescue Psych::SyntaxError => ex
return false
rescue => ex
return false
end
}
return true
end
def time_format; @options[:time_format]; end
def time_format=(tf); @options[:time_format] = tf; end
def db
@db ||= GDBM.new(@options[:archive_file])
end
def [](key); _yl(db[key]); end
def keys; db.keys; end
def keys_to_i; keys.map {|i| i.to_i }; end
def values; db.map {|k,v| _yl(v) }; end
@ -42,7 +64,9 @@ module Persistent
def _yd(data); YAML.dump(data); end
def _yl(data); YAML.load(data); end
def _md5(data); Digest::MD5.hexdigest(data); end
def _f(v); " %s %s" % [v[:time].strftime(@options[:time_format]), v[:cmd]]; end
# display a formatted time commend
def fmt(cmd); " %s %s" % [cmd[:time].strftime(@options[:time_format]), cmd[:cmd]]; end
def find(pat)
return values.select {|v|

1
persistent-shell-history.gemspec
View File

@ -16,6 +16,7 @@ See README for other implementation helpers.
}
s.rubyforge_project = "persistent-shell-history"
s.add_dependency("json")
s.files = `git ls-files`.split("\n")
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")