bump version for encoding fix

Using Marshal, instead of YAML. Reduces my library of +28,000 commands
from 168Mb to 2.1Mb. Also reduces the run time from 4.5 minutes to
5secs.

Included a check for whether the existing [old] format is used, and
migrate it to the new version.

TODOs

@@ -0,0 +1,17 @@
+* add a ~/.persistent-shell-history.yaml
+* move the db from gdbm to sqlite
+ - tables
+  ** hostname
+   -- id
+   -- name
+  ** command
+   -- id
+   -- value
+  ** record
+   -- id
+   -- timestamp
+   -- hostname_id
+   -- command_id
+* migration from current Marshal version need only to check the value, for if
+  it starts with "\x04\b{"
+

bin/bash_history.rb

@@ -1,11 +1,12 @@
 #!/usr/bin/env ruby
 
+require 'persistent-shell-history/binary-history-store'
 require 'persistent-shell-history/old-history-store'
 require 'optparse'
 
 options = {}
-bh_options = {}
-OptionParser.new do |opts|
+bh_options = {:archive_file => File.expand_path("~/.bash_history.db")}
+opts = OptionParser.new do |opts|
   opts.on('--inspect','inspect the data') do |o|
     options[:inspect] = o
   end
@@ -15,49 +16,73 @@ OptionParser.new do |opts|
   opts.on('-d','--db FILE','use database FILE instead of the default (~/.bash_history.db)') do |o|
     bh_options[:archive_file] = o
   end
+  opts.on('--migrate','check-for and migrate, only') do |o|
+    options[:migrate] = o
+  end
   opts.on('-l','--list','list history') do |o|
     options[:list] = o
   end
-  opts.on('--fix','fix times') do |o|
-    options[:fix] = o
-  end
   opts.on('--format FORMAT','specify a different strftime format. (default is "%F %T")') do |o|
     bh_options[:time_format] = o
   end
   opts.on('-f','--find PAT','find a command with pattern PAT') do |o|
     options[:find] = o
   end
-end.parse!(ARGV)
+end
 
+begin
+  opts.parse!(ARGV)
+rescue => ex
+  puts ex
+  puts opts
+end
+
+def migrate(old_bashhistorystore)
+  require 'fileutils'
+  # migrate
+  temp_db = GDBM.new("#{old_bashhistorystore.archive}.#{$$}")
+  old_bashhistorystore.keys.each {|key|
+    h = old_bashhistorystore[key]
+    h[:time] = h[:time].map {|t| t.to_i }
+    temp_db[key] = Marshal.dump(h)
+  }
+  old_bashhistorystore.db.close()
+  temp_db.close()
+  ts = "#{Time.now.year}#{Time.now.month}#{Time.now.day}#{Time.now.hour}#{Time.now.min}"
+  old_filename = "#{old_bashhistorystore.archive}.old.#{ts}"
+  FileUtils.mv(old_bashhistorystore.archive, old_filename)
+  puts "archived [#{old_bashhistorystore.archive}] to [#{old_filename}] ..."
+  FileUtils.mv("#{old_bashhistorystore.archive}.#{$$}", old_bashhistorystore.archive)
+end
+
+# First check the database, for whether it is the old format,
+# if so, convert it, and reopen it.
 bh = Persistent::Shell::OldHistoryStore.new(bh_options)
+migrate(bh) if bh.is_oldformat?
+bh.db.close unless bh.db.closed?
+
+exit(0) if options[:migrate]
+
+# re-open the history storage
+bh = Persistent::Shell::BinaryHistoryStore.new(bh_options)
+
+# load the new bash_history into the database
+unless (options[:inspect] or options[:find] or options[:list])
+  bh.load()
+  bh.db.reorganize()
+end
 
 if options[:inspect]
   p bh
-  p "storing #{bh.keys.count} commands"
-end
-if options[:fix]
-  count = 0
-  bh.db.each_pair do |k,v|
-    yv = bh._yl(v)
-    if yv[:time].nil?
-      yv[:time] = [Time.at(0)]
-      bh.db[k] = bh._yd(yv)
-      count += 1
-    elsif not yv[:time].kind_of? Array
-      yv[:time] = [yv[:time]]
-      bh.db[k] = bh._yd(yv)
-      count += 1
-    end
-  end
-  puts "fixed [#{count}] times values"
+  #p "storing #{bh.keys.count} commands"
 end
 if options[:find]
   bh.find(options[:find]).sort_by {|x| x[:time] }.each do |val|
-    puts bh._f(val)
+    puts bh.fmt(val)
   end
 elsif options[:list]
   bh.values_by_time.each do |val|
-    puts bh._f(val)
+    puts bh.fmt(val)
   end
 end
 

lib/persistent-shell-history.rb

@@ -1,5 +1,5 @@
 require 'persistent-shell-history/version'
-require 'persistent-shell-history/bash-history'
+require 'persistent-shell-history/history'
 
 module Persistent
   module Shell

lib/persistent-shell-history/binary-history-store.rb

@@ -0,0 +1,129 @@
+require 'digest/md5'
+require 'gdbm'
+require 'yaml'
+
+require 'persistent-shell-history/abstract-history-store'
+require 'persistent-shell-history/history'
+require 'persistent-shell-history/command'
+
+if RUBY_VERSION >= "1.9" # assuming you're running Ruby ~1.9
+  Encoding.default_external = Encoding::UTF_8
+  Encoding.default_internal = Encoding::UTF_8
+end
+
+module Persistent
+  module Shell
+    class BinaryHistoryStore < AbstractHistoryStore
+      OPTIONS = {
+        :file          => File.expand_path("~/.bash_history"),
+        :archive_file  => File.expand_path("~/.bash_history.db"),
+        :time_format   => "%F %T",
+      }
+
+      def initialize(opts = {})
+        @options = OPTIONS.merge(opts)
+      end
+
+      def archive; @options[:archive_file]; end
+      def archive=(arg); @options[:archive_file] = arg; end
+
+      def hist_file; @options[:file]; end
+      def hist_file=(arg); @options[:file] = arg; end
+
+      def time_format; @options[:time_format]; end
+      def time_format=(tf); @options[:time_format] = tf; end
+      def db
+        @db ||= GDBM.new(@options[:archive_file])
+      end
+      def [](key); _ml(db[key]); end
+      def keys; db.keys; end
+      def values; db.map {|k,v| _ml(v) }; end
+      def values_by_time
+        return db.map {|k,v|
+          data = _ml(v)
+          data[:time].map {|t|
+            data.merge(:time => t)
+          }
+        }.flatten.sort_by {|x|
+          x[:time]
+        }
+      end
+      def commands; values.map {|v| v[:cmd] }; end
+      def _md(data); Marshal.dump(data); end
+      def _ml(data); Marshal.load(data); end
+      def _md5(data); Digest::MD5.hexdigest(data); end
+
+      # display a formatted time commend
+      def fmt(cmd); " %s %s" % [Time.at(cmd[:time]).strftime(@options[:time_format]), cmd[:cmd]]; end
+
+      def find(pat)
+        return values.select {|v|
+          v if v[:cmd] =~ /#{pat}/
+        }.map {|v|
+          v[:time].map {|t|
+            v.merge(:time => t)
+          }
+        }.flatten
+      end
+
+      def load(filename = @options[:file])
+        open(filename) do |f|
+          f.each_line do |line|
+            if line =~ /^#(.*)$/
+              l = f.readline.chomp
+              key = _md5(l)
+              if db.has_key?(key)
+                times = _ml(db[key])[:time]
+                if times.kind_of? Array
+                  times.push($1.to_i)
+                else
+                  times = [times]
+                end
+                db[key] = _md({:cmd => l, :time => times.uniq })
+              else
+                db[key] = _md({:cmd => l, :time => [$1.to_i] })
+              end
+            else
+              key = _md5(line.chomp)
+              if db.has_key?(key)
+                times = _ml(db[key])[:time]
+                if times.kind_of? Array
+                  times.push($1.to_i)
+                else
+                  times = [times]
+                end
+                db[key] = _md({:cmd => l, :time => times.uniq })
+              else
+                db[key] = _md({:cmd => line.chomp, :time => [0] })
+              end
+            end
+          end
+        end
+      end
+
+      # returns a Persistent::Shell::History object from the current GDBM database.
+      # intended for marshalling to other history-stores
+      def to_history
+        history = History.new
+        values.each do |value|
+          value[:time].each do |t|
+            history << Command.new(value[:cmd], t.to_i)
+          end
+        end
+        return history
+      end
+
+      # create an output that looks like a regular ~/.bash_history file
+      def render(file)
+        File.open(file,'w+') do |f|
+          values.each do |v|
+            f.write("#" + v[:time].to_i.to_s + "\n") if v[:time] and not (v[:time].to_i == 0)
+            f.write(v[:cmd] + "\n")
+          end
+        end
+      end
+
+    end # class BinaryHistoryStore
+  end # Shell
+end # Persistent
+

lib/persistent-shell-history/command.rb

@@ -1,5 +1,7 @@
 
 require 'digest/md5'
+require 'json'
+
 module Persistent
   module Shell
     class Command < Struct.new(:cmd, :time)
@@ -9,6 +11,9 @@ module Persistent
       def to_h
         { :cmd => cmd, :time => time, }
       end
+      def to_json(*a)
+        to_h.to_json(*a)
+      end
     end
   end
 end

lib/persistent-shell-history/history.rb

@@ -1,17 +1,27 @@
 
 require 'persistent-shell-history/command'
+require 'json'
 
 module Persistent
   module Shell
+    # Abstract storage for command history
     class History
+      def initialize()
+        @cmds = Array.new
+      end
+      def commands; @cmds; end
+      def commands=(cmds); @cmds = cmds; end
+      def <<(arg); @cmds << arg; end
+      def to_a; commands.map {|c| c.to_h }; end
+      def to_json(*a); commands.to_json(*a); end
+    end
+    class BashHistory < History
       def initialize(filename = '~/.bash_history')
         @filename = File.expand_path(filename)
       end
+      def commands; (@cmds.nil? or @cmds.empty?) ? (@cmds = parse) : @cmds; end
       def file; @filename; end
       def file=(filename); @filename = File.expand_path(filename); end
-      def commands
-        @cmds ||= parse
-      end
       def parse(filename = @filename)
         cmds = Array.new
         open(filename) do |f|

lib/persistent-shell-history/old-history-store.rb

@@ -4,25 +4,49 @@ require 'gdbm'
 require 'yaml'
 
 require 'persistent-shell-history/abstract-history-store'
+require 'persistent-shell-history/history'
+require 'persistent-shell-history/command'
 
 module Persistent
   module Shell
     class OldHistoryStore < AbstractHistoryStore
-      SCHEMA_VERSION = "1"
       OPTIONS = {
         :file          => File.expand_path("~/.bash_history"),
         :archive_file  => File.expand_path("~/.bash_history.db"),
         :time_format   => "%F %T",
       }
+
       def initialize(opts = {})
         @options = OPTIONS.merge(opts)
-        _load unless db.has_key? "schema_version"
       end
+
+      def archive; @options[:archive_file]; end
+      def archive=(arg); @options[:archive_file] = arg; end
+
+      def hist_file; @options[:file]; end
+      def hist_file=(arg); @options[:file] = arg; end
+
+      # check the archive, whether it's the old format.
+      # If so, it needs to be converted to the BinaryHistoryStore
+      def is_oldformat?
+        db.keys[0..5].each {|key|
+          begin
+            YAML.load(db[key])
+          rescue Psych::SyntaxError => ex
+            return false
+          rescue => ex
+            return false
+          end
+        }
+        return true
+      end
+
       def time_format; @options[:time_format]; end
       def time_format=(tf); @options[:time_format] = tf; end
       def db
         @db ||= GDBM.new(@options[:archive_file])
       end
+      def [](key); _yl(db[key]); end
       def keys; db.keys; end
       def keys_to_i; keys.map {|i| i.to_i }; end
       def values; db.map {|k,v| _yl(v) }; end
@@ -40,7 +64,9 @@ module Persistent
       def _yd(data); YAML.dump(data); end
       def _yl(data); YAML.load(data); end
       def _md5(data); Digest::MD5.hexdigest(data); end
-      def _f(v); " %s %s" % [v[:time].strftime(@options[:time_format]), v[:cmd]]; end
+
+      # display a formatted time commend
+      def fmt(cmd); " %s %s" % [cmd[:time].strftime(@options[:time_format]), cmd[:cmd]]; end
     
       def find(pat)
         return values.select {|v|
@@ -52,8 +78,7 @@ module Persistent
         }.flatten
       end
     
-      def _load(filename = @options[:file])
-        #History.new(file).parse
+      def load(filename = @options[:file])
         open(filename) do |f|
           f.each_line do |line|
             if line =~ /^#(.*)$/
@@ -88,6 +113,19 @@ module Persistent
         end
       end
 
+      # returns a Persistent::Shell::History object from the current GDBM database.
+      # intended for marshalling to other history-stores
+      def to_history
+        history = History.new
+        values.each do |value|
+          value[:time].each do |t|
+            history << Command.new(value[:cmd], t.to_i)
+          end
+        end
+        return history
+      end
+
+      # create an output that looks like a regular ~/.bash_history file
       def render(file)
         File.open(file,'w+') do |f|
           values.each do |v|

lib/persistent-shell-history/version.rb

@@ -1,6 +1,6 @@
 module Persistent
   module Shell
-    VERSION = "0.0.1"
+    VERSION = "0.0.3"
   end
 end
 

persistent-shell-history.gemspec

@@ -16,6 +16,7 @@ See README for other implementation helpers.
 }
 
   s.rubyforge_project = "persistent-shell-history"
+  s.add_dependency("json")
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")