Updated Global Object's Recent Terms Refresh. Started to add an OAI Harvester for Literary Sources.

build.xml

@@ -107,6 +107,12 @@
 			<fileset dir="${basedir}/lib/jwebunit">
 				<include name="*.jar"/>
 			</fileset>
+			<fileset dir="${basedir}/lib/dlese-oai">
+				<include name="*.jar"/>
+			</fileset>
+			<fileset dir="${basedir}/lib/oai-xmlbeans">
+				<include name="*.jar"/>
+			</fileset>
 		</copy>
 
 	</target>

src/java/org/thdl/lex/Global.java

@@ -17,8 +17,6 @@ public class Global
 	private static long lastRefresh;
 	private int entryCount;
 	private List recentTerms;
-	private final GlobalRefresher REFRESHER = new GlobalRefresher();
-	private final Thread REFRESHER_THREAD = new Thread( REFRESHER );
 
 
 	/**
@@ -112,7 +110,7 @@ public class Global
 	 */
 	public void setRecentTerms( List recentTerms )
 	{
-		this.recentTerms = Collections.synchronizedList( recentTerms );
+		this.recentTerms = recentTerms;
 		setLastRefresh( System.currentTimeMillis() );
 	}
 
@@ -136,9 +134,9 @@ public class Global
 	 */
 	public List getRecentTerms() throws LexRepositoryException
 	{
-		if ( null == recentTerms )
+		if ( null == recentTerms || requiresRefresh() )
 		{
-			doRefresh();
+			refresh();
 		}
 		return recentTerms;
 	}
@@ -155,9 +153,12 @@ public class Global
 		long now = System.currentTimeMillis();
 		long lastUpdate = LexComponentRepository.getLastUpdate();
 		long sinceLastRefresh = now - getLastRefresh();
+
+		//LexLogger.debug( "Requires Refresh Logic: if " + sinceLastRefresh + " > " + getRefreshDelay() + " && " + lastUpdate + " > " + getLastRefresh() );
 		if ( sinceLastRefresh > getRefreshDelay() && lastUpdate > getLastRefresh() )
 		{
 			requiresRefresh = true;
+			LexLogger.debug( "Refresh Required at: " + System.currentTimeMillis() );
 		}
 		return requiresRefresh;
 	}
@@ -165,36 +166,27 @@ public class Global
 
 	/**
 	 *  Description of the Method
-	 *
-	 * @exception  LexRepositoryException  Description of the Exception
 	 */
-	public void refresh() throws LexRepositoryException
+	public void refresh()
 	{
-		if ( requiresRefresh() )
+		try
 		{
-			doRefresh();
+			int limit = getRecentTermsCount();
+			LexLogger.debug( "GlobalRefresher is starting a refresh for the " + limit + " most recent terms." );
+			setRecentTerms( LexComponentRepository.getRecentTerms( limit ) );
+			LexLogger.debug( "GlobalRefresher is finished refreshing..." );
+			LexLogger.debug( "Here's the new recent terms list: " + getRecentTerms().toString() );
+			LexComponentRepository.cleanup();
+			LexLogger.info( "GlobalRefresher finished a refresh..." );
+		}
+		catch ( Exception e )
+		{
+			StringWriter writer = new StringWriter();
+			e.printStackTrace( new PrintWriter( writer ) );
+			String stackTrace = writer.getBuffer().toString();
+			LexLogger.error( "GlobalRefresher Thread caught an Exception: " + stackTrace );
 		}
-	}
 
-
-	/**
-	 *  Description of the Method
-	 *
-	 * @exception  LexRepositoryException  Description of the Exception
-	 */
-	public void doRefresh() throws LexRepositoryException
-	{
-		LexLogger.info( "Checking if GlobalRefresher is busy..." );
-		if ( REFRESHER.getFinished() )
-		{
-			LexLogger.info( "GlobalRefresher is not busy. Starting refresh..." );
-			REFRESHER.setFinished( false );
-			REFRESHER_THREAD.start();
-		}
-		else
-		{
-			LexLogger.info( "GlobalRefresher was busy. Refresh not started..." );
-		}
 	}
 
 
@@ -216,79 +208,5 @@ public class Global
 		setRefreshDelay( refreshDelay );
 	}
 
-
-	/**
-	 *  Description of the Class
-	 *
-	 * @author     travis
-	 * @created    October 21, 2003
-	 */
-	class GlobalRefresher implements Runnable
-	{
-		private boolean finished;
-
-
-		/**
-		 *  Sets the finished attribute of the GlobalRefresher object
-		 *
-		 * @param  finished  The new finished value
-		 */
-		private void setFinished( boolean finished )
-		{
-			this.finished = finished;
-		}
-
-
-		/**
-		 *  Gets the finished attribute of the GlobalRefresher object
-		 *
-		 * @return    The finished value
-		 */
-		private boolean getFinished()
-		{
-			return finished;
-		}
-
-
-		/**
-		 *  Main processing method for the GlobalRefresher object
-		 */
-		public void run()
-		{
-			int limit = getRecentTermsCount();
-			try
-			{
-				LexLogger.info( "GlobalRefresher is starting a refresh..." );
-				setRecentTerms( LexComponentRepository.getRecentTerms( limit ) );
-				/*
-				    ILexComponent ilc = null;
-				    for ( Iterator it = getRecentTerms().iterator(); it.hasNext(); ilc = (ILexComponent) it.next() )
-				    {
-				    ilc.getMeta();
-				    }
-				  */
-				LexComponentRepository.cleanup();
-				LexLogger.info( "GlobalRefresher finished a refresh..." );
-				setFinished( true );
-			}
-			catch ( Exception e )
-			{
-				StringWriter writer = new StringWriter();
-				e.printStackTrace( new PrintWriter( writer ) );
-				String stackTrace = writer.getBuffer().toString();
-				LexLogger.error( "GlobalRefresher Thread caught an Exception: " + stackTrace );
-			}
-			setFinished( true );
-		}
-
-
-		/**
-		 *Constructor for the GlobalRefresher object
-		 */
-		GlobalRefresher()
-		{
-			setFinished( true );
-		}
-	}
 }
 

src/java/org/thdl/lex/LexActionServlet.java

@@ -65,17 +65,25 @@ public class LexActionServlet extends HttpServlet
 	 */
 	public void init( ServletConfig config ) throws ServletException
 	{
-		super.init( config );
-		initCommands();
-		config.getServletContext().setAttribute( "flatData", new LexFlatDataRepository() );
-		String delay = config.getInitParameter( "globalDataRefreshDelay" );
-		long refreshDelay = Long.parseLong( delay ) * 1000 * 60;
-		String recent = config.getInitParameter( "recentItems" );
-		int recentItems = Integer.parseInt( recent );
-		Global global = new Global( recentItems, refreshDelay );
-		config.getServletContext().setAttribute( LexConstants.GLOBAL_CONTEXT_ATTR, global );
-		LexLogger.debugComponent( global );
-		System.setProperty( "java.awt.headless", "true" );
+		try
+		{
+			super.init( config );
+			initCommands();
+			config.getServletContext().setAttribute( "flatData", new LexFlatDataRepository() );
+			config.getServletContext().setAttribute( "sources", new LexSourceRepository() );
+			String delay = config.getInitParameter( "globalDataRefreshDelay" );
+			long refreshDelay = Long.parseLong( delay ) * 1000;
+			String recent = config.getInitParameter( "recentItems" );
+			int recentItems = Integer.parseInt( recent );
+			Global global = new Global( recentItems, refreshDelay );
+			config.getServletContext().setAttribute( LexConstants.GLOBAL_CONTEXT_ATTR, global );
+			LexLogger.debugComponent( global );
+			System.setProperty( "java.awt.headless", "true" );
+		}
+		catch ( Exception e )
+		{
+			throw new ServletException( e );
+		}
 	}
 
 
@@ -216,6 +224,7 @@ public class LexActionServlet extends HttpServlet
 
 		commands.put( "setMetaPrefs", new PreferencesCommand( "menu.jsp" ) );
 		commands.put( "setMetaDefaults", new PreferencesCommand( "menu.jsp" ) );
+		commands.put( "refreshSources", new RefreshSourcesCommand( "test.jsp" ) );
 
 		commands.put( "abort", new AbortCommand( "menu.jsp" ) );
 

src/java/org/thdl/lex/LexComponentRepository.java

@@ -32,7 +32,7 @@ public class LexComponentRepository
 	public final static String ANYWHERE = "anywhere";
 	private static long start;
 
-	private static long lastUpdate;
+	private static long lastUpdate = now();
 
 
 	/**
@@ -399,11 +399,13 @@ public class LexComponentRepository
 			query = getSession().createQuery( queryString );
 			results = query.list();
 			endTransaction( false );
+			getSession().clear();
 		}
 		catch ( HibernateException he )
 		{
 			throw new LexRepositoryException( he );
 		}
+
 		return results;
 	}
 
@@ -445,6 +447,7 @@ public class LexComponentRepository
 			beginTransaction();
 			getSession().update( component );
 			endTransaction( true );
+			setLastUpdate( now() );
 		}
 		catch ( HibernateException he )
 		{
@@ -467,6 +470,7 @@ public class LexComponentRepository
 			beginTransaction();
 			getSession().delete( component );
 			endTransaction( true );
+			setLastUpdate( now() );
 		}
 		catch ( HibernateException he )
 		{

src/java/org/thdl/lex/LexSourceRepository.java

@@ -0,0 +1,185 @@
+package org.thdl.lex;
+
+import org.thdl.users.ThdlUserRepository;
+
+import java.util.*;
+import java.io.*;
+
+import org.dlese.dpc.oai.harvester.*;
+import org.dlese.dpc.datamgr.*;
+import org.dlese.dpc.index.*;
+
+import org.dlese.dpc.oai.harvester.structs.ScheduledHarvest;
+
+import org.openarchives.oai.x20.oaiDc.DcDocument;
+import org.openarchives.oai.x20.oaiDc.OaiDcType;
+
+
+/**
+ *  Description of the Class
+ *
+ * @author     travis
+ * @created    December 15, 2003
+ */
+public class LexSourceRepository
+{
+//attributes
+	private static LexSourceRepository instance;
+
+	private ScheduledHarvestManager harvestManager;
+
+	private ScheduledHarvest harvest;
+
+
+	/**
+	 *  Sets the harvest attribute of the LexSourceRepository object
+	 *
+	 * @param  harvest  The new harvest value
+	 */
+	public void setHarvest( ScheduledHarvest harvest )
+	{
+		this.harvest = harvest;
+	}
+
+
+	/**
+	 *  Gets the harvest attribute of the LexSourceRepository object
+	 *
+	 * @return    The harvest value
+	 */
+	public ScheduledHarvest getHarvest()
+	{
+		return harvest;
+	}
+
+
+
+	/**
+	 *  Sets the harvestManager attribute of the LexSourceRepository object
+	 *
+	 * @param  harvestManager  The new harvestManager value
+	 */
+	public void setHarvestManager( ScheduledHarvestManager harvestManager )
+	{
+		this.harvestManager = harvestManager;
+	}
+
+
+	/**
+	 *  Gets the harvestManager attribute of the LexSourceRepository object
+	 *
+	 * @return    The harvestManager value
+	 */
+	public ScheduledHarvestManager getHarvestManager()
+	{
+		return harvestManager;
+	}
+
+
+//accessors
+	/**
+	 *  Gets the instance attribute of the LexSourceRepository class
+	 *
+	 * @return                The instance value
+	 * @exception  Exception  Description of the Exception
+	 */
+	public static LexSourceRepository getInstance() throws Exception
+	{
+		if ( null == instance )
+		{
+			instance = new LexSourceRepository();
+		}
+		return instance;
+	}
+
+
+	/**
+	 *  Description of the Method
+	 *
+	 * @exception  LexRepositoryException  Description of the Exception
+	 */
+	public void refreshSources() throws LexRepositoryException
+	{
+		/*
+		    try
+		    {
+		    getHarvest().
+		    }
+		    catch ( Hexception h )
+		    {
+		    throw new LexRepositoryException( h );
+		    }
+		    catch ( OAIErrorException h )
+		    {
+		    throw new LexRepositoryException( h );
+		    }
+		  */
+	}
+
+
+	/**
+	 *  Description of the Method
+	 *
+	 * @return                             Description of the Return Value
+	 * @exception  LexRepositoryException  Description of the Exception
+	 */
+	public String[] xmlTesting() throws LexRepositoryException
+	{
+		String[] sa = null;
+		try
+		{
+			File file = new File( "/Users/travis/webapps/lex/dlese-oai/datastore.lib.virginia.edu/tibet/spt/SPT--OAI.php/oai_dc", "oai%3Alib.virginia.edu%3Athdl-267.xml" );
+			DcDocument dcd = DcDocument.Factory.parse( file );
+			OaiDcType oaiDc = dcd.getDc();
+			sa = new String[oaiDc.getTitleArray().length];
+			for ( int i = 0; i < sa.length; i++ )
+			{
+				sa[i] = oaiDc.getTitleArray()[i].toString();
+			}
+		}
+		catch ( Exception e )
+		{
+			throw new LexRepositoryException( e );
+		}
+		return sa;
+	}
+//constructors
+
+	/**
+	 *Constructor for the LexSourceRepository object
+	 *
+	 * @exception  Exception  Description of the Exception
+	 */
+	public LexSourceRepository() throws Exception
+	{
+		SimpleDataStore dataStore = new SimpleDataStore( "/Users/travis/webapps/lex/dlese-oai/scheduled-harvester", true );
+		File initialHarvestDir = new File( "/Users/travis/webapps/lex/dlese-oai/datastore.lib.virginia.edu" );
+		SimpleLuceneIndex harvestLogIndex = new SimpleLuceneIndex( "/Users/travis/webapps/lex/dlese-oai/lucene-index" );
+		ScheduledHarvestManager manager = new ScheduledHarvestManager( dataStore, initialHarvestDir, harvestLogIndex );
+		setHarvestManager( manager );
+
+		String repositoryName = "SPT";
+		String setSpec = "";
+		String baseURL = "http://datastore.lib.virginia.edu/tibet/spt/SPT--OAI.php";
+		String metadataPrefix = "oai_dc";
+		int seconds = 1000 * 1;
+		String harvestingInterval = Integer.toString( seconds );
+		String intervalGranularity = "YYYY -MM-DDThh:mm:ssZ";
+		String enabledDisabled = "enabled";
+		ScheduledHarvest harvest = new ScheduledHarvest( repositoryName, setSpec, baseURL, metadataPrefix, harvestingInterval, intervalGranularity, enabledDisabled );
+		setHarvest( harvest );
+
+		getHarvestManager().addScheduledHarvest( getHarvest() );
+	}
+//main
+
+	/**
+	 *  The main program for the LexSourceRepository class
+	 *
+	 * @param  args  The command line arguments
+	 */
+	public static void main( String[] args )
+	{
+	}
+}
+

src/java/org/thdl/lex/commands/RefreshSourcesCommand.java

@@ -0,0 +1,52 @@
+package org.thdl.lex.commands;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.thdl.lex.*;
+import org.thdl.lex.component.*;
+
+
+/**
+ *  Description of the Class
+ *
+ * @author     travis
+ * @created    October 6, 2003
+ */
+public class RefreshSourcesCommand extends LexCommand implements Command
+{
+	/**
+	 *  Description of the Method
+	 *
+	 * @param  req                   Description of the Parameter
+	 * @param  component             Description of the Parameter
+	 * @return                       Description of the Return Value
+	 * @exception  CommandException  Description of the Exception
+	 */
+	public String execute( HttpServletRequest req, ILexComponent component ) throws CommandException
+	{
+		try
+		{
+			LexSourceRepository lcr = LexSourceRepository.getInstance();
+			req.setAttribute( "testArray", lcr.xmlTesting() );
+
+		}
+		catch ( Exception lre )
+		{
+			throw new CommandException( lre );
+		}
+		return getNext();
+	}
+
+
+
+	/**
+	 *Constructor for the PreferencesCommand object
+	 *
+	 * @param  next  Description of the Parameter
+	 */
+	public RefreshSourcesCommand( String next )
+	{
+		super( next );
+	}
+}
+

src/jsp/jsp/menu.jsp

@@ -67,6 +67,14 @@
 	<input type="submit" value="Preferences" /> <br />
 	</p>
 	</form>
+
+	<form id="oai" action="/lex/action" method="get" >
+	<p>
+	<input type="hidden" name="cmd" value="refreshSources" />
+	<input type="submit" value="Refresh Sources" /> <br />
+	</p>
+	</form>
+
 	</div>
 	
 	</c:if>

src/jsp/jsp/test.jsp

@@ -0,0 +1,39 @@
+<%@ page buffer="512kb" autoFlush="false" import="org.thdl.lex.*,org.thdl.lex.component.*" errorPage="/jsp/error.jsp" contentType="text/html; charset=UTF-8"%>
+<%@ taglib prefix="c" uri="http://java.sun.com/jstl/core" %>
+<%@ taglib prefix="x" uri="http://java.sun.com/jstl/xml" %> 
+<% request.setCharacterEncoding("UTF-8"); %>
+
+<jsp:include page="header.jsf" flush="false" />
+<!--menu.jsp-->
+<c:set var="editMode" value="${ false }" />
+<c:if test="${ ! sessionScope.visit.user.guest }">
+	<c:set var="editMode" value="${ true }" />
+</c:if>
+
+<div id="columnLeft">
+
+	<form id="oai" action="/lex/action" method="get" >
+	<p>
+	<input type="hidden" name="cmd" value="refreshSources" />
+	<input type="submit" value="Refresh Sources" /> <br />
+	</p>
+	</form>
+
+</div><!--END COLUMN LEFT-->
+	
+<div id="columnCenter">
+	<jsp:include page="navLinks.jsf" flush="false"/>
+	<h1>THDL Tibetan Collaborative Dictionaries: Test Page</h1>
+	<p><c:out value="${ applicationScope.sources }"/></p>
+	<ol>
+	<c:forEach var="testString" items="${requestScope.testArray}">
+		<x:parse var="title" xml="${ testString }"/>
+		<li><x:out select="$title/xml-fragment"/></li>
+		
+	</c:forEach>
+	</ol>
+	
+</div><!--END COLUMN CENTER-->
+
+<jsp:include page="footer.jsf" flush="false" />
+