thdltools
/
Dictionary
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated Global Object's Recent Terms Refresh. Started to add an OAI Harvester for Literary Sources.

This commit is contained in:
travismccauley 2003-12-23 06:22:33 +00:00
parent c944cfc0d1
commit e62b68a996
13 changed files with 337 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
build.xml
View File

@ -107,6 +107,12 @@
<fileset dir="${basedir}/lib/jwebunit">
<include name="*.jar"/>
</fileset>
<fileset dir="${basedir}/lib/dlese-oai">
<include name="*.jar"/>
</fileset>
<fileset dir="${basedir}/lib/oai-xmlbeans">
<include name="*.jar"/>
</fileset>
</copy>
</target>

BIN
lib/dlese-oai/DLESETools.jar Normal file
View File

Binary file not shown.

BIN
lib/dlese-oai/lucene-1.3-rc1.jar Normal file
View File

Binary file not shown.

BIN
lib/oai-xmlbeans/oai-dc.jar Normal file
View File

Binary file not shown.

BIN
lib/oai-xmlbeans/xbean.jar Normal file
View File

Binary file not shown.

BIN
lib/oai-xmlbeans/xmlpublic.jar Normal file
View File

Binary file not shown.

126
src/java/org/thdl/lex/Global.java
View File

@ -17,8 +17,6 @@ public class Global
private static long lastRefresh;
private int entryCount;
private List recentTerms;
private final GlobalRefresher REFRESHER = new GlobalRefresher();
private final Thread REFRESHER_THREAD = new Thread( REFRESHER );
/**
@ -112,7 +110,7 @@ public class Global
*/
public void setRecentTerms( List recentTerms )
{
this.recentTerms = Collections.synchronizedList( recentTerms );
this.recentTerms = recentTerms;
setLastRefresh( System.currentTimeMillis() );
}
@ -136,9 +134,9 @@ public class Global
*/
public List getRecentTerms() throws LexRepositoryException
{
if ( null == recentTerms )
if ( null == recentTerms || requiresRefresh() )
{
doRefresh();
refresh();
}
return recentTerms;
}
@ -155,9 +153,12 @@ public class Global
long now = System.currentTimeMillis();
long lastUpdate = LexComponentRepository.getLastUpdate();
long sinceLastRefresh = now - getLastRefresh();
//LexLogger.debug( "Requires Refresh Logic: if " + sinceLastRefresh + " > " + getRefreshDelay() + " && " + lastUpdate + " > " + getLastRefresh() );
if ( sinceLastRefresh > getRefreshDelay() && lastUpdate > getLastRefresh() )
{
requiresRefresh = true;
LexLogger.debug( "Refresh Required at: " + System.currentTimeMillis() );
}
return requiresRefresh;
}
@ -165,36 +166,27 @@ public class Global
/**
* Description of the Method
*
* @exception LexRepositoryException Description of the Exception
*/
public void refresh() throws LexRepositoryException
public void refresh()
{
if ( requiresRefresh() )
try
{
doRefresh();
int limit = getRecentTermsCount();
LexLogger.debug( "GlobalRefresher is starting a refresh for the " + limit + " most recent terms." );
setRecentTerms( LexComponentRepository.getRecentTerms( limit ) );
LexLogger.debug( "GlobalRefresher is finished refreshing..." );
LexLogger.debug( "Here's the new recent terms list: " + getRecentTerms().toString() );
LexComponentRepository.cleanup();
LexLogger.info( "GlobalRefresher finished a refresh..." );
}
catch ( Exception e )
{
StringWriter writer = new StringWriter();
e.printStackTrace( new PrintWriter( writer ) );
String stackTrace = writer.getBuffer().toString();
LexLogger.error( "GlobalRefresher Thread caught an Exception: " + stackTrace );
}
}
/**
* Description of the Method
*
* @exception LexRepositoryException Description of the Exception
*/
public void doRefresh() throws LexRepositoryException
{
LexLogger.info( "Checking if GlobalRefresher is busy..." );
if ( REFRESHER.getFinished() )
{
LexLogger.info( "GlobalRefresher is not busy. Starting refresh..." );
REFRESHER.setFinished( false );
REFRESHER_THREAD.start();
}
else
{
LexLogger.info( "GlobalRefresher was busy. Refresh not started..." );
}
}
@ -216,79 +208,5 @@ public class Global
setRefreshDelay( refreshDelay );
}
/**
* Description of the Class
*
* @author travis
* @created October 21, 2003
*/
class GlobalRefresher implements Runnable
{
private boolean finished;
/**
* Sets the finished attribute of the GlobalRefresher object
*
* @param finished The new finished value
*/
private void setFinished( boolean finished )
{
this.finished = finished;
}
/**
* Gets the finished attribute of the GlobalRefresher object
*
* @return The finished value
*/
private boolean getFinished()
{
return finished;
}
/**
* Main processing method for the GlobalRefresher object
*/
public void run()
{
int limit = getRecentTermsCount();
try
{
LexLogger.info( "GlobalRefresher is starting a refresh..." );
setRecentTerms( LexComponentRepository.getRecentTerms( limit ) );
/*
ILexComponent ilc = null;
for ( Iterator it = getRecentTerms().iterator(); it.hasNext(); ilc = (ILexComponent) it.next() )
{
ilc.getMeta();
}
*/
LexComponentRepository.cleanup();
LexLogger.info( "GlobalRefresher finished a refresh..." );
setFinished( true );
}
catch ( Exception e )
{
StringWriter writer = new StringWriter();
e.printStackTrace( new PrintWriter( writer ) );
String stackTrace = writer.getBuffer().toString();
LexLogger.error( "GlobalRefresher Thread caught an Exception: " + stackTrace );
}
setFinished( true );
}
/**
*Constructor for the GlobalRefresher object
*/
GlobalRefresher()
{
setFinished( true );
}
}
}

31
src/java/org/thdl/lex/LexActionServlet.java
View File

@ -65,17 +65,25 @@ public class LexActionServlet extends HttpServlet
*/
public void init( ServletConfig config ) throws ServletException
{
super.init( config );
initCommands();
config.getServletContext().setAttribute( "flatData", new LexFlatDataRepository() );
String delay = config.getInitParameter( "globalDataRefreshDelay" );
long refreshDelay = Long.parseLong( delay ) * 1000 * 60;
String recent = config.getInitParameter( "recentItems" );
int recentItems = Integer.parseInt( recent );
Global global = new Global( recentItems, refreshDelay );
config.getServletContext().setAttribute( LexConstants.GLOBAL_CONTEXT_ATTR, global );
LexLogger.debugComponent( global );
System.setProperty( "java.awt.headless", "true" );
try
{
super.init( config );
initCommands();
config.getServletContext().setAttribute( "flatData", new LexFlatDataRepository() );
config.getServletContext().setAttribute( "sources", new LexSourceRepository() );
String delay = config.getInitParameter( "globalDataRefreshDelay" );
long refreshDelay = Long.parseLong( delay ) * 1000;
String recent = config.getInitParameter( "recentItems" );
int recentItems = Integer.parseInt( recent );
Global global = new Global( recentItems, refreshDelay );
config.getServletContext().setAttribute( LexConstants.GLOBAL_CONTEXT_ATTR, global );
LexLogger.debugComponent( global );
System.setProperty( "java.awt.headless", "true" );
}
catch ( Exception e )
{
throw new ServletException( e );
}
}
@ -216,6 +224,7 @@ public class LexActionServlet extends HttpServlet
commands.put( "setMetaPrefs", new PreferencesCommand( "menu.jsp" ) );
commands.put( "setMetaDefaults", new PreferencesCommand( "menu.jsp" ) );
commands.put( "refreshSources", new RefreshSourcesCommand( "test.jsp" ) );
commands.put( "abort", new AbortCommand( "menu.jsp" ) );

6
src/java/org/thdl/lex/LexComponentRepository.java
View File

@ -32,7 +32,7 @@ public class LexComponentRepository
public final static String ANYWHERE = "anywhere";
private static long start;
private static long lastUpdate;
private static long lastUpdate = now();
/**
@ -399,11 +399,13 @@ public class LexComponentRepository
query = getSession().createQuery( queryString );
results = query.list();
endTransaction( false );
getSession().clear();
}
catch ( HibernateException he )
{
throw new LexRepositoryException( he );
}
return results;
}
@ -445,6 +447,7 @@ public class LexComponentRepository
beginTransaction();
getSession().update( component );
endTransaction( true );
setLastUpdate( now() );
}
catch ( HibernateException he )
{
@ -467,6 +470,7 @@ public class LexComponentRepository
beginTransaction();
getSession().delete( component );
endTransaction( true );
setLastUpdate( now() );
}
catch ( HibernateException he )
{

185
src/java/org/thdl/lex/LexSourceRepository.java Normal file
View File

@ -0,0 +1,185 @@
package org.thdl.lex;
import org.thdl.users.ThdlUserRepository;
import java.util.*;
import java.io.*;
import org.dlese.dpc.oai.harvester.*;
import org.dlese.dpc.datamgr.*;
import org.dlese.dpc.index.*;
import org.dlese.dpc.oai.harvester.structs.ScheduledHarvest;
import org.openarchives.oai.x20.oaiDc.DcDocument;
import org.openarchives.oai.x20.oaiDc.OaiDcType;
/**
* Description of the Class
*
* @author travis
* @created December 15, 2003
*/
public class LexSourceRepository
{
//attributes
private static LexSourceRepository instance;
private ScheduledHarvestManager harvestManager;
private ScheduledHarvest harvest;
/**
* Sets the harvest attribute of the LexSourceRepository object
*
* @param harvest The new harvest value
*/
public void setHarvest( ScheduledHarvest harvest )
{
this.harvest = harvest;
}
/**
* Gets the harvest attribute of the LexSourceRepository object
*
* @return The harvest value
*/
public ScheduledHarvest getHarvest()
{
return harvest;
}
/**
* Sets the harvestManager attribute of the LexSourceRepository object
*
* @param harvestManager The new harvestManager value
*/
public void setHarvestManager( ScheduledHarvestManager harvestManager )
{
this.harvestManager = harvestManager;
}
/**
* Gets the harvestManager attribute of the LexSourceRepository object
*
* @return The harvestManager value
*/
public ScheduledHarvestManager getHarvestManager()
{
return harvestManager;
}
//accessors
/**
* Gets the instance attribute of the LexSourceRepository class
*
* @return The instance value
* @exception Exception Description of the Exception
*/
public static LexSourceRepository getInstance() throws Exception
{
if ( null == instance )
{
instance = new LexSourceRepository();
}
return instance;
}
/**
* Description of the Method
*
* @exception LexRepositoryException Description of the Exception
*/
public void refreshSources() throws LexRepositoryException
{
/*
try
{
getHarvest().
}
catch ( Hexception h )
{
throw new LexRepositoryException( h );
}
catch ( OAIErrorException h )
{
throw new LexRepositoryException( h );
}
*/
}
/**
* Description of the Method
*
* @return Description of the Return Value
* @exception LexRepositoryException Description of the Exception
*/
public String[] xmlTesting() throws LexRepositoryException
{
String[] sa = null;
try
{
File file = new File( "/Users/travis/webapps/lex/dlese-oai/datastore.lib.virginia.edu/tibet/spt/SPT--OAI.php/oai_dc", "oai%3Alib.virginia.edu%3Athdl-267.xml" );
DcDocument dcd = DcDocument.Factory.parse( file );
OaiDcType oaiDc = dcd.getDc();
sa = new String[oaiDc.getTitleArray().length];
for ( int i = 0; i < sa.length; i++ )
{
sa[i] = oaiDc.getTitleArray()[i].toString();
}
}
catch ( Exception e )
{
throw new LexRepositoryException( e );
}
return sa;
}
//constructors
/**
*Constructor for the LexSourceRepository object
*
* @exception Exception Description of the Exception
*/
public LexSourceRepository() throws Exception
{
SimpleDataStore dataStore = new SimpleDataStore( "/Users/travis/webapps/lex/dlese-oai/scheduled-harvester", true );
File initialHarvestDir = new File( "/Users/travis/webapps/lex/dlese-oai/datastore.lib.virginia.edu" );
SimpleLuceneIndex harvestLogIndex = new SimpleLuceneIndex( "/Users/travis/webapps/lex/dlese-oai/lucene-index" );
ScheduledHarvestManager manager = new ScheduledHarvestManager( dataStore, initialHarvestDir, harvestLogIndex );
setHarvestManager( manager );
String repositoryName = "SPT";
String setSpec = "";
String baseURL = "http://datastore.lib.virginia.edu/tibet/spt/SPT--OAI.php";
String metadataPrefix = "oai_dc";
int seconds = 1000 * 1;
String harvestingInterval = Integer.toString( seconds );
String intervalGranularity = "YYYY -MM-DDThh:mm:ssZ";
String enabledDisabled = "enabled";
ScheduledHarvest harvest = new ScheduledHarvest( repositoryName, setSpec, baseURL, metadataPrefix, harvestingInterval, intervalGranularity, enabledDisabled );
setHarvest( harvest );
getHarvestManager().addScheduledHarvest( getHarvest() );
}
//main
/**
* The main program for the LexSourceRepository class
*
* @param args The command line arguments
*/
public static void main( String[] args )
{
}
}

52
src/java/org/thdl/lex/commands/RefreshSourcesCommand.java Normal file
View File

@ -0,0 +1,52 @@
package org.thdl.lex.commands;
import javax.servlet.http.HttpServletRequest;
import org.thdl.lex.*;
import org.thdl.lex.component.*;
/**
* Description of the Class
*
* @author travis
* @created October 6, 2003
*/
public class RefreshSourcesCommand extends LexCommand implements Command
{
/**
* Description of the Method
*
* @param req Description of the Parameter
* @param component Description of the Parameter
* @return Description of the Return Value
* @exception CommandException Description of the Exception
*/
public String execute( HttpServletRequest req, ILexComponent component ) throws CommandException
{
try
{
LexSourceRepository lcr = LexSourceRepository.getInstance();
req.setAttribute( "testArray", lcr.xmlTesting() );
}
catch ( Exception lre )
{
throw new CommandException( lre );
}
return getNext();
}
/**
*Constructor for the PreferencesCommand object
*
* @param next Description of the Parameter
*/
public RefreshSourcesCommand( String next )
{
super( next );
}
}

8
src/jsp/jsp/menu.jsp
View File

@ -67,6 +67,14 @@
<input type="submit" value="Preferences" /> <br />
</p>
</form>
<form id="oai" action="/lex/action" method="get" >
<p>
<input type="hidden" name="cmd" value="refreshSources" />
<input type="submit" value="Refresh Sources" /> <br />
</p>
</form>
</div>
</c:if>

39
src/jsp/jsp/test.jsp Normal file
View File

@ -0,0 +1,39 @@
<%@ page buffer="512kb" autoFlush="false" import="org.thdl.lex.*,org.thdl.lex.component.*" errorPage="/jsp/error.jsp" contentType="text/html; charset=UTF-8"%>
<%@ taglib prefix="c" uri="http://java.sun.com/jstl/core" %>
<%@ taglib prefix="x" uri="http://java.sun.com/jstl/xml" %>
<% request.setCharacterEncoding("UTF-8"); %>
<jsp:include page="header.jsf" flush="false" />
<!--menu.jsp-->
<c:set var="editMode" value="${ false }" />
<c:if test="${ ! sessionScope.visit.user.guest }">
<c:set var="editMode" value="${ true }" />
</c:if>
<div id="columnLeft">
<form id="oai" action="/lex/action" method="get" >
<p>
<input type="hidden" name="cmd" value="refreshSources" />
<input type="submit" value="Refresh Sources" /> <br />
</p>
</form>
</div><!--END COLUMN LEFT-->
<div id="columnCenter">
<jsp:include page="navLinks.jsf" flush="false"/>
<h1>THDL Tibetan Collaborative Dictionaries: Test Page</h1>
<p><c:out value="${ applicationScope.sources }"/></p>
<ol>
<c:forEach var="testString" items="${requestScope.testArray}">
<x:parse var="title" xml="${ testString }"/>
<li><x:out select="$title/xml-fragment"/></li>
</c:forEach>
</ol>
</div><!--END COLUMN CENTER-->
<jsp:include page="footer.jsf" flush="false" />