/*
Copyright (C) 2000-2010  Ministere de la culture et de la communication (France), AJLSM
See LICENCE file
*/
package fr.gouv.culture.sdx.oai;

import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.Properties;
import java.util.Set;

import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.configuration.DefaultConfiguration;
import org.apache.avalon.framework.context.Context;
import org.apache.avalon.framework.context.ContextException;
import org.apache.avalon.framework.service.ServiceException;
import org.apache.cocoon.Constants;
import org.apache.cocoon.ProcessingException;
import org.apache.cocoon.serialization.XMLSerializer;
import org.apache.cocoon.xml.IncludeXMLConsumer;
import org.apache.cocoon.xml.XMLPipe;
import org.apache.commons.io.FileUtils;
import org.apache.excalibur.xml.sax.SAXParser;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

import fr.gouv.culture.oai.AbstractOAIHarvester;
import fr.gouv.culture.oai.OAIObject;
import fr.gouv.culture.oai.OAIRequest;
import fr.gouv.culture.oai.OAIRequestImpl;
import fr.gouv.culture.oai.util.OAIUtilities;
import fr.gouv.culture.sdx.document.IndexableDocument;
import fr.gouv.culture.sdx.document.OAIDocument;
import fr.gouv.culture.sdx.document.XMLDocument;
import fr.gouv.culture.sdx.documentbase.DocumentBase;
import fr.gouv.culture.sdx.documentbase.IDGenerator;
import fr.gouv.culture.sdx.documentbase.IndexParameters;
import fr.gouv.culture.sdx.exception.SDXException;
import fr.gouv.culture.sdx.exception.SDXExceptionCode;
import fr.gouv.culture.sdx.framework.Framework;
import fr.gouv.culture.sdx.pipeline.GenericPipeline;
import fr.gouv.culture.sdx.pipeline.Pipeline;
import fr.gouv.culture.sdx.repository.Repository;
import fr.gouv.culture.sdx.search.lucene.query.ComplexQuery;
import fr.gouv.culture.sdx.search.lucene.query.DateIntervalQuery;
import fr.gouv.culture.sdx.search.lucene.query.FieldQuery;
import fr.gouv.culture.sdx.search.lucene.query.SearchLocations;
import fr.gouv.culture.sdx.utils.Utilities;
import fr.gouv.culture.sdx.utils.configuration.ConfigurationUtils;
import fr.gouv.culture.sdx.utils.constants.ContextKeys;
import fr.gouv.culture.sdx.utils.database.Database;
import fr.gouv.culture.sdx.utils.database.DatabaseBacked;
import fr.gouv.culture.sdx.utils.database.DatabaseEntity;
import fr.gouv.culture.sdx.utils.database.Property;
import fr.gouv.culture.sdx.utils.save.SaveParameters;
import fr.gouv.culture.sdx.utils.save.Saveable;
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.SimpleTimeScheduler;
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeScheduler;
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTrigger;
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTriggerFactory;

public abstract class AbstractDocumentBaseOAIHarvester extends AbstractOAIHarvester implements DocumentBaseOAIHarvester {

    /**The underlying document base*/
    protected DocumentBase docbase = null;
    /**Id of the underlying document base*/
    protected String docbaseId = "";
    /**Pre-indexation pipeline*/
    protected Pipeline pipe = null;
    /**Underlying database to store any info*/
    protected Database _database = null;
    /**Requests in application.xconf*/
    protected Hashtable storedRequests = null;
    /**References to the underlying documentbase's/application's repositories*/
    protected Hashtable storeRepositoriesRefs = null;
    /**Time scheduler for stored requests*/
    protected TimeScheduler scheduler = null;
    /**IDGenerator for this object*/
    protected IDGenerator harvesterIdGen = null;

    //variables for sax stream handling
    protected String TEMPFILE_SUFFIX = ".sdx";
    protected File tempDir = null; // principal temporary directory
    protected File tempDirBatch = null; // temp directory for a batch, relative to tempDir 
    protected File harvestDoc = null;
    protected FileOutputStream fileOs = null;
    protected XMLDocument urlResource = null;
    protected ArrayList deletedDocs = null;
    protected int noHarvestedDocs = 0;
    protected int noDocsDeleted = 0;
    protected Set m_docsaddedIds = null;
    protected Set m_docsToDeleteIds = null;
    protected Set m_docsdeletedids = null;

    protected boolean keepDeletedRecords = false;//Defaulted, we will delete "deletedRecords" keeping our harvester in sync with source repository
    protected int noRecordsPerBatch = OAIObject.NUMBER_RECORDS_PER_RESPONSE;
    
    /** Force harvester to keep harvested records (default: false) 
     * <p>Force harvester to keep harvested records (XML files) in file system server.
     * <br/>
     * Default is false.
     * <br/>
     * This cas be change in document base configuration file:
     * &lt;oai-harvester keepHarvestedRecords="{true|false}" [...]&gt;</p>
     */
    protected boolean keepHarvestedRecords = false;
    
    /** Directory to store harvested documents
     * <p>Temporary path of the directory where the harvested documents will be stored.
     * <br/>Default is the servlet context temp dir (eg, $TOMCAT/work/...). If the
     * directory is not writable, the harvester will use the temporary directory
     * of the JVM (ie, <code>java.io.tmpdir</code> system property).
     * <br/>
     * This can be change in document base configuration file:
     * <pre>
     * &lt;oai-harvester tempDirPath="{/path/to/directory}" [...]&gt;
     * </pre>
     * To resolve the path, harvester uses the {@link Utilities#resolveFile(org.apache.avalon.framework.logger.Logger, String, Context, String, boolean)}.
     * <br/>
     * By default, this directory is deleted after the harvest. This can be change
     * with {@linkplain #keepHarvestedRecords} configuration attribute.
     * </p>
     * @see Utilities#resolveFile(org.apache.avalon.framework.logger.Logger, String, Context, String, boolean)
     */
    protected String tempDirPath = "";

    /** XML Transformer factory classe name.
     *<p>Default: Xalan, "org.apache.xalan.processor.TransformerFactoryImpl".
     *This cas be change in configuration file:
     *&lt;oai-harvester transformer-factory="{classe name}" [...]&gt;</p>
     */
    protected String transformerFactory = "";
    protected String defaultTransformerFactory = "com.icl.saxon.TransformerFactoryImpl"; // Saxon 6.x
    //protected String defaultTransformerFactory = "net.sf.saxon.TransformerFactoryImpl"; // Saxon 8.x and higher
    //protected String defaultTransformerFactory = "org.apache.xalan.processor.TransformerFactoryImpl"; // Xalan
    /**XML Transformer indent option.
     *<p>Default:no. This can be change in configuration file:
     *&lt;oai-harvester transformer-indent="yes|no" [...]&gt;</p>
     */
    protected String transformerIndent = "";
    protected String defaultTransformerIndent = "no";
    
    /**
     * Indexation at the end of harvesting option
     * <p>Default: true. This cas be change in configuraiton file:
     * &lt;oai-harvester index-at-index-end="yes|no" [...]&gt;</p>
     */
    protected boolean indexAtHarvestEnd = true;
    /**
     * Force indexation on harvest error option
     * <p>Default: false. This cas be change in configuraiton file:
     * &lt;oai-harvester force-index-on-harvest-error="yes|no" [...]&gt;</p>
     */
    protected boolean forceIndexOnHarvestError = false;

    // strings for XML transformer factory classe name
    protected static final String TRANSFORMER_FACTORY = "transformer-factory";
    // string for XML transformer indent option
    protected static final String TRANSFORMER_INDENT = "transformer-indent";
    // String for indexAtHarvestEnd option
    protected static final String INDEXATHARVESTEND = "index-at-harvest-end";
    // String for forceIndexOnHarvestError option
    protected static final String FORCEINDEXONHARVESTERROR = "force-index-on-harvest-error";
    //strings for internal database properties and sax output
    protected static final String OAI_HARVEST_ID = "oaiHarvestId";
    //internal database property for failed harvests caused by internal errors( errors in this implementation and not OAI repository errors)
    protected static final String OAI_FAILED_HARVEST = "oaiFailedHarvest";
    protected static final String OAI_HARVESTER_LAST_UPDATED = "oaiHarvesterLastUpdated";
    protected static final String OAI_HARVESTER_RESUMPTION_TOKEN = "oaiHarvesterResumptionToken";
    /*Field names for common request parameters*/
    protected static final String OAI_VERB = "oaiVerb";
    protected static final String OAI_IDENTIFIER = "oaiIdentifier";
    protected static final String OAI_METADATA_PREFIX = "oaiMetadataPrefix";
    protected static final String OAI_FROM = "oaiFrom";
    protected static final String OAI_UNTIL = "oaiUntil";
    protected static final String OAI_SET = "oaiSet";
    protected static final String NO_DOCS_DELETED = "noDocDeleted";
    protected static final String NO_DOCS_HARVESTED = "noDocHarvested";
    protected static final String ERROR_CODE = "errorCode";

    /**List OAI files with OAI properties*/
    protected Hashtable filesProperties = null;

    protected XMLSerializer cBytes = null;
    protected XMLPipe oaiStripper = null;

    /**Basic constructor*/
    public AbstractDocumentBaseOAIHarvester(DocumentBase base) {
        this.docbase = base;
        if (this.docbase != null)
            this.docbaseId = this.docbase.getId();
    }


    /** OAI harvester configuration
     * <p>Configures the OAI harvester reading <code>application.xconf</code> file
     * wich may contains a section such as:
     * <pre>
     * &lt;sdx:documentBase [...]>
     *   &lt;sdx:oai-harvester
     *           adminEmail="{some.body@some.where}"
     *           keepDeletedRecords="{true|false}"
     *           noRecordsPerBatch="{number}"
     *           transformer-factory="{Transformer factory classe name}"
     *           transformer-indent="{yes|no}"
     *           keepHarvestedRecords="{true|false}"
     *           tempDirPath="{directory path}">
     *     &lt;sdx:oai-data-providers>
     *       &lt;sdx:oai-repository [...]>[...]&lt;/sdx:oai-repository>
     *       [...]
     *     &lt;/sdx:oai-data-providers>
     *   &lt;/sdx:oai-harvester>
     * &lt;/sdx:documentBase>
     * </pre>
     * </p>
     * @param Configuration
     * @see #keepDeletedRecords
     * @see #noRecordsPerBatch
     * @see #transformerFactory
     * @see #transformerIndent
     * @see #keepHarvestedRecords
     * @see #tempDirPath
     * */
    public void configure(Configuration configuration) throws ConfigurationException {

    	String[] args = {this.docbaseId};
    	OAIUtilities.logInfo(logger, SDXExceptionCode.OAI_HARVESTER_CONFIGURING,args);
    	

    	super.userAgent = configuration.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.USER_AGENT, "SDX OAI Harvester");
        this.keepDeletedRecords = configuration.getAttributeAsBoolean(DocumentBaseOAIHarvester.ConfigurationNode.KEEP_DELETED_RECORDS, this.keepDeletedRecords);
        this.noRecordsPerBatch = configuration.getAttributeAsInteger(DocumentBaseOAIHarvester.ConfigurationNode.NO_RECORDS_PER_BATCH, this.noRecordsPerBatch);
        this.transformerFactory = configuration.getAttribute(TRANSFORMER_FACTORY, this.defaultTransformerFactory);
        this.transformerIndent = configuration.getAttribute(TRANSFORMER_INDENT, this.defaultTransformerIndent);
        this.indexAtHarvestEnd = configuration.getAttributeAsBoolean(INDEXATHARVESTEND, this.indexAtHarvestEnd);
        this.forceIndexOnHarvestError = configuration.getAttributeAsBoolean(FORCEINDEXONHARVESTERROR, this.forceIndexOnHarvestError);
        configureTempDir(configuration);
        configureAdminEmails(configuration);
        this.configureDataProviders(configuration);
        this.configurePipeline(configuration);
        configureDatabase(configuration);
        configureHarvestIDGenerator(configuration);
        
        String[] args2 = {this.docbaseId,this.getHarvesterId()};
        OAIUtilities.logInfo(logger, SDXExceptionCode.OAI_HARVESTER_CONFIGURED,args2);
    }

    /** Configures the temporary directory
     *  <p>Configures the temporary directory where harvested documents will be 
     *  stored in sub-directories. There will be one sub-directory per batch of
     *  the harvest. This directory will be deleted after harvest. This can be 
     *  change with <code>keepHarvestedRecords</code> configuration attribute.
     *  @see #keepHarvestedRecords
     *  @see #tempDirPath
     *  @param Configuration
     *  @throws ConfigurationException
     *  */
    protected void configureTempDir(Configuration conf)
    throws ConfigurationException
    {
    	this.keepHarvestedRecords = conf.getAttributeAsBoolean("keepHarvestedRecords", false);
        this.tempDirPath = conf.getAttribute("tempDirPath", null);
        boolean m_default = true;
        
        // May be we have a tempDirPath configuration attribute
        if( Utilities.checkString(this.tempDirPath) ) 
        {
	        try {
	        	tempDir = Utilities.resolveFile(logger, conf.getLocation(), this.getContext(), this.tempDirPath, true);
	        	m_default = false;
	        } catch(Exception e) {
	        	// TODO (MP) : SDXException code
	        	OAIUtilities.logError(logger, "Enable to initialize the temporary directory: "+this.tempDirPath+" to store harvested records. Uses default instead.", e);
	        	m_default = true;
	        }
        }
        
        // may be not !
        if ( m_default || tempDir == null ) {
        	try {
            	tempDir = (File) getContext().get(Constants.CONTEXT_UPLOAD_DIR);
            } catch (ContextException ce) {
            	// TODO (MP) : SDXException code
            	throw new ConfigurationException("Enable to initialize the OAI harvester default temporary directory: "+tempDir.getAbsolutePath(), ce);
            }
        }
        
        /* Building the directory name for this harvester:
         * {application name}_oaiHarvest/{document base id}/{time in millisecondes} (eg, sdxtest_oaiHarvests/sdxworld/1205318048800)
         */ 
        String childDir = Utilities.getStringFromContext(ContextKeys.SDX.Application.DIRECTORY_NAME, getContext()) 
        					+ "_oaiHarvests"  + File.separator 
        					+ this.docbaseId + File.separator
        					+ Utilities.encodeURL(Long.toString(new Date().getTime()), null);
        try {
	        if ( tempDir.canWrite() ) {
	            tempDir = new File(tempDir, childDir);
	            OAIUtilities.logDebug(logger, "Initialize temporary directory to store harvested records: " + tempDir.getAbsolutePath());
	            tempDir = Utilities.checkDirectory(tempDir.getAbsolutePath(), logger);
	        }
	        else {
	        	logger.error("Enable to create temporary directory to store harvested records: " + tempDir.getAbsolutePath());
	            tempDir = new File(Utilities.getSystemTempDir(), childDir);
	            OAIUtilities.logDebug(logger, "Initialize temporary directory to store harvested records: " + tempDir.getAbsolutePath());
	        }
        } catch (SDXException sdxe) {
        	// TODO (MP): SDXException code
        	throw new ConfigurationException("Enable to initialize the OAI harvester default temporary directory: "+tempDir.getAbsolutePath(), sdxe);
        }

    }

    /**Configures the internal database*/
    protected void configureDatabase(Configuration configuration) throws ConfigurationException {
        DatabaseBacked internalDb = new DatabaseBacked();
        try {
            internalDb.enableLogging(this.logger);
            internalDb.contextualize(Utilities.createNewReadOnlyContext(getContext()));
            internalDb.service(this.manager);
            internalDb.setId(getHarvesterId());
            internalDb.configure(configuration);
            internalDb.init();
            this._database = internalDb.getDatabase();
        } catch (SDXException e) {
            throw new ConfigurationException(e.getMessage(), e);
        } catch (ServiceException e) {
            throw new ConfigurationException(e.getMessage(), e);
        } catch (ContextException e) {
            throw new ConfigurationException(e.getMessage(), e);
        }

    }

    /**Configures the id generator for harvests*/
    protected void configureHarvestIDGenerator(Configuration configuration) throws ConfigurationException {
        this.harvesterIdGen = ConfigurationUtils.configureIDGenerator(this.logger, configuration);
        this.harvesterIdGen.setDatabase(this._database);
    }


    /**Returns an id for this harvester based upon the underlying document base id*/
    protected String getHarvesterId() {
        String hid = Framework.SDXNamespacePrefix + "_"
        				+ OAIObject.Node.Prefix.OAI + "_"
        				+ "harvester" + "_"
        				+ this.docbaseId;
        return hid;
    }

    /**Configures a list of admin emails
     * can be sub-elements, a single attribute,
     * or both
     *
     * @param configuration
     * @throws ConfigurationException
     */
    protected void configureAdminEmails(Configuration configuration) throws ConfigurationException {
        //configure the admin email
        ArrayList locAdminEmailsList = new ArrayList();
        String firstAdminEmail = configuration.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL, null);
        Configuration[] locAdminEmails = configuration.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL);
        if (Utilities.checkString(firstAdminEmail))
            locAdminEmailsList.add(firstAdminEmail);
        for (int i = 0; i < locAdminEmails.length; i++) {
            Configuration locAdminEmail = locAdminEmails[i];
            if (locAdminEmail != null) {
                String value = locAdminEmail.getValue();
                if (Utilities.checkString(value))
                    locAdminEmailsList.add(value);
            }
        }
        if (locAdminEmailsList.size() <= 0)//no admin email throw an error TODO:make this exception better
            ConfigurationUtils.checkConfAttributeValue(DocumentBaseOAIHarvester.ConfigurationNode.ADMIN_EMAIL, null, configuration.getLocation());

        super.adminEmails = (String[]) locAdminEmailsList.toArray(new String[0]);
        //releasing resources
        locAdminEmailsList.clear();
        locAdminEmailsList = null;

    }

    /**Configures data providers info that can be reused
     * and from which requests can be automatically executed
     *
     * @param configuration
     * @throws ConfigurationException
     * @see #storedRequests
     */
    protected void configureDataProviders(Configuration configuration) throws ConfigurationException {
        if (configuration != null) {
            Configuration dataProvidersConf = configuration.getChild(DocumentBaseOAIHarvester.ConfigurationNode.OAI_DATA_PROVIDERS, false);
            if (dataProvidersConf != null) {
                Configuration[] repoRequestConfs = dataProvidersConf.getChildren(DocumentBase.ConfigurationNode.OAI_REPOSITORY);
                if (repoRequestConfs != null) {
                    for (int x = 0; x < repoRequestConfs.length; x++) {
                        Configuration repoRequestConf = repoRequestConfs[x];
                        if (repoRequestConf != null) {
                            if (storedRequests == null) storedRequests = new Hashtable();
                            String repoUrl = repoRequestConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.URL);//On pourrait controler l'URL en faisant un verb Identify -mp
                            String repoGranularity = repoRequestConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.GRANULARITY, OAIObject.Node.Value.STRING_GRANULARITY_SECOND);//Configure repo granularity. Default is seconds "yyyy-MM-dd'T'HH:mm:ss'Z'"
                            checkGranularity(repoGranularity);
                            configureStoreRepositories(repoUrl, repoRequestConf);
                            Configuration updateConf = repoRequestConf.getChild(DocumentBaseOAIHarvester.ConfigurationNode.UPDATE, false);
                            Configuration[] verbConfs = repoRequestConf.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.OAI_VERB);
                            if (verbConfs != null) {
                                for (int y = 0; y < verbConfs.length; y++) {
                                    try {
                                        OAIRequest request = null;
                                        Configuration verbConf = verbConfs[y];
                                        if (verbConf != null) {
                                            String verb = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.NAME);
                                            String mdPrefix = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.METADATA_PREFIX);
                                            ConfigurationUtils.checkConfAttributeValue(DocumentBaseOAIHarvester.ConfigurationNode.METADATA_PREFIX, mdPrefix, verbConf.getLocation());
                                            String verbId = verbConf.getAttribute("id");
                                            ConfigurationUtils.checkConfAttributeValue("id", verbId, verbConf.getLocation());
                                            if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_GET_RECORD)) {
                                                verb = OAIRequest.VERB_STRING_GET_RECORD;
                                                Configuration[] idsConf = verbConf.getChildren(DocumentBaseOAIHarvester.ConfigurationNode.OAI_IDENTIFIER);
                                                if (idsConf != null) {
                                                    for (int z = 0; z < idsConf.length; z++) {
                                                        Configuration idConf = idsConf[z];
                                                        if (idConf != null) {
                                                            String id = idConf.getValue();
                                                            request = new OAIRequestImpl();
                                                            request.enableLogging(this.logger);
                                                            request.setRepositoryURL(repoUrl);
                                                            request.setGranularity(repoGranularity);
                                                            request.setVerbString(verb);
                                                            request.setMetadataPrefix(mdPrefix);
                                                            request.setVerbId(verbId);
                                                            request.setIdentifier(id);
                                                            Utilities.isObjectUnique(storedRequests, request.getRequestURL(), request);
                                                            storedRequests.put(request.getRequestURL(), request);
                                                            configureUpdateTriggers(request.getRequestURL(), updateConf);
                                                        }

                                                    }
                                                }

                                            } else if (verb.equalsIgnoreCase(OAIRequest.VERB_STRING_LIST_RECORDS)) {
                                                verb = OAIRequest.VERB_STRING_LIST_RECORDS;
                                                String from = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.FROM, null);
                                                String until = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.UNTIL, null);
                                                String set = verbConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.SET, null);
                                                boolean useLastHarvestDate = verbConf.getAttributeAsBoolean(DocumentBaseOAIHarvester.ConfigurationNode.ATTRIBUTE_USE_LAST_HARVEST_DATE, /*true by default*/true);
                                                request = new OAIRequestImpl();
                                                request.enableLogging(this.logger);
                                                request.setRepositoryURL(repoUrl);
                                                request.setGranularity(repoGranularity);
                                                request.setVerbString(verb);
                                                request.setMetadataPrefix(mdPrefix);
                                                request.setVerbId(verbId);
                                                request.setFrom(from);
                                                request.setUseLastHarvestDate(useLastHarvestDate);
                                                request.setUntil(until);
                                                request.setSetIdentifier(set);
                                                Utilities.isObjectUnique(storedRequests, request.getRequestURL(), request);
                                                this.storedRequests.put(request.getRequestURL(), request);
                                                configureUpdateTriggers(request.getRequestURL(), updateConf);
                                            } else//TODOException:
                                                throw new ConfigurationException("this verb action is not supported for harvesting : " + verb);
                                        }
                                    } catch (ConfigurationException e) {
                                    	OAIUtilities.logWarn(this.logger, e.getMessage(), e);//logging this so that all configs don't fail
                                    } catch (SDXException e) {
                                    	OAIUtilities.logWarn(this.logger, e.getMessage(), e);//logging this so that all configs don't fail
                                    }


                                }
                            }

                            // now we can configure indexation pipeline for these repository
                            this.configurePipeline(repoRequestConf);

                        }


                    }
                    if (this.scheduler != null) this.scheduler.start();
                }

            }

        }
    }

    /**Configures time triggers for
     * stored requests
     *
     * @param requestUrl    The request url
     * @param updateConf    The configuration for updates
     * @throws ConfigurationException
     * @see #scheduler
     * @see #storedRequests
     */
    protected void configureUpdateTriggers(String requestUrl, Configuration updateConf) throws ConfigurationException {
        if (Utilities.checkString(requestUrl) && updateConf != null) {
            TimeTrigger trigger = new TimeTriggerFactory().createTimeTrigger(updateConf);
            if (trigger != null) {
                if (this.scheduler == null) this.scheduler = new SimpleTimeScheduler();
                this.scheduler.addTrigger(requestUrl, trigger, this);
            }
        }
    }


    /**Configures the repositories
     * to which data will be stored
     * based upon their repository url
     *
     * @param repoUrl   The repository/data provider url
     * @param oaiRepoConf The configuration
     * @throws ConfigurationException
     */
    protected void configureStoreRepositories(String repoUrl, Configuration oaiRepoConf) throws ConfigurationException {

    	//Store repoURL
        if (Utilities.checkString(repoUrl)) {
            String ref = oaiRepoConf.getAttribute(DocumentBaseOAIHarvester.ConfigurationNode.SDX_REPOSITORY, null);
            /*check for the sdxrepository attribute, if it exists, get the repository object and add it to the local hashtable*/
            if (Utilities.checkString(ref)) {
                try {
                    Repository repo = null;
                    Context appRepos = (Context) getContext().get(ContextKeys.SDX.Application.REPOSITORIES);
                    if (appRepos != null)
                        repo = (Repository) appRepos.get(ref);
                    if (repo == null)
                        repo = this.docbase.getRepository(ref);
                    if (this.storeRepositoriesRefs == null) this.storeRepositoriesRefs = new Hashtable();
                    //populating the hashtable
                    Utilities.isObjectUnique(this.storeRepositoriesRefs, repoUrl, repo);
                    storeRepositoriesRefs.put(repoUrl, repo);
                } catch (SDXException e) {
                    String[] args = new String[1];
                    args[0] = ref;
                    SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_LOAD_REFERENCED_REPO, args, null);
                    throw new ConfigurationException(sdxE.getMessage(), sdxE);
                } catch (ContextException e) {
                    throw new ConfigurationException(e.getMessage(), e);
                }
            }
        }

    }

    /**
     * Check the granularity of an AOI provider : YYYY-MM-DDThh:mm:ssZ or YYYY-MM-DD
     * @param granularity
     * @return true or false
     * @throws ConfigurationException
     */
    public boolean checkGranularity(String granularity) throws ConfigurationException{
        if(Utilities.checkString(granularity)){
        	if(granularity.equals(OAIObject.Node.Value.STRING_GRANULARITY_SECOND)) return true;
        	else if(granularity.equals(OAIObject.Node.Value.STRING_GRANULARITY_DAY)) return true;
        	else{
        		String[] args = new String[2];
                args[0] = granularity;
                args[1] = repoUrl;
        		SDXException sdxE = new SDXException(logger, SDXExceptionCode.ERROR_UNKNOWN_OAI_GRANULARITY, args, null);
                throw new ConfigurationException(sdxE.getMessage(), sdxE);
        	}
        } else return false;
    }


    /**Configures the preIndexation pipeline
     *
     * @param configuration
     * @throws ConfigurationException
     * @see #pipe
     */
    protected void configurePipeline(Configuration configuration) throws ConfigurationException {
        //at this point, we should have a <sdx:pipeline> element
        Configuration pipeConf = configuration.getChild(Utilities.getElementName(Pipeline.CLASS_NAME_SUFFIX), false);
        //testing if we have something
        if (pipeConf != null) {
            //creating the pipeline and assigning the class field
            this.pipe = new GenericPipeline();
            //setting the super.getLog() for the pipeline object
            this.pipe.enableLogging(this.logger);
            //giving the object the cocoon service manager
            try {
                this.pipe.service(this.manager);
                //pass OAI base properties to the pipe
                this.pipe.contextualize(super.getContext());
                //configuring the pipeline object from 'application.xconf'
                this.pipe.configure(pipeConf);
            } catch (ServiceException e) {
                throw new ConfigurationException(e.getMessage(), e);
            } catch (ContextException e) {
                throw new ConfigurationException(e.getMessage(), e);
            }

        }
    }

    /**Creates a new temporary directory for
     * writing harvested records before the will
     * be indexed
     * @return File
     * @throws SDXException
     * @throws IOException
     */
    protected File getNewTempDirBatch() 
    	throws SDXException, IOException 
	{
    	if ( this.tempDir==null ) {
    		OAIUtilities.logWarn(this.logger, SDXException.OAI_HARVESTER_USES_DEFAULT_SYSTEM_TEMPDIR, null);
        }
        File ret = null;
        String childDir = "harvest-" + Utilities.encodeURL(Long.toString(new Date().getTime()), null);
        if ( Utilities.checkString(super.resumptionToken) ) {
            childDir += "-" + OAIObject.Node.Name.RESUMPTION_TOKEN + "-" + super.resumptionToken;
        }
        if (this.tempDir.canWrite()) {
            ret = new File(this.tempDir, childDir);
            String[] args = {ret.getCanonicalPath()};
            OAIUtilities.logDebug(logger, SDXExceptionCode.OAI_HARVESTER_INITIALIZE_TEMPDIR, args);
        }
        else {
        	String[] args = {this.tempDir.getCanonicalPath()};
            OAIUtilities.logDebug(logger, SDXExceptionCode.OAI_HARVESTER_INITIALIZE_TEMPDIR, args);
            ret = new File(Utilities.getSystemTempDir(), childDir);
            String[] args2 = {ret.getCanonicalPath()};
            OAIUtilities.logDebug(logger, SDXExceptionCode.OAI_HARVESTER_ENABLE_CREATE_TEMPDIR,args2);
        }
        Utilities.checkDirectory(ret.getCanonicalPath(), logger);
        return ret;
    }

    /**Deletes the directory
     * represented by the tempDirBatch
     * class field
     *
     */
    protected void deleteTempDirBatch() {
    	if (tempDirBatch!=null){
	        if (!this.keepHarvestedRecords) {
	            try {
	                FileUtils.deleteDirectory(this.tempDirBatch);//deleting the old tempDirBatch if possible
	            } catch (IOException e) {
	                OAIUtilities.logWarn(logger, e.getMessage(), e);
	            }
	        }
	        else {
	        	// On ne supprime pas le repertoire temporaire mais on avertie l'administrateur
	        	String[] args = {this._database.getId(), tempDirBatch.getAbsolutePath()};
        		OAIUtilities.logDebug(logger, SDXExceptionCode.OAI_HARVESTER_KEEP_RECORDS,args);
	        }
    	}
    }
    
    /** Deletes the directory represented by the tempDir class field */
    protected void deleteTempDir() {
    	if (tempDir!=null){
	        if (!this.keepHarvestedRecords) {
	            try {
	                FileUtils.deleteDirectory(this.tempDir);//deleting the old tempDir if possible
	            } catch (IOException e) {
	                OAIUtilities.logWarn(logger, e.getMessage(), e);
	            }
	        }
	        else {
	        	// On ne supprime pas le repertoire temporaire mais on avertie l'administrateur
	        	String[] args = {this._database.getId(),tempDir.getAbsolutePath()};
        		OAIUtilities.logDebug(logger, SDXExceptionCode.OAI_HARVESTER_KEEP_RECORDS,args);
	        }
    	}
    }

    /**Establishes the tempDirBatch class field
     *
     * @throws SDXException
     * @throws IOException
     */
    protected void initTempDir() throws SDXException, IOException {
        this.tempDirBatch = getNewTempDirBatch();
    }

    /**Get's the current date in iso8601 format
     *
     * @return String
     */
    protected String getIsoDate() {
        return fr.gouv.culture.sdx.utils.Date.formatDate(fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date());
    }


    /**Sets up resources to capture an oai record
     *
     *
     * @throws SAXException
     */
    protected void prepareRecordCapture() throws SAXException {
        try {

            if (this.tempDirBatch == null || !this.tempDirBatch.exists()) initTempDir();
            this.harvestDoc = File.createTempFile("oaiHarvestedRecord", TEMPFILE_SUFFIX, tempDirBatch);
            this.fileOs = new FileOutputStream(this.harvestDoc);

            //Initialise OAI properties for the record
            if (filesProperties == null)
                filesProperties = new Hashtable();
            Properties props = new Properties();
            filesProperties.put(this.harvestDoc.getName(), props);//we can work with file name, automatically generated and unique
            this.currentDatestamp = this.currentMetadtaUrlIdentifier = this.currentOaiIdentifier = this.currentOaiStatus = null;
            this.deleteRecord=false;
            if(cBytes==null){
            	cBytes = new XMLSerializer();
            	cBytes.enableLogging(logger);
            	// transformer configuration
            	DefaultConfiguration conf = new DefaultConfiguration("conf");
                DefaultConfiguration cfTf = new DefaultConfiguration("transformer-factory");
                cfTf.setValue(this.transformerFactory);
                ((DefaultConfiguration)conf).addChild(cfTf);
                cfTf = new DefaultConfiguration("indent");
                cfTf.setValue(this.transformerIndent);
                ((DefaultConfiguration)conf).addChild(cfTf);
                cBytes.configure(conf);
            }
            cBytes.setOutputStream(this.fileOs);
            if(oaiStripper==null){
            	// We must strip the OAI envelop
                oaiStripper = new OAIEnvelopStripper();
                oaiStripper.setConsumer(cBytes);
                firstXmlConsumer = oaiStripper;
            }
            super.setConsumer(oaiStripper);

        } catch (IOException e) {
            throw new SAXException(e.getMessage(), e);
        } catch (SDXException e) {
            throw new SAXException(e.getMessage(), e);
        } catch (ConfigurationException e) {
            throw new SAXException(e.getMessage(), e);
        }

    }

    /**Ends the capture of an oai record.
    *Store properties (dateStamp and OAI identifier) needed for indexation.
    *
    *
    * @throws Exception
    */
    protected void captureRecord() throws Exception {
        if (this.fileOs != null && this.harvestDoc != null) {

            this.fileOs.flush();
            this.fileOs.close();

            // shoulHarvestCurrentDocument = shouldHarvestDocument(); //we want to know if it's a good idea to harvest this document vs the docs we have in the document base

            if (OAIObject.Node.Value.DELETED.equalsIgnoreCase(currentOaiStatus))
            	resetRecordCaptureFields(true);
            else{
	            /* Setting OAI properties (dateStamp, OAI identifier) for the current record.
	             * Those informations will be used for indexation. */ 
	            Properties oaiProps = (Properties)filesProperties.get(this.harvestDoc.getName());//FIXME: do we need to prevent the case where properties are null ? -mp
	            if (this.currentDatestamp != null)
	                oaiProps.setProperty("datestamp", this.currentDatestamp);
	            else oaiProps.setProperty("datestamp", "");
	            if (this.currentOaiIdentifier != null)
	                oaiProps.setProperty("oaiid", this.currentOaiIdentifier);
	            else oaiProps.setProperty("oaiid", "");
	            if (this.currentOaiStatus != null)
	                oaiProps.setProperty("oaistatus", this.currentOaiStatus);
	            else oaiProps.setProperty("oaistatus", "");
				resetRecordCaptureFields(false);
            }

        }
    }

    /**Resets the class fields for record capture
     * possibility deleting the current <code>harvetDoc</code>
     * object underlying file
     *
     * @param deleteDoc flag for deletion of actual file
     */
    protected void resetRecordCaptureFields(boolean deleteDoc) {
        if (this.fileOs != null) {
            try {
                this.fileOs.flush();
                this.fileOs.close();
                this.fileOs = null;
            } catch (IOException e) {
                OAIUtilities.logException(logger, e);
            }
        }

        if (this.harvestDoc != null) {
            if (deleteDoc)
                this.harvestDoc.delete();
            this.harvestDoc = null;
        }

        super.setConsumer(super.firstXmlConsumer);//resetting the consumer to the first external consumer
    }


    /**Sets up resources to delete an oai record
     *Add the record to the list of the records to removed
     */
    protected void prepareRecordForDeletion() {
        if (this.deletedDocs == null) this.deletedDocs = new ArrayList();
        if (Utilities.checkString(this.currentOaiIdentifier)) {
            XMLDocument deleteDoc = null;
            try {
                deleteDoc = new XMLDocument(this.currentOaiIdentifier);//constructs an XMLDocument. We should pass the sdxdocid, not the sdxoaiid
            } catch (SDXException e) {
                //do nothing here
            }
            if (deleteDoc != null) {
            	this.deletedDocs.add(deleteDoc);
            	this.noDocsDeleted++;
            	if (null == m_docsToDeleteIds) m_docsToDeleteIds = new HashSet();
            	this.m_docsToDeleteIds.add(this.currentOaiIdentifier);
            	try{
        	        if (this.harvestDoc.exists())
        	            this.harvestDoc.delete();//delete the file in the temporary directory
        	    }catch(SecurityException se){
        	    	OAIUtilities.logException(logger, se);
        	    }
            }
        }
    }


    /*
     * Methode privee chargee de calculer si on peut lancer l'indexation ou non
     * suivant la configuration du moissonneur
     */
    protected boolean isStartsIndexation() {
    	boolean ret = false;
    	// On verifie si le moissonneur peut lancer l'indexation
        boolean indexAtHarvestEnd = true; // indique si on doit publier a la fin de la moisson ou non
		boolean forceIndexOnHarvestError = false; // force l'indexation des <records> moissonnes meme lorsque la moisson a echouee
		/* On veut savoir si on doit indexer :
		 * Si parametre « indexAtHarvestEnd » est faux : on n'indexe pas
		 * Si on a une erreur, on indexe si la parametre « forceIndexOnHarvestError » est vrai
		 * Sinon, on n'indexe pas
		 */
		boolean m_index = (!indexAtHarvestEnd) ? false : ( (OAIUtilities.checkString(this.errorCode)) ? forceIndexOnHarvestError : true );
		if (m_index) {
			OAIUtilities.logInfo(logger, SDXExceptionCode.OAI_HARVESTER_INDEXATION_LAUNCH,null );
			ret = true;
		}
		else {
			if (!indexAtHarvestEnd) {
				OAIUtilities.logInfo(logger, SDXExceptionCode.OAI_HARVESTER_INDEXATION_NOTLAUNCH_DESACTIVATE,null );
			}
			else if (OAIUtilities.checkString(this.errorCode)) {
				String[] args = {this.errorCode};
				OAIUtilities.logInfo(logger, SDXExceptionCode.OAI_HARVESTER_INDEXATION_NOTLAUNCH_ERROR, args);
			}
		}
		return ret;
    }

    /**Reads the documents from <code>tempDirBatch</code>
     * and indexes them in the corresponding document
     * base, any marked deletions will be carried out
     * as well
     *
     * @return boolean
     * @throws SDXException
     * @throws SAXException
     * @throws ProcessingException
     * @throws IOException
     * @see fr.gouv.culture.oai.AbstractOAIHarvester#storeHarvestedData()
     */
    protected boolean storeHarvestedData() 
    throws ProcessingException, IOException, SDXException, SAXException 
    {
        boolean dataHarvested = false; // marqueur indiquant qu'on a bien indexe
        boolean m_startIndexation = isStartsIndexation(); // marqueur indiquant qu'on peut lancer l'indexation
        
        if (m_startIndexation && docbase != null) {
        	
            //deleting any docs which have been removed from the repo
            if (this.deletedDocs != null && this.deletedDocs.size() > 0 )
                deleteOAIDocuments();

            IndexableDocument[] indexDocs = null;
            
            //creating our docs from the disk
            if (null != this.tempDir) {
            	
            	// Construction de la liste des fichiers a indexer
            	Set m_files = listFilesToIndex();
                ArrayList docs = new ArrayList();
                if (null == m_docsaddedIds) m_docsaddedIds = new HashSet();
                IndexParameters indexParams = getIndexParameters();
                
                // Preparation des documents a indexer
                String fileName = null;
                File m_file = null;
                int i = 0, m_nbFiles = m_files.size();
                java.util.Iterator m_it = m_files.iterator();
                while (m_it.hasNext() && null != (m_file = (File) m_it.next())) {

                	fileName = m_file.getName();

                    //Recovers OAI properties (dateStamp, OAI identifier) for the current OAI record
                    Properties oaiProps = (Properties)filesProperties.get(fileName);
                    this.currentDatestamp = oaiProps.getProperty("datestamp");
                    this.currentOaiIdentifier = oaiProps.getProperty("oaiid");
                    this.currentOaiStatus = oaiProps.getProperty("oaistatus");

                    //We don't want to indexe document that must be deleted
                    if ( this.currentOaiStatus != null
                    		&& !OAIObject.Node.Value.DELETED.equalsIgnoreCase(this.currentOaiStatus) ) 
                    {

                        //Constructs OAIDocument wich will be indexed
                        OAIDocument metadataDoc = new OAIDocument();
                        metadataDoc.setDateString(this.currentDatestamp);
                        metadataDoc.setIdentifier(this.currentOaiIdentifier);
                        m_docsaddedIds.add(this.currentOaiIdentifier);

                        try {
                            //metadataDoc.setContent(new File(tempDirBatch, fileName).toURI().toURL());
                        	metadataDoc.setContent(m_file.toURI().toURL());
                            if (docs == null) docs = new ArrayList();
                            docs.add(metadataDoc);
                            this.noHarvestedDocs++;//keeping track of all additions
                        } catch (MalformedURLException e) {
                        	OAIUtilities.logException(logger, e);
                        }

                    }

                    i++;

                }
                
                // Maintenant, on lance l'indexation
                indexDocs = (IndexableDocument[]) docs.toArray(new IndexableDocument[0]);
                if (indexDocs != null) {
                    Repository repo = null;
                    if ( Utilities.checkString(this.repoUrl) 
                    		&& storeRepositoriesRefs != null )
                    {
                        repo = (Repository) this.storeRepositoriesRefs.get(this.repoUrl);
                    }
                    /* FIXME (MP) : A-t-on besoin de consommer les evenements 
                     * SAX envoyes par le processus d'indexation ? 
                     * Si oui, attention ! ce content handler provoque une exception
                     * Caused by: java.io.IOException: Mauvais descripteur de fichier
                     * at java.io.FileOutputStream.writeBytes(Native Method)
                     * [...]
                     * at com.icl.saxon.output.XMLEmitter.writeAttribute(XMLEmitter.java:264)
                     */
                    this.docbase.index(indexDocs, repo, indexParams, /*this*/null);
                    dataHarvested = true;
                    docs = null;
                    indexDocs = null;
                }
                
            }

        }
        
        return dataHarvested;
    }

    private Set listFilesToIndex() {
    	return listFilesToIndex(null,null,null);
    }
    /**
     * M&eacute;thode priv&eacute;e pour lister les fichiers &agrave; indexer. Ces fichiers
     * ont tous une extension « .sdx ». Ils sont contenus dans des sous-dossiers du dossier
     * « tempDir ».
     * 
     * @param dir	File	Le dossier princpal dans lequel on travail. Il peut etre nul. Dans ce
     * cas, on travaillera dans « tempDir »
     * @param filter	FilenameFilter	Le filtre utilis&eacute; pour dresser la liste des fichiers &agrave;
     * indexer. Il peut etre nul. Dans ce cas, on en cree un.
     */
    private Set listFilesToIndex(File dir, FilenameFilter filter, Set files) {
    	files = (null != files) ? files : new HashSet();
    	dir = (null != dir) ? dir : this.tempDir; // si le dossier passe en parametre est nul, on travaille sur tempDir.
    	filter = (null != filter) ? filter : new FilenameFilter() { // si le filtre passe en parametre est nul, on en cree un qui prend tous les dossiers les et fichiers avec l'extension « .sdx »
												public boolean accept(File dir, String name) {
													return (dir.isDirectory() || name.endsWith(".sdx"));
												}
											};
    	if (null != dir) {
    		File[] m_files = dir.listFiles(filter);
    		for (int i = 0, l = m_files.length; i < l; i++) { // Boucle sur la liste des fichiers et dossiers dans «dir»
    			if (m_files[i].isDirectory()) listFilesToIndex(m_files[i], filter, files);
				else files.add(m_files[i]); 
    		}
    	}
    	return files;
    }


/**Delete OAI documents from the current document base.
     *
     * @throws IOException
     * @throws ProcessingException
     * @throws SDXException
     * @throws SAXException
     */
    protected void deleteOAIDocuments() 
    		throws IOException, ProcessingException, SDXException, SAXException 
	{

        if ( this.deletedDocs != null && this.deletedDocs.size() > 0 )
        {
        	
        	String[] args = {this.deletedDocs.size()+""};
        	OAIUtilities.logDebug(logger, SDXExceptionCode.OAI_HARVESTER_DELETEDDOCUMENT_START,args);

        	if (null == m_docsdeletedids) m_docsdeletedids = new HashSet();
        	
            //Construct the SearchLocations for the query
            SearchLocations slocs = new SearchLocations();
            slocs.enableLogging(this.logger);
            //slocs.addIndex(this.docbase.getIndex());
            slocs.addDocumentBase(this.docbase);

            ComplexQuery cq = new ComplexQuery();
            cq.enableLogging(this.logger);
            cq.setUp(slocs, fr.gouv.culture.sdx.search.lucene.query.Query.OPERATOR_OR);

            //boucle sur les documents a supprimer
            for( int i = 0; i < this.deletedDocs.size(); i++ ){
                fr.gouv.culture.sdx.document.XMLDocument doc = (XMLDocument) this.deletedDocs.get(i);
                String sdxoaiid = doc.getId();
                m_docsdeletedids.add(sdxoaiid); // on ajoute l'identifiant a la liste des documents OAI a supprimer pour sortir l'information en fin de moisson
                FieldQuery fq = new FieldQuery();
                fq.setUp(slocs, /*sdxoaiid*/sdxoaiid, /*fieldname: sdxoaiid*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIID);
                cq.addComponent(fq);
            }

            cq.prepare();//prepare the query

            fr.gouv.culture.sdx.search.lucene.query.Results res = cq.execute();//execute the query

            //we have results
            if ( res != null && res.count() > 0) {
                String[] ids = res.getDocIds();
                XMLDocument[] docs = new XMLDocument[ids.length];
                for ( int j = 0; j < docs.length ; j++ ){
                    docs[j] = new XMLDocument(ids[j]);
                }
                //delete the documents
                OAIUtilities.logDebug(logger, "Founds "+docs.length+" OAI document(s) to delete..."); // TODO (MP): error code et message
                this.docbase.delete(docs, null);
            }

        }

    }


    /**Handles the resumption token by issuing another request
     * based upon the request from which the resumption token was received.
     *
     */
    protected void handleResumptionToken() {
    	
        if (Utilities.checkString(super.resumptionToken) 
        		&& Utilities.checkString(super.repoUrl)) 
        {
        	String verb = "";
        	if (this.requestParams!=null) {
        		verb = this.requestParams.getParameter(OAIObject.Node.Name.VERB, "");
        	}
        	else if (this.storedRequests.size() > 0) {
        		verb = ( (OAIRequest)this.storedRequests.get(super.requestUrl) ).getVerbString();
        	}
        	
        	OAIUtilities.logDebug(logger, "Handle resumption token " + super.resumptionToken + " for the harvester of the document base " + this.docbase.getId() + "...");
        	
            if (Utilities.checkString(verb)) 
            {
                newRequestUrl = this.repoUrl + OAIRequest.URL_CHARACTER_QUESTION_MARK + 
                				OAIObject.Node.Name.VERB + OAIRequest.URL_CHARACTER_EQUALS + verb + 
                				OAIRequest.URL_CHARACTER_AMPERSAND + OAIObject.Node.Name.RESUMPTION_TOKEN + OAIRequest.URL_CHARACTER_EQUALS + super.resumptionToken;

                this.resetAllFields();

            }

            super.resetResumptionToken();//resetting the resumption token here so we don't loop

        }
    }

    //TODO-TEST: this should be preparing for a merge with the metadata, done
    /**Prepares to read a url value from an oai record and
     * retrieve the XML behind.
     *@see #identifierName
     *@see #currentMetadtaUrlIdentifier
     */
    protected void prepareResourceFromUrlIdentifierCapture() {
        if (Utilities.checkString(this.currentMetadtaUrlIdentifier)) {
            try {
                URL resourceUrl = new URL(this.currentMetadtaUrlIdentifier);
                XMLDocument resource = new XMLDocument();
                resource.setId(this.currentMetadtaUrlIdentifier);
                resource.setContent(resourceUrl);
                this.urlResource = resource;
            } catch (MalformedURLException e) {
            	OAIUtilities.logException(logger, e);
            } catch (SDXException e) {
                OAIUtilities.logException(logger, e);
            }
        }
    }

    /**Captures the xml from a url taken from an oai record and adds
     * it to the oai-record as a sibling of the <metadata/> element
     *
     */
    protected void captureResourceFromUrlIdentifier() {
        if (this.urlResource != null) {
            try {
                IncludeXMLConsumer include = new IncludeXMLConsumer(super.synchronizedXmlConsumer);
                urlResource.setConsumer(include);
                include.startElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource", new AttributesImpl());
                SAXParser parser = null;
                try {
                    parser = (SAXParser) this.manager.lookup(SAXParser.ROLE);
                    urlResource.parse(parser);
                } finally {
                    if (parser != null)
                        this.manager.release(parser);
                    this.urlResource = null;
                    include.endElement(Framework.SDXNamespaceURI, "urlResource", "sdx:urlResource");
                }
            } catch (SAXException e) {
                OAIUtilities.logException(logger, e);
            } catch (ServiceException e) {
            	OAIUtilities.logException(logger, e);
            } catch (SDXException e) {
            	OAIUtilities.logException(logger, e);
            }
        }

    }


    /**Resets necessary class fields
     *
     */
    protected void resetAllFields() {
        this.deletedDocs = null;
        this.urlResource = null;
        resetRecordCaptureFields(false);
        deleteTempDirBatch();
        this.tempDirBatch = null;
		super.resetAllFields();
    }

    /** Ends the harvest */
    protected void endHarvest(){
    	noHarvestedDocs = noDocsDeleted = 0;
    	deleteTempDir();
        this.tempDir = null;
    }


    /**Builds simple index parameters for indexation of
     * oai records into the undelryi
     * @return IndexParameters
     */
    protected IndexParameters getIndexParameters() {
        IndexParameters params = new IndexParameters();
        params.setSendIndexationEvents(IndexParameters.SEND_ALL_EVENTS);
        params.setBatchMax(this.noRecordsPerBatch);
        if ( this.pipe != null ) params.setPipeline(this.pipe);
        else if ( this.docbase != null ) params.setPipeline(this.docbase.getIndexationPipeline());
        params.setPipelineParams(super.getHarvestParameters());
        return params;
    }

    /**Sends the details of stored harvesting requests
     * to the current consumer
     *
     * @throws SAXException
     */
    public void sendStoredHarvestingRequests() 
    throws SAXException 
    {
        try {
            if (this.storedRequests != null && this.storedRequests.size() > 0) {
                this.acquire();
                super.acquireSynchronizedXMLConsumer();
                super.startElement(Framework.SDXNamespaceURI,
                        fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS,
                        Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS,
                        null);
                Enumeration requests = storedRequests.elements();
                if (requests != null) {
                    while (requests.hasMoreElements()) {
                        OAIRequest request = (OAIRequest) requests.nextElement();
                        if (request != null)
                            request.toSAX(this);
                    }
                }

                super.endElement(Framework.SDXNamespaceURI,
                        fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS, Framework.SDXNamespacePrefix + ":" +
                        fr.gouv.culture.sdx.utils.constants.Node.Name.STORED_HARVEST_REQUESTS);
            }
        } catch (InterruptedException e) {
            throw new SAXException(e.getMessage(), e);
        } finally {
            super.releaseSynchronizedXMLConsumer();
            this.release();
        }

    }


    /**Triggers an OAI request to a repository based
     * upon a trigger name (also a request url)
     *
     * @param triggerName
     */
    public synchronized void targetTriggered(String triggerName) {
    	String[] args = {triggerName};
    	if (logger.isInfoEnabled()){
			OAIUtilities.logInfo(logger, SDXExceptionCode.OAI_HARVESTER_LAUNCHING, args);
    	}
    	if (docbase==null){
    		if(logger.isErrorEnabled()){
    			OAIUtilities.logError(logger, SDXExceptionCode.OAI_HARVESTER_LAUNCHING_ERROR_DOCUMENTBASE, args);
    		}
    		return;
    	}
        //verify that we have this trigger defined
        OAIRequest request = (OAIRequest) this.storedRequests.get(triggerName);
        if (request != null) {

			// Test if we have to force the use of the from parameter (in application.xconf) instead of the last haverst date
        	boolean useLastHarvestDate = request.getUseLastHarvestDate();
        	if ( useLastHarvestDate ){
        		try {
						DatabaseEntity dbe = this._database.getEntity(request.getVerbId());
						if ( dbe != null ){
							String lastHarvestDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED);
							request.setFrom(lastHarvestDate);
						}
					} catch (SDXException e) {
						OAIUtilities.logException(logger, e);
					}
        	}

            String requestUrl = request.getRequestURL();
            String[] args2 = {requestUrl};
            if (logger.isInfoEnabled()){
            	OAIUtilities.logDebug(logger, SDXExceptionCode.OAI_HARVESTER_REQUEST_BUILDED,args2);
            }
            this.receiveSynchronizedRequest(requestUrl, triggerName);// send the request with the original one
            this.endHarvest();
            if (logger.isInfoEnabled()){
            	OAIUtilities.logInfo(logger, SDXExceptionCode.OAI_HARVESTER_REQUEST_ENDED,args2);
            }
        }

    }


    public void startElement(String s, String s1, String s2, Attributes attributes) throws SAXException {
        if (Utilities.checkString(this.identifierName)) {
            /*looking for an element name or the VALUE of an attribute with the name "name" matching super.indentifierName
            *so one could create a metadata identifier element like
            *<super.identifierName>myIdentifierValue</super.identifierName>
            *or
            *<anyElementName name="super.identifierName">myIdentifierValue</anyElementName>
            */
            if (attributes != null && this.identifierName.equals(attributes.getValue(fr.gouv.culture.sdx.utils.constants.Node.Name.NAME)))
                this.captureElemContent = true;
        }
        super.startElement(s, s1, s2, attributes);
    }

    public void endElement(String s, String s1, String s2) throws SAXException {
        if (super.sBuff != null && super.sBuff.length() > 0) {
            String content = super.sBuff.toString();

            if (!OAIObject.Node.Xmlns.OAI_2_0.equals(s)) {
                if (fr.gouv.culture.sdx.utils.constants.Node.Name.FIELD.equals(s1)) {
                    super.currentMetadtaUrlIdentifier = content;
                    try {
                        prepareResourceFromUrlIdentifierCapture();
                    } catch (Exception e) {
                        //if we can't build the document we don't just fail completely we will continue
                    	OAIUtilities.logException(logger, e);
                    }
                }
            }

        }
        super.endElement(s, s1, s2);
    }

    /**Querys the underlying data structures
     * based upon current sax flow
     * position/set class fields and
     * determines whether an oai record should be
     * harvested
     *
     * @return boolean indicates whether the record should be handled
     */
    protected boolean shouldHarvestDocument() {
    	boolean ret = true;
    	try {

    		//First control: do we have a previous harvest for this provider/verb ? If so, look for the datestamp of the current oai doc vs. the time of the last harvest
    		OAIRequest request = (OAIRequest) this.storedRequests.get(super.requestUrl);
    		if (request != null && request.getVerbId()!=null && !request.getVerbId().equals("") && this._database.entityExists(request.getVerbId())){
				DatabaseEntity dbe = this._database.getEntity(request.getVerbId());
				String formattedDate = dbe.getProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED);
				java.util.Date lastUpdatedDate = fr.gouv.culture.sdx.utils.Date.parseDate(formattedDate);
				java.util.Date currentDocumentDate = fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp);
				if (currentDocumentDate.getTime() > lastUpdatedDate.getTime())
					return ret;//the timestamp of the current doc is more recent than the last harvest, we should harvest it
    		}

			//Second control: search for an sdxdocument which correspond to the current oai id
    		else if (Utilities.checkString(super.currentOaiIdentifier)) {

    			//The complexQuery
    			ComplexQuery cq = new ComplexQuery();
    			SearchLocations slocs = new SearchLocations();
    			slocs.enableLogging(this.logger);
    			slocs.addDocumentBase(this.docbase);
    			cq.enableLogging(this.logger);
    			cq.setUp(slocs,fr.gouv.culture.sdx.search.lucene.query.Query.OPERATOR_AND);
    			//The fieldQuery
    			FieldQuery fq = new FieldQuery();
    			fq.setUp(slocs, /*sdxoaiid*/super.currentOaiIdentifier, /*fieldname: sdxoaiid*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIID);
    			cq.addComponent(fq);
    			//The dateQuery
                DateIntervalQuery dq = new DateIntervalQuery();
                java.util.Date currentOaiDocumentDate= fr.gouv.culture.sdx.utils.Date.parseDate(super.currentDatestamp);
                dq.setUp(slocs, /*sdxoaidate*/OAIDocument.INTERNAL_FIELD_NAME_SDXOAIDATE, /*from*/currentOaiDocumentDate, /*to*/null, true/*bounds inclusive*/);
                cq.addComponent(dq);

    			cq.prepare();
    			fr.gouv.culture.sdx.search.lucene.query.Results res = cq.execute();

    			//we have result(s)
    			if (res != null && res.count() == 1 ){//1 doc exists with this sdxoaiid and a sdxoaidate more recent than the current doc, we should not harvest it
					ret = false;
    			}
    			else if(res != null && res.count() > 1) {//More than 1 result: error
    				OAIUtilities.logError(logger, "A problem occured during harvesting: "+res.count()+" documents correspond to the OAI identifier: "+super.currentOaiIdentifier, null);
    			}
    			else return ret;//no document with this id exists, so we should harvest it
    		}
    	} catch (SDXException e) {
    		OAIUtilities.logException(logger, e);
    	}

    	return ret;
    }

    /**Saves critical data about a harvest
     *
     * @param dataHarvested
     * @throws SAXException
     */
    protected void saveCriticalFields(boolean dataHarvested) 
    		throws SAXException 
	{

		try {

			OAIRequest request = (OAIRequest) this.storedRequests.get(super.requestUrl);

			String verbId = request.getVerbId();
			String[] args = new String[13];
			args[0] = super.requestUrl;
			args[1] = super.repoUrl;

			if ( request != null && Utilities.checkString(verbId) )
			{

	        	DatabaseEntity dbe = this._database.getEntity(verbId);
	        	if (dbe == null) dbe = new DatabaseEntity(verbId);
	        	if(!Utilities.checkString(dbe.getId())) dbe.setId(verbId);//is it necessary ? -mp

	            //Now we have to clear the database
	            Property[] props = dbe.getProperties();
	            if (props.length > 0){
		            for(int i=0 ; i < props.length; i++){
		            	//Property prop = props[i];
		            	dbe.deleteProperty(props[i].getName());
		            }
	            }

	            //URL request
				if ( Utilities.checkString(super.requestUrl) ) {
					dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REQUEST_URL, super.requestUrl);
				}

				//URL repo
				if ( Utilities.checkString(super.repoUrl) ) {
					dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL, super.repoUrl);
				}

				//Granularity
				String granularity = request.getGranularity();
				if ( Utilities.checkString(granularity) 
					&& Utilities.checkString(super.responseDate) )
				{
					
					String resd = "";
					
					//Store last harvest date with the correct granularity for the current provider
					if ( granularity.equals(OAIObject.Node.Value.STRING_GRANULARITY_SECOND) ) 
					{
						resd = super.responseDate;
					}

					else if( granularity.equals(OAIObject.Node.Value.STRING_GRANULARITY_DAY) ) 
					{
						resd = super.responseDate.substring(0, super.responseDate.indexOf("T") );
					}
					
					dbe.addProperty( LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_LAST_UPDATED, resd );
					args[2] = (Utilities.checkString(resd)) ? resd : "";

				}

				//Verb request
				String verb = request.getVerbString();
				if ( Utilities.checkString(verb) ) {

					//Verb as a string
					dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_VERB, verb);
					args[3] = (Utilities.checkString(verb)) ? verb : "";

					//MetadataPrefix
					String mdPrefix = request.getMetadataPrefix();
					args[4] = mdPrefix;
					if ( Utilities.checkString(mdPrefix) ) {
						dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_METADATA_PREFIX, mdPrefix);
					}

					//Case : verb=GetRecord
					String id = "";
					if ( verb.equals(OAIRequest.VERB_STRING_GET_RECORD) )
					{
						//identifier
						id = requestParams.getParameter(OAIRequest.URL_PARAM_NAME_IDENTIFIER, null);
						if ( Utilities.checkString(id) ) {
							dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_IDENTIFIER, id);
						}
					}
					args[5] = (Utilities.checkString(id)) ? id : "";

					//Case ListRecords
					if (verb.equals(OAIRequest.VERB_STRING_LIST_RECORDS)) {

						//from
						String from = request.getFrom();
						if ( Utilities.checkString(from) ) {
							dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_FROM, from);
						}
						args[6] = (Utilities.checkString(from)) ? from : "";

						//until
						String until = request.getUntil();
						if ( Utilities.checkString(until) ) {
							dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_UNTIL, until);
						}
						args[7] = (Utilities.checkString(until)) ? until : "";

						//set
						String set = request.getSetIdentifier();
						if ( Utilities.checkString(set) ) {
							dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_SET, set);
						}
						args[8] = (Utilities.checkString(set)) ? set : "";

					}

				}

				if ( Utilities.checkString(super.resumptionToken) ) {
					dbe.addProperty(LuceneDocumentBaseOAIHarvester.OAI_HARVESTER_RESUMPTION_TOKEN, super.resumptionToken);
				}
				dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_DELETED, Integer.toString(this.noDocsDeleted));
				dbe.addProperty(LuceneDocumentBaseOAIHarvester.NO_DOCS_HARVESTED, Integer.toString(this.noHarvestedDocs));
				
				args[9] = (this.noDocsDeleted > 0) ? this.noDocsDeleted+"" : "0";
				args[10] = (this.noHarvestedDocs > 0) ? this.noHarvestedDocs+"" : "0";
				//args[11] = (null != this.m_docsaddedIds && !this.m_docsaddedIds.isEmpty()) ? this.m_docsaddedIds.toString() : "";
				// args[12] = (null != this.m_docsdeletesIds && !this.m_docsdeletesIds.isEmpty()) ? this.m_docsdeletesIds.toString() : "";

				StringBuffer m_sb = null;
				if ( null != this.m_docsaddedIds && !this.m_docsaddedIds.isEmpty() ) {
					m_sb = new StringBuffer( this.m_docsaddedIds.size() );
					Iterator m_it = this.m_docsaddedIds.iterator();
					int i = 0;
					while ( m_it.hasNext() ) {
						i++;
						m_sb.append( "\n\t" + i + " : " + (String) m_it.next() );
					}
					m_it = null;
					m_sb.trimToSize();
				}
				args[11] = (null != m_sb && m_sb.length() > 0) ? m_sb.toString() : "" ;
				
				m_sb = null;
				if ( null != this.m_docsdeletedids && !this.m_docsdeletedids.isEmpty() ) {
					m_sb = new StringBuffer( this.m_docsdeletedids.size() );
					Iterator m_it = this.m_docsdeletedids.iterator();
					int i = 0;
					while ( m_it.hasNext() ) {
						i++;
						m_sb.append( "\n\t" + i + " : " + (String) m_it.next() );
					}
					m_it = null;
					m_sb.trimToSize();
				}
				args[12] = (null != m_sb && m_sb.length() > 0) ? m_sb.toString() : "" ;
				
				this._database.update(dbe);
				this._database.optimize();
				
				OAIUtilities.logInfo(logger, SDXExceptionCode.OAI_HARVESTER_SUMMARY, args);

			}

		} catch (Exception e) {
			throw new SAXException(e.getMessage(), e);
		}
    }

    /**Generates an id to associate
     * with a harvest
     *
     * @return String
     */
    protected String generateNewHarvestId() {
        return this.harvesterIdGen.generate();
    }

    /**Sends sax events to the current consumer
     * with summary details of the all the past harvests
     *
     * @throws SAXException
     */
    public void sendPastHarvestsSummary() throws SAXException {
        //build a query get the doc
        try {
            DatabaseEntity[] dbes = this._database.getEntities();
            if (dbes != null && dbes.length > 0) {
                this.acquire();
                acquireSynchronizedXMLConsumer();
                super.startElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS,
                        Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS, null);
                for (int i = 0; i < dbes.length; i++) {
                    DatabaseEntity dbe = dbes[i];
                    AttributesImpl atts = new AttributesImpl();
                    String repoUrl = null;
                    if (dbe != null) {
                        Property[] props = dbe.getProperties();
                        if (props != null) {
                            for (int j = 0; j < props.length; j++) {
                                Property prop = props[j];
                                if (prop != null) {
                                    String propName = prop.getName();
                                    if (Utilities.checkString(propName) && !LuceneDocumentBaseOAIHarvester.OAI_HARVEST_ID.equals(propName)) {
                                        String propVal = prop.getValue();
                                        if (propName.equals(LuceneDocumentBaseOAIHarvester.OAI_REPOSITORY_URL)) {
                                            repoUrl = propVal;
                                        } else {
                                            if (Utilities.checkString(propVal))
                                                atts.addAttribute("", propName, propName, OAIObject.Node.Type.CDATA, propVal);
                                        }
                                    }
                                }

                            }
                        }

                    }
                    sendElement(OAIObject.Node.Xmlns.OAI_2_0, OAIObject.Node.Name.REQUEST, OAIObject.Node.Name.REQUEST,
                            atts, repoUrl);
                }
                super.endElement(Framework.SDXNamespaceURI, fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS,
                        Framework.SDXNamespacePrefix + ":" + fr.gouv.culture.sdx.utils.constants.Node.Name.PREVIOUS_HARVESTS);
            }

        } catch (Exception e) {
            throw  new SAXException(e.getMessage(), e);
        } finally {
            releaseSynchronizedXMLConsumer();
            this.release();
        }


    }

    /**Retrieves the time when the harvester was last updated
     *
     * @return Date
     */
    public Date lastUpdated() {
        return this.docbase.lastModificationDate();
    }

    /**Destroys all summary data pertaining to past harvests
     * but not the actual oai records harvested
     *
     */
    public void purgePastHarvestsData() {
        try {
            DatabaseEntity[] dbes = this._database.getEntities();
            for (int i = 0; i < dbes.length; i++) {
                DatabaseEntity dbe = dbes[i];
                this._database.delete(dbe);
            }
            this._database.optimize();
        } catch (SDXException e) {
            OAIUtilities.logException(logger, e);
        }
    }

    /**Stores data about harvesting failures caused
     * by problems other than  oai errors sent from
     * a queried repository
     *
     * @param e
     */
    protected void storeFailedHarvestData(Exception e) {
        try {
            //TODO: call this method when harvesting fails/major exception thrown
            //create a special property "failedHarvest"
            DatabaseEntity dbe = new DatabaseEntity(generateNewHarvestId());
            String message = "noMessage";
            if (e != null) message = e.getMessage();
            dbe.addProperty(OAI_FAILED_HARVEST, message);
            //create a _database entity and store all relevant class fields
            dbe.addProperty(OAI_REQUEST_URL, this.requestUrl);
            //save it to the database
            this._database.update(dbe);
            this._database.optimize();
            //TODO:later perhaps we can try to auto-reexecute these failed request, keying on the OAI_FAILED_HARVEST property ???
        } catch (SDXException e1) {
            OAIUtilities.logException(logger, e1);
        }
    }

    /** Save the timeStamp of the Harvester
     * @see fr.gouv.culture.sdx.utils.save.Saveable#backup(fr.gouv.culture.sdx.utils.save.SaveParameters)
     */
    public void backup(SaveParameters save_config) throws SDXException {
        if(save_config != null)
            if(save_config.getAttributeAsBoolean(Saveable.ALL_SAVE_ATTRIB,false))
            {
                String oai_path = DocumentBase.ConfigurationNode.OAI_HARVESTER;
                File oai_dir = new File(save_config.getStoreCompletePath()+ File.separator +oai_path);
                if(!oai_dir.exists())
                    oai_dir.mkdir();
            }
    }

    /** Restore the timeStamp of the Harvester
     * @see fr.gouv.culture.sdx.utils.save.Saveable#restore(fr.gouv.culture.sdx.utils.save.SaveParameters)
     */
    public void restore(SaveParameters save_config) throws SDXException {
        // TODO Auto-generated method stub
    }

    /** Close OAI harvester.
     */
    public void close(){
    	if(scheduler!=null && storedRequests!=null){
    		String tn=null;
    		for(Enumeration e = storedRequests.elements() ; e.hasMoreElements() ;){
    			tn = (String) e.nextElement();
    			if (Utilities.checkString(tn)){
    				scheduler.removeTrigger(tn);
    			}
    		}
    		storedRequests = new Hashtable();
    		scheduler = null;
    	}
    }
}