/*
Copyright (C) 2000-2010  Ministere de la culture et de la communication (France), AJLSM
See LICENCE file
 */
package fr.gouv.culture.sdx.documentbase;

// SDX imports

//import fr.gouv.culture.sdx.query.SDXResults;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.text.DecimalFormat;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Vector;

import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.configuration.DefaultConfigurationBuilder;
import org.apache.avalon.framework.context.Context;
import org.apache.avalon.framework.context.ContextException;
import org.apache.avalon.framework.service.ServiceException;
import org.apache.cocoon.ProcessingException;
import org.apache.cocoon.environment.SourceResolver;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.search.ParallelMultiSearcher;
import org.apache.lucene.search.Searchable;
import org.apache.lucene.search.Searcher;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;

import fr.gouv.culture.oai.OAIRepository;
import fr.gouv.culture.sdx.application.Application;
import fr.gouv.culture.sdx.document.Document;
import fr.gouv.culture.sdx.document.IndexableDocument;
import fr.gouv.culture.sdx.document.IndexableFieldProperty;
import fr.gouv.culture.sdx.exception.SDXException;
import fr.gouv.culture.sdx.exception.SDXExceptionCode;
import fr.gouv.culture.sdx.oai.LuceneDocumentBaseOAIHarvester;
import fr.gouv.culture.sdx.oai.LuceneDocumentBaseOAIRepository;
import fr.gouv.culture.sdx.repository.Repository;
import fr.gouv.culture.sdx.search.lucene.DateField;
import fr.gouv.culture.sdx.search.lucene.FieldList;
import fr.gouv.culture.sdx.search.lucene.analysis.MetaAnalyzer;
import fr.gouv.culture.sdx.search.lucene.query.Index;
import fr.gouv.culture.sdx.search.lucene.query.LuceneIndex;
import fr.gouv.culture.sdx.utils.Utilities;
import fr.gouv.culture.sdx.utils.configuration.ConfigurationUtils;
import fr.gouv.culture.sdx.utils.constants.ContextKeys;
import fr.gouv.culture.sdx.utils.constants.Node;
import fr.gouv.culture.sdx.utils.logging.LoggingUtils;
import fr.gouv.culture.sdx.utils.save.SaveParameters;
import fr.gouv.culture.sdx.utils.save.Saveable;

/**
 *	A document base within an SDX application.
 *
 * <p>
 * A document base is a very important document in SDX development. A document base
 * is where documents are searched and retrieved, thus added (indexed), deleted or
 * updated. A search cannot occur in a smaller unit than the document base. To exclude
 * some parts of a document base, one should use query constructions, possibly filters.
 * <p>
 * A document base has a structure ; this structure is basically a list of fields. An
 * application may have many document bases, and these document bases may have different
 * structures. As always, indexable documents (XML, HTML or the like) with different
 * structures can be indexed within a single document base.
 * <p>
 * Most applications will have only one document base, but in some cases it could
 * be interesting to have more than one, like when different kinds of documents are
 * never searched at the same time, in this case it would speed up the searching and
 * indexing process if they are separated in different document bases.
 * <p>
 * A document base uses an indexer to index documents. It uses repositories
 * to store the documents, either indexable ones or attached ones (i.e. non-indexable documents
 * that are logically dependant of the indexable documents, images or the like).
 * An application can get a searcher to perform searches within this document base,
 * possibly with other document bases.
 * <p>
 * In order to work properly, a document base must be instantiated given the following sequence :
 * 1) creation,
 * 2) setting the super.getLog() (optional, but suggested for errors messages),
 * 3) configuration,
 * 4) initialization.
 *
 * @see #enableLogging
 * @see #configure
 * @see #init
 *
 */
/**
 * @author mpichot
 *
 */
public class LuceneDocumentBase extends SDXDocumentBase {

    /************
     Class members
     ************/

    /** The sub-indexes for this document base (first entry is the activeIndex) */
    protected Vector luceneSearchIndexList = new Vector();
    /** The active index for this document base */
    protected LuceneIndex luceneActiveIndex;
    /** The temporary index for this document base */
    protected LuceneIndex luceneCurrentIndex;
    /** The (Lucene) fields that are to be handled by the index. */
    protected FieldList _fieldList;
    /** The list of fields with a XML type*/
    protected HashMap _xmlFieldList;

    /** Number of subindexes */
    protected int subIndexCount;
    /** Number of indexed doc since last split */
    protected long lastDocCount;

    /***************************
    Directory names for indexes
    ****************************/
    protected final String INDEX_DIR_CURRENT = "current";
    protected final String INDEX_DIR_MAIN = "main";


    /**************************************************************
     String representation of the keys used by the properties obvject
     **************************************************************/


    /***************************************
     Directory names to be provided to Lucene
     ***************************************/

    /** The directory name for the index that stores documents' indexation. */
    protected final String SEARCH_INDEX_DIRECTORY_NAME = "sdx-search-index";

    /**********************************************************************
     Attribute names for the configuration element in the configuration file
     **********************************************************************/

    /** The implied attribute stating whether this document base is to be exposed to remote access or not. */
    public static final String DBELEM_ATTRIBUTE_REMOTE_ACCESS = "remote-access";

    /*************************************************************************
     Child element names of the configuration element in the configuration file
     *************************************************************************/


    /************************************************************************
     Internal fields for documents (kind of metadata required/proposed by SDX)
     //TODO : move them to a Document/super-DB/application class ? -pb
     ************************************************************************/

    /** The element used to define system fields in sdx.xconf. */
    public static final String ELEMENT_NAME_LUCENE_SDX_INTERNAL_FIELDS = "luceneSDXInternalFields"; //TODO : as is, is it the right place for defining this ? Isn't it Framework dependant (as sdxConf is) or, in a better way, application dependant ? -pb

    /** Internal field name for identifying ordered relationships between documents. */
    // private final String RELATION_PROPERTY_ORDER = "order"; //TODO : use value defined in a DB/Relationship  class  ? -pb

    /*
        TODO :
        Some of the values below could probably be moved next to the value sets as they are defined above.
        In such a case, their names, visibility and modifiers shohld be harmonized.
        Some other ones could also be moved to DB or Document related classes.
        I still need futher investigations to dipatch them accurately ;-)
        - pb
    */


    /**Creates the document base.
     *
     * After a document base is created, the super.getLog()  could be set (optional, but suggested for errors messages) ;
     * it should then be configured and after, initialized in order to work properly.
     *
     * @see #enableLogging
     * @see #configure
     * @see #init
     */
    public LuceneDocumentBase() {
    }

    /** Sets the configuration options for this document base.
     *
     *
     * @param configuration      The configuration object from which to build a document base.
     * <p>Sample configuration entry:
     <pre>&lt;sdx:documentBase sdx:id = "myDocumentBaseName" sdx:type = "lucene">
     &nbsp;&nbsp;&lt;sdx:fieldList xml:lang = "fr-FR" sdx:variant = "" sdx:analyzerConf = "" sdx:analyzerClass = "">
     &lt;sdx:field code = "fieldName" type = "word" xml:lang = "fr-FR" sdx:analyzerClass = "" sdx:analyzerConf = ""/>
     &lt;sdx:field code = "fieldName2" type = "field" xml:lang = "fr-FR" brief = "true"/>
     &lt;sdx:field code = "fieldName3" type = "date" xml:lang = "fr-FR"/>
     &lt;sdx:field code = "fieldName4" type = "unindexed" xml:lang = "fr-FR"/>
     &lt;/sdx:fieldList>
     &lt;sdx:index>
     &lt;sdx:pipeline sdx:id = "sdxIndexationPipeline">
     &lt;sdx:transformation src = "path to stylesheet, can be absolute or relative to the directory containing this file" sdx:id = "step2" sdx:type = "xslt"/>
     &lt;sdx:transformation src = "path to stylesheet, can be absolute or relative to the directory containing this file" sdx:id = "step3" sdx:type = "xslt" keep = "true"/>
     &lt;/sdx:pipeline>
     &lt;/sdx:index>
     &lt;sdx:repositories>
     &lt;sdx:repository baseDirectory = "blah4" depth = "3" extent = "100" sdx:type = "FS" sdx:default = "true" sdx:id = "blah4"/>
     &lt;sdx:repository ref = "blah2"/>
     &lt;/sdx:repositories>
     &lt;/sdx:documentBase>
     </pre>
     */
    public void configure(Configuration configuration) throws ConfigurationException {
        //configuring the super class
        super.configure(configuration);
    }

    /**
     * Configures the Lucene document base
     * @param configruation Configuration
     * @throws ConfigurationException
     */
    protected void configureDocumentBase(Configuration configuration) 
    throws ConfigurationException 
    {
        configureFieldList(configuration);
        configureSearchIndex();
        initializeVectorizedIndex();
        // Optimize if needed
        if(this.autoOptimize) this.optimize();
        // get the exceding document count in view of index splitting
        this.lastDocCount = this.luceneCurrentIndex.size() % this.splitDoc;
    }


    /**
     * Configures the fields list
     * @param configuration
     * @throws ConfigurationException
     */
    protected void configureFieldList(Configuration configuration) throws ConfigurationException {
        try {
            /*check for the ref attribute, if it exists, get the fiekdList object and add it to the local hashtable
            *if the attribute doesn't exist create the fieldList like below, we also need to handle DEFAULTS with refs*/
            //getting the list of fields for indexing
        	Configuration l_fieldListConf = null;
        	if(configuration.getName().equals(DocumentBase.ConfigurationNode.FIELD_LIST))
        		l_fieldListConf = configuration;
        	else
        		l_fieldListConf = configuration.getChild(DocumentBase.ConfigurationNode.FIELD_LIST, true);
            String ref = l_fieldListConf.getAttribute(Repository.ConfigurationNode.REF, null);
            if (Utilities.checkString(ref)) {
                Context appFieldLists = (Context) super.getContext().get(ContextKeys.SDX.Application.FIELD_LISTS);
                if (appFieldLists != null)
                    this._fieldList = (FieldList) appFieldLists.get(ref);
                this._fieldList = (FieldList) this._fieldList.clone();
                this._fieldList = (FieldList) super.setUpSdxObject(this._fieldList);
                this._fieldList.configure(configuration);//going to add any additional fields to this l_fieldListConf reference
                if (this._fieldList == null) {
                    String[] args = new String[1];
                    args[0] = ref;
                    throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_LOAD_REFERENCED_FIELD_LIST, args, null);
                }
            } else {
                //at this point, we should have <sdx:l_fieldListConf> containing a list of fields
                //TODO this functionality should be encapsulated in FieldList.java
                this._fieldList = ConfigurationUtils.configureFieldList(super.getLog(), getServiceManager(), Utilities.createNewReadOnlyContext(getContext()), new DefaultIDGenerator().generate(), configuration);
                this._fieldList = ConfigurationUtils.configureLuceneFieldList(super.getLog(), super.getContext(), this._fieldList);
            }

            super.getContext().put(ContextKeys.SDX.DocumentBase.FIELD_LIST, _fieldList);

        } catch (SDXException e) {
            //we don't want all repositories configurations to fail so we won't throw this farther out
            //the creation of the SDXException should log this message
        } catch (ConfigurationException e) {
            //we don't want all repository configurations to fail so we won't throw this farther out
            LoggingUtils.logException(super.getLog(), e);
        } catch (ContextException e) {
            LoggingUtils.logException(super.getLog(), e);
        }

    }

    /**
     * Reloads the fieldList of an application
     * @param appConfString The path of the configuration file wich contain the new fieldList (eg, file:///myFiles/application.xconf, cocoon://myApplication/conf/application.xconf)
     * @throws SDXException
     */
    public void reloadFieldList(String appConfString) throws SDXException{

    	if(appConfString!=null && !appConfString.equals("")){

    		Configuration appConf = null;

    		try {
    			DefaultConfigurationBuilder appConfigBuild = new DefaultConfigurationBuilder(true);
    			org.apache.excalibur.source.Source source = null;
    			org.apache.excalibur.source.SourceResolver resolver = (org.apache.excalibur.source.SourceResolver) _manager.lookup(SourceResolver.ROLE);
    			source = resolver.resolveURI(appConfString);
    			appConf = appConfigBuild.build(org.apache.cocoon.components.source.SourceUtil.getInputSource(source));
    		} catch (Exception e){
    			String[] args = new String[2];
    			args[0] = _id;
    			args[1] = appConfString;
    			//by itself, the creation of the SDXException should log the message
    			new SDXException(super.getLog(), SDXExceptionCode.ERROR_APP_CONFIG_FILE_READ, args, e);
    		}

    		// the fieldList configuration for the current document base
    		Configuration m_conf = null;

    		try {
    			//we already have the correct part of configuration file for the fields list
    			if(appConf.getName().equals(DocumentBase.ConfigurationNode.FIELD_LIST))
    				m_conf = appConf;

    			//we have to parse the complet configuration file to sdx:application from sdx:fieldlist of the correct documentBase (id = dbid)
    			else if( appConf.getName().toLowerCase().equals(Application.CLASS_NAME_SUFFIX.toLowerCase()) && Utilities.checkString(_id) ){

    				String documentBaseElementName = Utilities.getElementName(DocumentBase.CLASS_NAME_SUFFIX);//return String "documentBase"
    				Configuration[] dbsConf = new Configuration[appConf.getChild(documentBaseElementName+"s").getChildren(documentBaseElementName).length];
    				dbsConf = appConf.getChild(documentBaseElementName+"s")/*sdx:documentBases*/.getChildren(documentBaseElementName);//sdx:documentBase
    				if(dbsConf!=null && dbsConf.length > 0){
    					for(int i=0; i < dbsConf.length; i++){
    						if( dbsConf[i].getAttribute("id").equals(_id) ){
    							m_conf= dbsConf[i].getChild(DocumentBase.ConfigurationNode.FIELD_LIST, true);
    							break;
    						}
    					}
    				}

    			}

    			configureFieldList(m_conf);//configure the new fieldList
    			configureSearchIndex();//update Lucene indexes

    		} catch (ConfigurationException e) {
    			String[] args = new String[2];
    			args[0] = this._id;
    			args[1] = appConf.getLocation();
    			//by itself, the creation of the SDXException should log the message
    			throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_APP_CONFIG_FILE_READ, args, e);
    		}

    	}

    }

    /**
     * Replaces the current fieldList by the new one
     * @param fieldList The new fieldList wich replace the old one
     * @throws ConfigurationException
     */
    public void replaceFieldList(FieldList fieldList) throws ConfigurationException{
    	this._fieldList = fieldList;
    	try{
    		configureSearchIndex();//update Lucene indexes
    	} catch (ConfigurationException e){
    		String[] args = new String[1];
			args[0] = this._id;
			//by itself, the creation of the SDXException should log the message
			new SDXException(super.getLog(), SDXExceptionCode.ERROR_APP_CONFIG_FILE_READ, args, e);
    	}
    }


    //SDX search architecture/Lucene specific
    //TODORefactor: in the future this may be declared in the parent class and overridden here
    /**
     * Configures Lucene search index
     */
    protected void configureSearchIndex() 
    throws ConfigurationException 
    {
        try {
            //Getting configuration info for the search index
            String rmiHost = Utilities.getStringFromContext(ContextKeys.SDX.Framework.RMI_HOST, super.getContext());
            Integer rmiPort = (Integer) super.getContext().get(ContextKeys.SDX.Framework.RMI_PORT);
            String appId = Utilities.getStringFromContext(ContextKeys.SDX.Application.ID, super.getContext());

            // set the index root directory
            //by using our document base dir path from the properties object, as this way we are sure it is the same each time
            this.baseIndexDir = Utilities.getStringFromContext(ContextKeys.SDX.DocumentBase.DIRECTORY_PATH, super.getContext()) + File.separator + SEARCH_INDEX_DIRECTORY_NAME + File.separator;

            //creating the search index directory
            String sdxSearchIndexPath = this.baseIndexDir + INDEX_DIR_MAIN + File.separator;
            //testing the directory, to ensure it is available and we have access
            File sdxSearchIndexDir = Utilities.checkDirectory(sdxSearchIndexPath, super.getLog());
            //building the search index
            luceneActiveIndex = new LuceneIndex(sdxSearchIndexDir, rmiHost, rmiPort, appId, getId(), this.useCompoundFiles);
            luceneActiveIndex = (LuceneIndex) super.setUpSdxObject(luceneActiveIndex, super._configuration);
            luceneActiveIndex.setId(INDEX_DIR_MAIN);
            //creating a metaAnalyzer for the search index
            MetaAnalyzer mAnalyzer = new MetaAnalyzer();
            //setting the super.getLog()
            mAnalyzer.enableLogging(super.getLog());
            //setting up the metaAnalyzer
            mAnalyzer.setUp(_fieldList);
            //passing the metaAnalyzer to the search index
            luceneActiveIndex.setMetaAnalyzer(mAnalyzer);
            luceneActiveIndex.init();

            //configuring temporary index
            String sdxTempIndexPath = this.baseIndexDir + INDEX_DIR_CURRENT + File.separator;
            File sdxTempIndexDir = Utilities.checkDirectory(sdxTempIndexPath, super.getLog());
            luceneCurrentIndex = new LuceneIndex(luceneActiveIndex, sdxTempIndexDir);
            luceneCurrentIndex = (LuceneIndex) super.setUpSdxObject(luceneCurrentIndex, super._configuration);
            luceneCurrentIndex.setId(INDEX_DIR_CURRENT);
            luceneCurrentIndex.setMetaAnalyzer(mAnalyzer);
            luceneCurrentIndex.init();

            // set the current index as the temp index for the main index
            //luceneActiveIndex.setBatch(sdxTempIndexDir);

        } catch (SDXException e) {
            throw new ConfigurationException(e.getMessage(), e);
        } catch (ContextException e) {
            throw new ConfigurationException(e.getMessage(), e);
        }
    }

    /**Creates an OAIRepository for the documentbase, using the older configuration
     * @return OAIRepository
     */
    public OAIRepository createOAIRepository(String repoId) {
    	if ( !Utilities.checkString(repoId) ) {
  			repoId = "default";
    	}
    	String[] args = {repoId};
    	LoggingUtils.logDebug(this._logger, SDXExceptionCode.OAI_REPOSITORY_CREATE_NEW_INSTANCE,args);
    	OAIRepository ret = null;
    	Hashtable _h = null;
    	try{
    		if ( "default".equals(repoId) && this.oaiRepositories.get( repoId ) == null ) {
    			// pas d'enrepot oai par defaut, s'il n'y en a qu'un on le prend
    			_h = (Hashtable) this.oaiRepositories.elements().nextElement();
    		}
    		else {
    			_h = (Hashtable) this.oaiRepositories.get(repoId);
    		}
    		configureOAIRepository( (Configuration) _h.get("configuration") );
    		ret = this.oaiRepo;
    	} catch (ConfigurationException _ce){
    		new ConfigurationException(_ce.getMessage(), _ce);
    	} catch(Exception e){
    		if(this.oaiRepositories==null || this.oaiRepositories.isEmpty()){
    			String[] args2 = {repoId,this._id};
    			new SDXException(SDXExceptionCode.OAI_NO_OAI_REPOSITORY_AVAILABLE, args);
    		} else if(this.oaiRepositories.get(repoId)==null){
    			String[] args2 = {repoId,this._id};
    			new SDXException(SDXExceptionCode.OAI_NO_OAI_REPOSITORY_ID_AVAILABLE, args);
    		}
    	} finally {
    		_h = null;
    		repoId = null;
    	}
    	return ret;
    }
    
    /**Creates the default OAIRepository for the documentbase, using the older configuration
     * @see #createOAIRepository(String)
     */
    public OAIRepository createOAIRepository() {
    	return createOAIRepository("default");
    }

    /**Creates the OAIRepository for the documentbase
     * Configures an OAIRespository based on the configuration that must
     * start with an element &lt;oai-repository&gt;
     * @param configuration	The configuration
     * @return OAIRepository
     */
    public OAIRepository createOAIRepository(Configuration configuration) {
    	OAIRepository ret = null;
    	try{
    		configureOAIRepository(configuration);
    		ret = this.oaiRepo;
    	} catch (ConfigurationException _ce){
			new ConfigurationException(_ce.getMessage(), _ce);
		}
    	return ret;
    }

    /** Configures on or more OAI repositories.
     * @param configuration
     * @throws ConfigurationException
     */
    protected void configureOAIRepositories(Configuration configuration)
    	throws ConfigurationException 
	{
    	Configuration[] _confs = configuration.getChildren(DocumentBase.ConfigurationNode.OAI_REPOSITORY);
    	if (_confs != null) {
    		int l = _confs.length;
    		if ( l>0 ) {
    			LoggingUtils.logDebug(_logger, "Configuring "+l+" OAI repositories for the Lucene document base "+this._id);
	    		if(this.oaiRepositories==null){
	    			this.oaiRepositories = new Hashtable( l );
	    		}
	    		Configuration _conf;
	            for (int x = 0; x < l; x++) {
	            	_conf = _confs[x];
	                if (_conf != null) {
	                	configureOAIRepository(_conf);
	                }
	            }
	            _conf = null;
	            LoggingUtils.logDebug(_logger, l+" OAI repositories configured for the Lucene document base "+this._id);
    		}
    	}
    	_confs = null;
    }

    /** Configures an OAIRespository
     * Configures an OAIRespository based on the configuration element &lt;oai-repository&gt;
     * @param configuration	The configuration
     * @see fr.gouv.culture.sdx.documentbase.SDXDocumentBase#configureOAIRepository(org.apache.avalon.framework.configuration.Configuration)
     */
    /* TODO (MP) : On configure toujours les entrepots OAI eu deploiment de la 
     * base de document. On ne les utilise pas a ce moment la. Lorsque la base
     * de document recoit une requete pour l'un de ses entrepots OAI, l'XSP qui
     * commande les requetes (cf. $webapp/sdx/oai/oai.xsp) cree un nouvel objet
     * OAIRepository dans le cas ou la requete ne possede pas de resumptionToken.
     * Ceci signifie clairement que les entrepots OAI configurer au deploiment
     * ne servent jamais. Faut-il les supprimer apres la configuration initiale ?
     * Le fait de conserver la configuration initiale permet d'avertir les eventuels
     * problemes de configuration des le deploiment. C'est une bonne chose.
     * [2009-02-04]
     */
    protected void configureOAIRepository(Configuration configuration) 
    	throws ConfigurationException 
	{
    	int nb = this.oaiRepositories.size() + 1;
    	Configuration oaiRepoConf = (configuration.getName().equals(DocumentBase.ConfigurationNode.OAI_REPOSITORY)) ? configuration : null;
        if (oaiRepoConf != null) {
        	LoggingUtils.logDebug(_logger, "Configuring the "+nb+"e OAI repository for the Lucene document base "+this._id);
            LuceneDocumentBaseOAIRepository luceneOaiRepo = new LuceneDocumentBaseOAIRepository(this);
            try {
                luceneOaiRepo.enableLogging(super.getLog());
                luceneOaiRepo.contextualize(super.getContext());
                luceneOaiRepo.service(super.getServiceManager());
                luceneOaiRepo.configure(oaiRepoConf);
                addOAIRepository(luceneOaiRepo, configuration, luceneOaiRepo.getId());
                LoggingUtils.logDebug(this._logger, "Adding "+luceneOaiRepo.getId()+" Lucene OAI repository");
            } catch (ServiceException e) {
                throw new ConfigurationException(e.getMessage(), e);
            } catch (ContextException e) {
                throw new ConfigurationException(e.getMessage(), e);
            }
            LoggingUtils.logDebug(_logger, "OAI repository configured.");
        }
    }

    /**
     * Configures the OAI harverster of this Lucene document base.
     * @params configuration	The configuration object.
     */
    protected void configureOAIHarvester(Configuration configuration) throws ConfigurationException {
        Configuration oaiHarvConf = configuration.getChild(DocumentBase.ConfigurationNode.OAI_HARVESTER, false);
        if (oaiHarvConf != null) {
            LuceneDocumentBaseOAIHarvester luceneOAIHarvester = new LuceneDocumentBaseOAIHarvester(this);
            try {
                luceneOAIHarvester.enableLogging(super.getLog());
                luceneOAIHarvester.contextualize(super.getContext());
                luceneOAIHarvester.service(super.getServiceManager());
                luceneOAIHarvester.configure(oaiHarvConf);
                super._oaiHarv = luceneOAIHarvester;
            } catch (ServiceException e) {
                throw new ConfigurationException(e.getMessage(), e);
            } catch (ContextException e) {
                throw new ConfigurationException(e.getMessage(), e);
            }
        }
    }

    /** Adds one or more indexables documents to the search index of Lucene.
     * 
     * <p>After adding the document to the search index, this method recycles the 
     * Lucene searcher if :
     * <ol>
     * <li>The auto-optimize option is false</li>
     * <li>More than one documents are added to the search index</li>
     * </ol>
     * </p>
     * @see fr.gouv.culture.sdx.documentbase.SDXDocumentBase#index(fr.gouv.culture.sdx.document.IndexableDocument[], fr.gouv.culture.sdx.repository.Repository, fr.gouv.culture.sdx.documentbase.IndexParameters, org.xml.sax.ContentHandler)  
     */
    public synchronized void index(IndexableDocument[] docs, Repository repository, IndexParameters params, ContentHandler handler) throws SDXException, SAXException, ProcessingException {
    	super.index(docs, repository, params, handler);
    	if( !autoOptimize || docs.length <= 1 ) getLuceneIndex().recycleLuceneIndexSearcher();
    }

    /**Deletes documents to this base.
     * <p>Deletes one or more documents to this LuceneDocumentBase and recycle
     * Lucene searcher if deletes only one document or the LuceneDocumentBase is
     * not autoOptimize.</p>
     *
     * @param docs		The document to add and to index.
     * @param handler	A content handler to feed with information.
     * @see SDXDocumentBase#delete(Document, ContentHandler)
     */
    public synchronized void delete(Document[] docs, ContentHandler handler) throws SDXException, SAXException, ProcessingException {
        try {
            super.delete(docs, handler);
        } finally {
        	/* if delete just one document or the LuceneDocumentBase is not 
        	 * autoOptimize : recycle the searcher and change the date of 
        	 * the last-modification-timestamp file*/
            if ( docs.length <= 1 || !autoOptimize ) {
            	getLuceneIndex().recycleSearcherReader();
            	this.luceneActiveIndex.indexModified(); // TODO (MP): Cause des problemes dans certains environnements : on active la modification de la date de modification de l'index pendant une indexation ce qui entraine le recyclage des lecteurs/chercheurs d'index Lucene pendant le processus d'indexation qui en meme temps optimise ses index a chaque passe du batch ! La solution est soit de ne pas modifier la date d'index apres cette suppression, soit de ne pas optimiser pendant le batch, seulement a la fin.
            }
        }
    }

    /**Sets the default pipeline parameters and ensures the params have a pipeline
     *
     * @param params The params object provided by the user at indexation time
     */
    protected IndexParameters setBaseParameters(IndexParameters params) {
        //we check if we have lucene params and set them to the index
        if (params != null && params instanceof LuceneIndexParameters)
            setSearchIndexParameters((LuceneIndexParameters) params);

        return super.setBaseParameters(params);
    }

    /* (non-Javadoc)
     * @see fr.gouv.culture.sdx.documentbase.SDXDocumentBase#getXMLFieldList()
     */
    public HashMap getXMLFieldList() {
        if(this._xmlFieldList == null){
            this._xmlFieldList = new HashMap();
            for(Enumeration elem = this._fieldList.getFields(); elem.hasMoreElements();){
                fr.gouv.culture.sdx.search.lucene.Field f = (fr.gouv.culture.sdx.search.lucene.Field)elem.nextElement();
                if(f.getFieldType() == fr.gouv.culture.sdx.search.lucene.Field.XML)
                    this._xmlFieldList.put(f.getCode(), f);
            }
        }
        return this._xmlFieldList;
    }


    /** Gets the Index object for indexing and searching.
     * @return The LuceneIndex object.
     */
    public Index getIndex() {
        return luceneActiveIndex; //TODO : this method seems to be only used by sdx.xsl -pb
        //TODO : name this method getSearchIndex-) ? -pb
        /*A: we need this to be generic enough to support other types of search indexes,
        *ie remote indicies or any other index which someone may design in the future-rbp
        */
    }
    public LuceneIndex getLuceneIndex()
    {
        return luceneActiveIndex;
    }

    //TODO : possibly move this method to the LuceneIndex class -pb
    /**Sets the search index parameters for indexation performance
     *
     * @param params The lucene specific params to user
     */
    protected void setSearchIndexParameters(LuceneIndexParameters params) {
        luceneActiveIndex.setParameters(params);
    }


    //TODO : possibly move this method to the LuceneIndex class -pb
    /**Writes a document to the search index
     *
     * @param indexationDoc The Document to add
     * @param batchIndex
     * @throws SDXException
     */
    protected void addToSearchIndex(Object indexationDoc, boolean batchIndex) throws SDXException {
        //writing the Lucene index document to the search index
        luceneActiveIndex.writeDocument((org.apache.lucene.document.Document) indexationDoc, batchIndex, this.autoOptimize);
    }

    //TODO : possibly move this method to the LuceneIndex class -pb
    protected void deleteFromSearchIndex(String docId) throws SDXException {
        //TODO : transfer a batch control here -pb

        int i = luceneSearchIndexList.size()-1;
        LuceneIndex currentIndex;

        // This loop only optimize the sub-indexes and not the active and current ones (done after)
        while(i >= 0 && ((currentIndex = (LuceneIndex) luceneSearchIndexList.get(i)) != null))
        {
            currentIndex.deleteDocument(docId);
            i--;
        }
    }


    //TODO : possibly move this method to the LuceneIndex class -pb
    protected void compactSearchIndex() throws SDXException {
        luceneActiveIndex.mergeCurrentBatch(this.autoOptimize);
    }


    //TODO : possibly move this method to the LuceneIndex class -pb
    //TODOLuceneSpecific: move to lucene index and let the document base add the few system fields
    /*TODO, this could probably be refactored to return a generic object like Parameters and moved to the parent class
    *the sub-class would then call this method and use the generic object to build it's particular object for indexation*/
    protected Object getIndexationDocument(IndexableDocument doc, String storeDocId, String repoId, IndexParameters params) throws SDXException {

	    	//if (_indexationLoggerLevel <= Priority.DEBUG_INT) {
				//	_indexationLogger.debug("Génération du document Lucene : "+storeDocId);
				//}
        org.apache.lucene.document.Document lDoc = new org.apache.lucene.document.Document();
        float docBoost = doc.getBoost();
        lDoc.setBoost(docBoost);

        //getting Lucene fields
        Enumeration fieldVals = doc.getFieldValues();
        while (fieldVals != null && fieldVals.hasMoreElements()) {
            IndexableFieldProperty prop = (IndexableFieldProperty) fieldVals.nextElement();
            if (prop != null) {
                String fieldName = prop.getName();
                String[] fieldValues = prop.getValues();
                float fieldBoost = prop.getBoost();
                fr.gouv.culture.sdx.search.lucene.Field sdxF = _fieldList.getField(fieldName);
                for (int i = 0; i < fieldValues.length; i++) {
                    String fieldValue = fieldValues[i];
                    if (sdxF != null) {
                        Field f = sdxF.getLuceneField(fieldValue);
                        if (f != null) {
                            f.setBoost(fieldBoost);
                            lDoc.add(f);
                        }
                    } else {
                        //TODO: what do we do with non defined fields??? a policy at DocumentBase configuration Time For now we will have a NPE...maybe just index and tokenize no store!
                        //else we add an unindexed field
                        if (Utilities.checkString(fieldValue)) {
                        	// MAJ Lucene 2.1.0
                            //lDoc.add(Field.UnIndexed(fieldName, fieldValue));
                        	lDoc.add( new Field(fieldName, fieldValue, Field.Store.YES, Field.Index.NO, Field.TermVector.NO) );
                        	String[] args = {fieldName,fieldValue,this._id};
                            LoggingUtils.logWarn(this._logger, SDXExceptionCode.WARN_UNDECLARED_FIELD, args);
                        }
                    }
                }
            }
            // else  return null //TODOLogging:log.warn or info? the unknown fields
        }

        //if we dont have a document id after the parse or for the attached document, we have a problem and need throw an error
        Utilities.checkDocument(super.getLog(), doc);

        //adding system metadata
        Field fId = null;
        if (Utilities.checkString(storeDocId))
        	// MAJ Lucene 2.1.0
            //fId = Field.Keyword(INTERNAL_FIELD_NAME_SDXDOCID, storeDocId);
        	fId = new Field(INTERNAL_FIELD_NAME_SDXDOCID, storeDocId, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO);
        if (fId != null)
            lDoc.add(fId);

        // MAJ Lucene 2.1.0
        //Field fId2 = Field.Keyword(INTERNAL_FIELD_NAME_SDXALL, INTERNAL_SDXALL_FIELD_VALUE);
        Field fId2 = new Field(INTERNAL_FIELD_NAME_SDXALL, INTERNAL_SDXALL_FIELD_VALUE, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO);
        if (fId2 != null)
            lDoc.add(fId2);

        Field fId3 = null;
        String docbaseId = this.getId();
        if (Utilities.checkString(docbaseId))
        	// MAJ Lucene 2.1.0
            //fId3 = Field.Keyword(INTERNAL_FIELD_NAME_SDXDBID, docbaseId);
        	fId3 = new Field(INTERNAL_FIELD_NAME_SDXDBID, docbaseId, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO);
        if (fId3 != null)
            lDoc.add(fId3);

        //Warning : what if this DB is shared between applications ?
        Field fId4 = null;
        String appId = Utilities.getStringFromContext(ContextKeys.SDX.Application.ID, super.getContext());
        if (Utilities.checkString(appId))
        	// MAJ Lucene 2.1.0
            //fId4 = Field.Keyword(INTERNAL_FIELD_NAME_SDXAPPID, appId);
        	fId4 = new Field(INTERNAL_FIELD_NAME_SDXAPPID, appId, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO);
        if (fId4 != null)
            lDoc.add(fId4);
        Field fId5 = null;
        String doctype = doc.getDocType();
        if (Utilities.checkString(doctype))
        	// MAJ Lucene 2.1.0
            //fId5 = Field.Keyword(INTERNAL_FIELD_NAME_SDXDOCTYPE, doctype);
        	fId5 = new Field(INTERNAL_FIELD_NAME_SDXDOCTYPE, doctype, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO);
        if (fId5 != null)
            lDoc.add(fId5);

        Field fId6 = null;
        long modDate = fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date().getTime();
        //getting the documents indexation date parameter
        modDate = Long.parseLong(params.getPipelineParams().getParameter(SDX_DATE_MILLISECONDS, String.valueOf(modDate)));
        // MAJ Lucene 2.1.0
        //fId6 = Field.Keyword(INTERNAL_FIELD_NAME_SDXMODDATE, DateField.timeToString(modDate));
        fId6 = new Field(INTERNAL_FIELD_NAME_SDXMODDATE, DateField.timeToString(modDate), Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO);
        if (fId6 != null)
            lDoc.add(fId6);


        Field fId7 = null;
        String length = Integer.toString(doc.getLength());
        if (Utilities.checkString(length))
        	// MAJ Lucene 2.1.0
            //fId7 = Field.Keyword(INTERNAL_FIELD_NAME_SDXCONTENTLENGTH, length);
        	fId7 = new Field(INTERNAL_FIELD_NAME_SDXCONTENTLENGTH, length, Field.Store.YES, Field.Index.NO, Field.TermVector.NO);
        if (fId7 != null)
            lDoc.add(fId7);

        Field fId8 = null;
        if (Utilities.checkString(repoId))
        	// MAJ Lucene 2.1.0
            //fId8 = Field.Keyword(INTERNAL_FIELD_NAME_SDXREPOID, repoId);
        	fId8 =  new Field(INTERNAL_FIELD_NAME_SDXREPOID, repoId, Field.Store.YES, Field.Index.UN_TOKENIZED, Field.TermVector.NO);
        if (fId8 != null)
            lDoc.add(fId8);

        // Any additional fields handled by the document itself (for instance OAI)
        doc.addAdditionalSystemFields(lDoc);

        return lDoc;
    }


    /**
     * Returns the last modification date of the Lucene search index.
     */
    public Date lastModificationDate() {
        return luceneActiveIndex.getLastModificationDate();
    }

    /**
     * Returns the creation date of the Lucene search index.
     */
    public Date creationDate() {
        return luceneActiveIndex.getCreationDate();
    }

    public void init() throws SDXException {
        super.init();
        //initializing the search index
        this.luceneActiveIndex.init();
    }

    protected boolean initToSax(){
        if(!super.initToSax())
            return false;
        else{
            try{
                if((this.luceneActiveIndex != null) && (this.luceneActiveIndex.getQueryParser() != null))
                    this._xmlizable_objects.put("Query_Parser", this.luceneActiveIndex.getQueryParser().getClass().getName());
                else
                    this._xmlizable_objects.put("Query_Parser", "none");
            }catch(SDXException e){
            }catch(IOException e){
            }
            this._xmlizable_objects.put(Utilities.getElementName(FieldList.CLASS_NAME_SUFFIX),this._fieldList);
            String prefix=Utilities.getElementName(LuceneIndex.CLASS_NAME_SUFFIX) + "_";
            if(this.luceneActiveIndex != null){
                //id always empty?
                //this._xmlizable_objects.put(prefix+"id",this.luceneActiveIndex.getId());
                this._xmlizable_objects.put(prefix+"encoding",this.luceneActiveIndex.getEncoding());
                this._xmlizable_objects.put(prefix+"xml-lang",this.luceneActiveIndex.getXmlLang());
                this._xmlizable_objects.put(prefix+"document_count",String.valueOf(this.luceneActiveIndex.size()));
                this._xmlizable_objects.put(prefix+"path",this.luceneActiveIndex.getIndexPath());
                this._xmlizable_objects.put(prefix+"engine","lucene");
                this._xmlizable_objects.put(prefix+"use_compound_files",String.valueOf(this.luceneActiveIndex.isUsingCompoundFiles()));
                this._xmlizable_objects.put(prefix+"is_optimized",String.valueOf(this.isIndexOptimized()));
                this._xmlizable_objects.put(prefix+"useMetadata",String.valueOf(isUseMetadata()));
                if(this.creationDate() != null)
                    this._xmlizable_objects.put(prefix+"creation_date",this.creationDate().toString());
                if(this.lastModificationDate() != null)
                    this._xmlizable_objects.put(prefix+"last_modification_date",this.lastModificationDate().toString());


                // get the index directory and loop through index files
                String iPath = this.luceneActiveIndex.getIndexPath();
                File iDir = new File(iPath);
                if(iDir != null)
                {
                    File[] iFiles = iDir.listFiles(indexContentFilter);
                    long iFullSize = 0;
                    int iFileNumber = 0;
                    // compute full index size and number of index files
                    while(iFiles != null && iFileNumber < iFiles.length)
                    {
                        iFullSize += iFiles[iFileNumber].length();
                        iFileNumber++;
                    }
                    this._xmlizable_objects.put(prefix+"full_size", String.valueOf(iFullSize/1024) + " Ko");
                    this._xmlizable_objects.put(prefix+"count_files", String.valueOf(iFileNumber));
                }
            }

            return true;
        }
    }

    /**Init the LinkedHashMap _xmlizable_volatile_objects with the objects in 
     * order to describ them in XML.
     * <p>Some objects need to be refresh each time a toSAX is called.</p>
     * */
    protected void initVolatileObjectsToSax() {
        super.initVolatileObjectsToSax();

        String prefix = Utilities.getElementName(LuceneIndex.CLASS_NAME_SUFFIX) + "_";

        if(this.luceneActiveIndex != null)
        {
            this._xmlizable_objects.put(prefix+"document_count",String.valueOf(this.luceneActiveIndex.size()));
            this._xmlizable_objects.put(prefix+"use_compound_files",String.valueOf(this.luceneActiveIndex.isUsingCompoundFiles()));
            this._xmlizable_objects.put(prefix+"is_optimized",String.valueOf(this.isIndexOptimized()));
            this._xmlizable_objects.put(prefix+"useMetadata",String.valueOf(isUseMetadata()));
            if(this.creationDate() != null)
                this._xmlizable_objects.put(prefix+"creation_date",this.creationDate().toString());
            if(this.lastModificationDate() != null)
                this._xmlizable_objects.put(prefix+"last_modification_date",this.lastModificationDate().toString());
            this._xmlizable_objects.put(prefix+"split_size", String.valueOf(this.splitSize/1024) + " Ko");
            this._xmlizable_objects.put(prefix+"split_doc", String.valueOf(this.splitDoc));

            // get the index directory and loop through index files
            File iDir = new File(baseIndexDir);
            File[] iSubDirs = iDir.listFiles(indexDirFilter);

            this._xmlizable_objects.put(prefix+"full_size", String.valueOf(iDir.length()) + " Ko");
            this._xmlizable_objects.put(prefix+"count_files", String.valueOf(iSubDirs.length));
        }
    }

    /**
     * Process an optimization of the indexes and repositories and system databases
     */
    public synchronized void optimize()
    {
        try {

//        	optimizing all indexes
    		LuceneIndex currentIndex;
    		int i = luceneSearchIndexList.size() - 1;
    		while(i > 0 && ((currentIndex = (LuceneIndex) luceneSearchIndexList.get(i)) != null))
    		{
    			currentIndex.optimize();
    			i--;
    		}

    		// delete and recreate current index after the main index optimization
    		try {
    			luceneCurrentIndex.freeResources();
    			((LuceneIndex) luceneSearchIndexList.get(1)).freeResources();
    			((LuceneIndex) luceneSearchIndexList.get(i)).optimize();
    		} catch (IOException e) {
    			LoggingUtils.logException(getLog(), e);
    		}


            renewKeyIndex();

            //optimizing any repositories that are not optimized
            this.optimizeRepositories();
            //release any pooled connections
            this.releasePooledRepositoryConnections();
            //optimizing our internal datastructure
            //TODO: use inner class when javac compiler bug is fixed-rbp
            //super.lookupDatabase.optimizeDatabase();
            optimizeDatabase();
        } catch(SDXException e) {
            LoggingUtils.logException(super.getLog(), e);
        }
    }

    /** Merges a batch of documents
     * <p>
     * Merges a batch of documents (in memory) into the physical index on the 
     * file system and optimize this one if necessary (depends of the 
     * <code>autoOptimize</code> attribute for the current Document Base).
     * </p>
     */
    public synchronized void mergeCurrentBatch(){
    	try {
    		//merge all indexes
    		LuceneIndex currentIndex;
    		for( int i = 0; i < luceneSearchIndexList.size(); i++ ){
    			currentIndex = (LuceneIndex) luceneSearchIndexList.get(i);
    			if(currentIndex != null) currentIndex.mergeCurrentBatch(this.autoOptimize);
    		}

    		renewKeyIndex();

    		//optimizing any repositories that are not optimized
  			this.optimizeRepositories();
    		//release any pooled connections
    		this.releasePooledRepositoryConnections();
  			optimizeDatabase();
    	} catch(SDXException e) {
    		// TODO (MP) : meilleure gestion d'erreur.
    		this._logger.error(null,e);
    	}
    }

    /**
     * Modifies the last modfication timestamp file*/
    public void indexModified(){
    	try {
			luceneActiveIndex.indexModified();
		} catch (SDXException e) {
			LoggingUtils.logException(super.getLog(), e);
		}
    }

    /** Splits current index
     * <p>Splits the current big index into 2 smaller one</p>
     * @throws IOException, SDXException
     */
    public void splitIndex(boolean currentIndex) 
    throws IOException, SDXException
    {
        // choose the correct index to split
        LuceneIndex index;
        if(currentIndex) index = luceneCurrentIndex;
        else index = luceneActiveIndex;

        if(index == null) return;

        // free ressources to allow the copy
        index.freeResources();
        // add a subIndex with the data coming from an other index
        addSubIndex(index);

        // refresh the two mains index
        renewKeyIndex();
    }


    /**
     * Initializes the index vector
     * <p>Initializes the index vector by searching all sub index in it's directory
     * <br/>
     * NB : working as intended.</p>
     * @throws ConfigurationException
     */
    protected void initializeVectorizedIndex() 
    throws ConfigurationException
    {
        // clear the list
        luceneSearchIndexList.clear();

        // add the two "key" indexes : main and current
        luceneSearchIndexList.add(luceneActiveIndex);
        luceneSearchIndexList.add(luceneCurrentIndex);

        // get the root index path in order to loop on every subIndex
        File indexDirectory = new File(baseIndexDir);

        this.subIndexCount = 0;
        int counter = 0;
        int maxCount = 0;

        // list all sub-index directories (see filter for more details)
        File[] iSubDir = indexDirectory.listFiles(indexDirFilter);

        // links index to sub-dirs (handle creation and configuration)
        while(iSubDir != null && counter < iSubDir.length)
        {
            File tempDir = iSubDir[counter];
            // set the right subIndexCount for addSubIndex method
            this.subIndexCount = Integer.parseInt(tempDir.getName());
            this.addSubIndex();
            // memorize the highest subIndexCount for setting it to subIndexCount after loop's end
            // (in case of deleted directories)
            if(Integer.parseInt(tempDir.getName()) > maxCount)
                maxCount = Integer.parseInt(tempDir.getName());
            counter++;
        }
        // set the subIndexCount to the last subIndex number
        this.subIndexCount = maxCount;
    }

    /**
     * Adds a splitted sub-index and update configuration aftermath
     * @throws SDXException If it's impossible to configure or initialize the sub-index to add.
     */
    protected void addSubIndex()
    throws ConfigurationException
    {
        File subDir = new File(baseIndexDir + File.separatorChar + getFormatedSubIndexId(this.subIndexCount));
        if(subDir != null) {
            try {
                // Configure the new index
                LuceneIndex subIndex = new LuceneIndex(luceneActiveIndex, subDir);
                subIndex = (LuceneIndex) super.setUpSdxObject(subIndex, super._configuration);
                MetaAnalyzer mAnalyzer = new MetaAnalyzer();
                mAnalyzer.enableLogging(super.getLog());
                mAnalyzer.setUp(_fieldList);
                subIndex.setMetaAnalyzer(mAnalyzer);
                subIndex.init();
                // Add the index to the list
                this.luceneSearchIndexList.add(subIndex);
                subDir = null;
            } catch (SDXException sdxe) {
            	String[] args = new String[1];
                args[0] = sdxe.getMessage();
                new SDXException(super.getLog(), SDXExceptionCode.ERROR_INITIALIZING_LUCENE_INDEX, args, sdxe);
                throw new ConfigurationException(args[0], super._configuration, sdxe);
            } catch(ConfigurationException e) {
            	String[] args = new String[1];
                args[0] = e.getMessage();
                new SDXException(super.getLog(), SDXExceptionCode.ERROR_CONFIGURE_LUCENE_INDEX, args, e);
                throw e;
            }
        }
        // increment subIndexCount
        this.subIndexCount++;
    }

    /**
     * Remove a splitted sub-index and update configuration aftermath
     * Currently of no use as there is no plan to do so, just here as a reminder for future functionnalities
     */
    protected void removeSubIndex()
    {

    }

    /**
     * Tests splitting conditions
     * <p>Returns true when splitting condition are reached.
     * If so, should be followed by a splitIndex() call.
     * Controls order:
     * <ol>
     * <li>size of the index</li>
     * <li>number of documents</li>
     * </ol>
     * </p>
     * @param currentIndex boolean to indicate the test concerns the current 
     *                     index (<code>true</code>) or the active one (<code>false</code>)
     * @return <code>true</code> when splitting condition are reached,  
     *         <code>false</code> otherwise.
     * @throws SDXException
     */
    public boolean splitCheck(boolean currentIndex) 
    throws SDXException
    {
        // if no split needed, then dont check anything else
        if(!splitActive) return false;

        // choose the correct index to split
        LuceneIndex index;
        if(currentIndex) index = luceneCurrentIndex;
        else index = luceneActiveIndex;

        if(index == null) return false;

        // check if size should be taken into account
        if(splitSize > 0)
            if(getIndexSize(index) > splitSize) return true;

        // check if document number should be taken into account
        if(splitDoc > 0)
        {
            index.init();
            if(index.size() > splitDoc) return true;
        }

        return false;
    }

    /**
     * Returns the index size
     * @param index LuceneIndex
     * @return long index size as long
     */
    protected long getIndexSize(LuceneIndex index)
    {
        if(index == null) return -1;
        long iFullSize=0;
        // get the index directory and loop through index files
        String iPath = index.getIndexPath();

        File iDir = new File(iPath);
        if(iDir != null)
        {
            File[] iFiles = iDir.listFiles(indexContentFilter);
            int iFileNumber = 0;
            // compute full index size and number of index files
            while(iFiles != null && iFileNumber < iFiles.length)
            {
                iFullSize += iFiles[iFileNumber].length();
                iFileNumber++;
            }
            iFiles = null;
        }
        iDir = null;
        return iFullSize;
    }


    /**
     * Filter for index directories
     */
    FilenameFilter indexDirFilter = new FilenameFilter() {
        public boolean accept(File dir, String name) {
            return Utilities.isNameMatchIndexFiles(name);
        }
    };

    /**
     * Filter for index content (all but directories)
     */
    FilenameFilter indexContentFilter = new FilenameFilter() {
        public boolean accept(File dir, String name) {
            return !Utilities.isNameMatchIndexFiles(name) && !name.startsWith("current");
        }
    };

    /**
     * Returns the Lucene index searcher
     *  <p>Returns the index searcher for all this document base indexes.</p>
     *  @return Searcher
     *  @throws SDXException If it's not possible to build MultiSearcher.
     *  @see ParallelMultiSearcher
     */
    public Searcher getSearcher() 
    throws SDXException
    {
        try {
            Searchable[] searchers = new Searcher[luceneSearchIndexList.size()];
            for (int i = 0; i < luceneSearchIndexList.size(); i++) {
                LuceneIndex index = (LuceneIndex) luceneSearchIndexList.elementAt(i);
                searchers[i] = index.getSearcher();
            }
            return new ParallelMultiSearcher(searchers);
        } catch (IOException e) {
            //can't build multisearcher
            String[] args = new String[1];
            args[0] = e.getMessage();
            throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_BUILD_MULTISEARCHER, args, e);
        }
    }

    /**
     * Return the Lucene index reader
     *  <p>Returns the index reader for all this document base indexes.</p>
     *  @return IndexReader
     *  @throws SDXException If it's not possible to build MultiReader.
     *  @see MultiReader
     */
    public IndexReader getIndexReader() 
    throws SDXException
    {
        try {
            IndexReader[] readers = new IndexReader[luceneSearchIndexList.size()];
            for (int i = 0; i < luceneSearchIndexList.size(); i++)
                readers[i] = ((LuceneIndex) luceneSearchIndexList.get(i)).getReader();
            return new MultiReader(readers);
        } catch (IOException e) {
            //can't build multisearcher
            String[] args = new String[1];
            args[0] = e.getMessage();
            throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_BUILD_MULTIREADER, args, e);
        }
    }


    /**
     * Gets the formated sub-index number (for directories name)
     * @param subIndexNumber int representing the number of the sub-index
     * @return sub-index number formatted as a String
     */
    protected String getFormatedSubIndexId(int subIndexNumber)
    {
        DecimalFormat df = new DecimalFormat("0000");
        return df.format(subIndexNumber);
    }

    /**
     * Adds a splitted sub-index and update configuration aftermath
     * @param index LuceneIndex
     * @throws SDXException If nt's not possible to configure and initialize th sub-index.
     */
    protected void addSubIndex(LuceneIndex index)
    throws SDXException
    {
        File subDir = new File(baseIndexDir + File.separatorChar + getFormatedSubIndexId(subIndexCount));
        if(!subDir.exists())
        {
            try {
                // use the "safe copy" to do things fast and secured
                Utilities.safeCopy(new File(index.getIndexPath()), subDir);
                // Configure the new index
                LuceneIndex subIndex = new LuceneIndex(index, subDir);
                subIndex = (LuceneIndex) super.setUpSdxObject(subIndex, super._configuration);
                MetaAnalyzer mAnalyzer = new MetaAnalyzer();
                mAnalyzer.enableLogging(super.getLog());
                mAnalyzer.setUp(_fieldList);
                subIndex.setMetaAnalyzer(mAnalyzer);
                subIndex.init();
                // Add the index to the list
                luceneSearchIndexList.add(subIndex);
            } catch (SDXException e) {
            	String[] args = new String[1];
                args[0] = e.getMessage();
                new SDXException(super.getLog(), SDXExceptionCode.ERROR_ADDING_LUCENE_SUB_INDEX, args, e);
            } catch(ConfigurationException e) {
            	String[] args = new String[1];
                args[0] = e.getMessage();
                new SDXException(super.getLog(), SDXExceptionCode.ERROR_CONFIGURE_LUCENE_INDEX, args, e);
            }
        }

        // increment subIndexCount
        this.subIndexCount++;
    }

    /**
     * Refreshes data for the main and current index
     * @throws SDXException If it's impossible to freeing resources or 
     *                      initializing Lucene index.
     */
    protected void renewKeyIndex()
    throws SDXException
    {
        try {
            this.luceneActiveIndex.freeResources();
            this.luceneActiveIndex.init();
            ((LuceneIndex) this.luceneSearchIndexList.get(0)).freeResources();
            ((LuceneIndex) this.luceneSearchIndexList.get(0)).init();
            this.luceneCurrentIndex.freeResources();
            this.luceneCurrentIndex.init();
            ((LuceneIndex) this.luceneSearchIndexList.get(1)).freeResources();
            ((LuceneIndex) this.luceneSearchIndexList.get(1)).init();
        } catch (IOException e) {
        	String[] args = new String[1];
            args[0] = e.getMessage();
            throw new SDXException(this._logger, SDXExceptionCode.ERROR_FREEING_LUCENE_RESOURCES, args, e);
        } catch (SDXException sdxe) {
        	String[] args = new String[1];
            args[0] = sdxe.getMessage();
        	throw new SDXException(this._logger, SDXExceptionCode.ERROR_INITIALIZING_LUCENE_INDEX, args, sdxe);
        }
    }

    /** Saves the DocumentBase data objects
     * @param save_config SaveParameters
     * @throws SDXException
     * @see fr.gouv.culture.sdx.utils.save.Saveable#backup(fr.gouv.culture.sdx.utils.save.SaveParameters)
     */
    public void backup(SaveParameters save_config) 
    throws SDXException {
        super.backup(save_config);
        if(save_config != null)
            if(save_config.isAllElementSelected())
            {
                save_config.setAttribute(Node.Name.TYPE,"LUCENE");
            }
    }

    /**
     * Save the indexes files
     */
    protected void backupIndexes(SaveParameters save_config) throws SDXException
    {
        if(save_config != null)
            if(save_config.isAllElementSelected())
            {
                if(luceneSearchIndexList != null && luceneSearchIndexList.size()>0)
                {
                    String indexes_path = Utilities.getElementName(Index.CLASS_NAME_SUFFIX)+"es";
                    File indexes_dir = new File(save_config.getStoreCompletePath() + File.separator + indexes_path);
                    if(!indexes_dir.exists())
                        indexes_dir.mkdir();

                    for(Enumeration _enum = this.luceneSearchIndexList.elements(); _enum.hasMoreElements();)
                    {
                        Index index = (Index)_enum.nextElement();
                        if(index != null)
                        {
                            //save each index
                            SaveParameters indexsave = new SaveParameters(Utilities.getElementName(Index.CLASS_NAME_SUFFIX),save_config, indexes_path);
                            if(index instanceof LuceneIndex)
                            {
                                ((LuceneIndex)index).backup(indexsave);
                            }
                        }
                    }
                }
            }
    }

    /**
     * Save the timestamp files
     */
    protected void backupTimeStamp(SaveParameters save_config) throws SDXException
    {
        if(save_config != null)
            if(save_config.isAllElementSelected())
            {
                save_config.savePathInConfig(Utilities.getElementName(Index.CLASS_NAME_SUFFIX)+"es");
                String save_dir = save_config.getStoreBasePath()+
                                save_config.getAttribute(Saveable.PATH_ATTRIB,"");

                File file=new File(this.baseIndexDir + File.separator + "creation-timestamp");
                if(file.exists())
                {
                    SaveParameters timestampsave = new SaveParameters("timestamp",save_config);
                    timestampsave.savePathInConfig(null);
                    timestampsave.setAttribute(Node.Name.ID,"creation");
                    File copy = new File(save_dir + File.separator + "creation-timestamp");
                    try{
                        copy.createNewFile();
                        copy.setLastModified(file.lastModified());
                    }catch(IOException e)
                    {
                        String[] args = new String[2];
                        args[0] = this.getId();
                        args[1] = e.getMessage();
                        throw new SDXException(getLog(), SDXExceptionCode.ERROR_BACKUP_DOCUMENTBASE, args, e);
                    }
                }

                file=new File(this.baseIndexDir + File.separator + "last-modification-timestamp");
                if(file.exists())
                {
                    SaveParameters timestampsave = new SaveParameters("timestamp",save_config);
                    timestampsave.setAttribute(Node.Name.ID,"last-modification");
                    timestampsave.savePathInConfig(null);
                    File copy = new File(save_dir + File.separator + "last-modification-timestamp");
                    try{
                        copy.createNewFile();
                        copy.setLastModified(file.lastModified());
                    }catch(IOException e)
                    {
                        String[] args = new String[2];
                        args[0] = this.getId();
                        args[1] = e.getMessage();
                        throw new SDXException(getLog(), SDXExceptionCode.ERROR_BACKUP_DOCUMENTBASE, args, e);
                    }
                }
            }
    }

    /** Restore the DocumentBase data objects
     * @see fr.gouv.culture.sdx.utils.save.Saveable#restore(fr.gouv.culture.sdx.utils.save.SaveParameters)
     */
    public void restore(SaveParameters save_config) throws SDXException {
        super.restore(save_config);
    }

    /**
     * Save the indexes files
     */
    protected void restoreIndexes(SaveParameters save_config) throws SDXException
    {
        if(save_config != null)
            if(save_config.isAllElementSelected())
            {
                if(luceneSearchIndexList != null && luceneSearchIndexList.size()>0)
                {
                    //String indexes_path = Utilities.getElementName(Index.CLASS_NAME_SUFFIX)+"es";

                    for(Enumeration _enum = this.luceneSearchIndexList.elements(); _enum.hasMoreElements();)
                    {
                        Index index = (Index)_enum.nextElement();
                        if(index != null)
                        {
                            if(index instanceof LuceneIndex)
                            {
                                //save each index
                                SaveParameters indexsave = save_config.getSaveParametersById(((LuceneIndex)index).getId());
                                ((LuceneIndex)index).restore(indexsave);
                            }
                        }
                    }
                }
            }
    }

    /**
     * Restore the timestamp files
     */
    protected void restoreTimeStamp(SaveParameters save_config) throws SDXException
    {
        if(save_config != null)
            if(save_config.isAllElementSelected())
            {
                String save_dir = save_config.getStoreCompletePath();
                SaveParameters timestampsave = save_config.getSaveParametersById("creation");
                if(timestampsave != null)
                {
                    File file=new File(save_dir + File.separator + "creation-timestamp");
                    if(!file.exists()){
                        String[] args = new String[2];
                        args[0] = this.getId();
                        args[1] = file.getAbsolutePath() + " not found.";
                        throw new SDXException(getLog(), SDXExceptionCode.ERROR_BACKUP_DOCUMENTBASE, args, null);
                    }

                    File copy = new File(this.baseIndexDir + File.separator + "creation-timestamp");
                    try{
                        if(!copy.exists())
                            copy.createNewFile();
                        copy.setLastModified(file.lastModified());
                    }catch(IOException e)
                    {
                        String[] args = new String[2];
                        args[0] = this.getId();
                        args[1] = e.getMessage();
                        throw new SDXException(getLog(), SDXExceptionCode.ERROR_BACKUP_DOCUMENTBASE, args, e);
                    }
                }

                timestampsave = save_config.getSaveParametersById("last-modification");
                if(timestampsave != null)
                {
                    File file=new File(save_dir + File.separator + "last-modification-timestamp");
                    if(!file.exists()){
                        String[] args = new String[2];
                        args[0] = this.getId();
                        args[1] = file.getAbsolutePath() + " not found.";
                        throw new SDXException(getLog(), SDXExceptionCode.ERROR_BACKUP_DOCUMENTBASE, args, null);
                    }

                    File copy = new File(this.baseIndexDir + File.separator + "last-modification-timestamp");
                    try{
                        if(!copy.exists())
                            copy.createNewFile();
                        copy.setLastModified(file.lastModified());
                    }catch(IOException e)
                    {
                        String[] args = new String[2];
                        args[0] = this.getId();
                        args[1] = e.getMessage();
                        throw new SDXException(getLog(), SDXExceptionCode.ERROR_BACKUP_DOCUMENTBASE, args, e);
                    }
                }
            }
    }

    /** Returns the number of documents in all Lucene sub indexes.
     * 
     * @return the number of document in all sub indexes
     * 
     * <br/>TODO - This needs to be periodically written to a .properties file
     * <br/>TODO -  we a configurable generic mechanism to save such information 
     * to a .properties file like certain queries, terms, etc. which should be 
     * updated after indexation/deletion
     * 
     */
    public int docCount(){
        int docCount = 0;
        for (int i = 0; i < luceneSearchIndexList.size(); i++) {
            LuceneIndex li = (LuceneIndex) luceneSearchIndexList.get(i);
            docCount+=li.docCount();
        }
        return docCount;
    }

    /**
	 * Merges a batch of documents (in memory) into the physical index on the file system.
	 * @deprecated This method is deprecated since SDX v. 2.3. Use mergeCurrentBatch() instead.
	 */
	public void mergeBatch() throws SDXException {
		mergeCurrentBatch();
	}
}