/*
Copyright (C) 2000-2010  Ministere de la culture et de la communication (France), AJLSM
See LICENCE file
 */
package fr.gouv.culture.sdx.documentbase;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.DateFormat;
import java.util.Date;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Hashtable;

import javax.xml.transform.OutputKeys;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.sax.SAXTransformerFactory;
import javax.xml.transform.sax.TransformerHandler;

import org.apache.avalon.framework.configuration.Configuration;
import org.apache.avalon.framework.configuration.ConfigurationException;
import org.apache.avalon.framework.context.Context;
import org.apache.avalon.framework.context.ContextException;
import org.apache.avalon.framework.logger.Logger;
import org.apache.avalon.framework.parameters.Parameters;
import org.apache.avalon.framework.service.ServiceException;
import org.apache.cocoon.ProcessingException;
import org.apache.cocoon.xml.XMLConsumer;
import org.apache.excalibur.xml.sax.SAXParser;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

import fr.gouv.culture.sdx.document.AbstractIndexableDocument;
import fr.gouv.culture.sdx.document.BinaryDocument;
import fr.gouv.culture.sdx.document.Document;
import fr.gouv.culture.sdx.document.HTMLDocument;
import fr.gouv.culture.sdx.document.IndexableDocument;
import fr.gouv.culture.sdx.document.OAIDocument;
import fr.gouv.culture.sdx.document.ParsableDocument;
import fr.gouv.culture.sdx.document.XMLDocument;
import fr.gouv.culture.sdx.exception.SDXException;
import fr.gouv.culture.sdx.exception.SDXExceptionCode;
import fr.gouv.culture.sdx.framework.Framework;
import fr.gouv.culture.sdx.repository.FSRepository;
import fr.gouv.culture.sdx.repository.Repository;
import fr.gouv.culture.sdx.repository.RepositoryConnection;
import fr.gouv.culture.sdx.repository.URLRepository;
import fr.gouv.culture.sdx.search.lucene.query.FieldQuery;
import fr.gouv.culture.sdx.search.lucene.query.SearchLocations;
import fr.gouv.culture.sdx.thesaurus.LuceneThesaurus;
import fr.gouv.culture.sdx.utils.Identifiable;
import fr.gouv.culture.sdx.utils.Utilities;
import fr.gouv.culture.sdx.utils.configuration.ConfigurationUtils;
import fr.gouv.culture.sdx.utils.constants.ContextKeys;
import fr.gouv.culture.sdx.utils.constants.Node;
import fr.gouv.culture.sdx.utils.database.Database;
import fr.gouv.culture.sdx.utils.database.DatabaseEntity;
import fr.gouv.culture.sdx.utils.logging.LoggingUtils;
import fr.gouv.culture.sdx.utils.logging.LoggingIndexation;
import fr.gouv.culture.sdx.utils.lucene.LuceneDataStore;
import fr.gouv.culture.sdx.utils.save.SaveParameters;
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.SimpleTimeScheduler;
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeScheduler;
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTrigger;
import fr.gouv.culture.util.apache.avalon.cornerstone.services.scheduler.TimeTriggerFactory;

/**
 * an abstract document base	class handling SDX based configurations
 */
public abstract class SDXDocumentBase extends AbstractDocumentBase implements SDXDocumentBaseTarget {

	/**Time scheduler for stored requests*/
	protected TimeScheduler scheduler = null;
	/** Whether the original documents should be stored or not. */
	protected boolean keepOriginalDocuments = true;
	/** The base index directory path */
	protected String baseIndexDir;
	/** Whether the index may be split or not */
	protected boolean splitActive = false;
	/** Size limit for index splitting. (byte unit) */
	protected long splitSize = 500*1024*1024;
	/** Size unit for index splitting. */
	protected String splitUnit = "m";
	/** Document limit for index splitting */
	protected long splitDoc = 500000;
	/** Compound File format usage defaulted to true */
	protected boolean useCompoundFiles = true;
	/** Auto-optimization when uploading / deleting documents. Defaulted to true (as in SDX 2.2) */
	protected boolean autoOptimize = true;
	protected boolean isDatadirShared = false;
	/** A flag to know if the index is optimized or not */
	protected boolean _isIndexOptimized = false;
	/* String representation for our default pipeline parameter. */
	protected final String DOC_URL = "docUrl";
	/* String representation for our pipeline parameter. */
	protected final String SDX_USER = "sdxUser";
	/* String representation for our pipeline parameter. */
	protected final String SDX_DATE = "sdxDate";
	/* String representation for our pipeline parameter. */
	protected final String SDX_ISO8601_DATE = "sdxISO8601Date";
	/* String representation for a pipeline parameter. */
	protected final String SDX_DATE_MILLISECONDS = "sdxDateMilliseconds";
	protected static final String[] _documentAdditionStatus = {"failure", "ignored", "added", "refreshed", "replaced"};
	protected static final int DOC_ADD_STATUS_FAILURE = 0;
	protected static final int DOC_ADD_STATUS_IGNORED = 1;
	protected static final int DOC_ADD_STATUS_ADDED = 2;
	protected static final int DOC_ADD_STATUS_REFRESHED = 3;
	protected static final int DOC_ADD_STATUS_REPLACED = 4;
	protected static final String SDX_DATABASE_FORMAT = "sdx_database_format";
	protected static final String SDX_DATABASE_VERSION = "sdx_database_version";
	protected static final String SDX_DATABASE_VERSION_2_3 = "sdx_2.3";
	protected static final String ELEMENT_NAME_DEFAULT_HPP = "default_hpp";
	protected static final String ELEMENT_NAME_DEFAULT_MAXSORT = "default_maxsort";

	protected Configuration _configuration;
	
	protected LoggingIndexation _ilogger = null;
	protected int _ilevel = -1;

	protected void setConfiguration(Configuration configuration){
		this._configuration = configuration;
	}

	protected Configuration getConfiguration(){
		return this._configuration;
	}


	/**
	 * Configures SDX document base
	 */
	public void configure(Configuration configuration) throws ConfigurationException {
		LoggingUtils.logInfo(_logger, SDXExceptionCode.INFO_DOCUMENTBASE_CONFIGURING, null);
		this.setConfiguration(configuration);
		configureBase(configuration);
		String[] args = {this._id};
		LoggingUtils.logInfo(_logger, SDXExceptionCode.INFO_DOCUMENTBASE_ID_SETTING, args);
		super.configure(configuration, isUseMetadata());
		this.configureSplit(configuration);
		this.configurePipeline(configuration);
		this.configureIdGenerator(configuration);
		//adding the repositories to this documentbase
		this.configureRepositories(configuration);
		//doing any documentbase specific configuration in subclasses
		configureDocumentBase(configuration);
		//configuring oai components from subclasses
		configureOAIComponents(configuration);
		// indexation logging level give by the application configuration
		if ("sdxuserdb".equals(this._id)) {
			this._ilevel = 0; // Pour la base des utilisateurs, on ne veut pas de log d'indexation
		}
		else {
			try {
				this._ilevel = ( (Integer) this._context.get(ContextKeys.SDX.Application.INDEXATION_LOGGING_LEVEL) ).intValue();
			} catch (Exception e) {
				// FIXME (MP): Handle exception ?
			}
		}
		LoggingUtils.logInfo(_logger, SDXExceptionCode.INFO_DOCUMENTBASE_CONFIGURED, args);
	}

	protected void configureBase(Configuration configuration) throws ConfigurationException {
		String dbDirPath = "";
		//at this point, we have a <sdx:documentBase> configuration object...
		if (configuration.getName().toLowerCase().indexOf(DocumentBase.CLASS_NAME_SUFFIX.toLowerCase()) > -1) {
			try {
				this.keepOriginalDocuments = configuration.getAttributeAsBoolean(DBELEM_ATTRIBUTE_KEEP_ORIGINAL, true);
				this.useCompoundFiles = configuration.getAttributeAsBoolean(ATTRIBUTE_COMPOUND_FILES, true);
				//setting the id from the configuration file
				setId(configuration.getAttribute(Identifiable.ConfigurationNode.ID, super.getId()));
				//Optimization method configuration
				this.autoOptimize = configuration.getAttributeAsBoolean(ATTRIBUTE_AUTO_OPTIMIZE, true);
				// Optimization method configuration
				try {
					this.isDatadirShared = ( (Boolean) this._context.get(ContextKeys.SDX.Application.IS_DATADIR_SHARED) ).booleanValue();
				} catch (Exception e) {
					// FIXME (MP): Handle exception ?
				}
				//configuring the cron if needed
				configureOptimizeTriggers(configuration.getChild(ELEMENT_NAME_OPTIMIZE, false));

			} catch (SDXException sdxE) {
				throw new ConfigurationException(sdxE.getMessage(), sdxE);
			}
			if (!Utilities.checkString(getId())) {
				String[] args = new String[1];
				args[0] = configuration.getLocation();
				SDXException sdxE = new SDXException(super.getLog(), SDXExceptionCode.ERROR_INVALID_ID_VALUE, args, null);
				throw new ConfigurationException(sdxE.getMessage(), sdxE);
			}
			//getting the path for the location where this document base will reside
			dbDirPath = Utilities.getStringFromContext(ContextKeys.SDX.Application.DOCUMENTBASES_DIRECTORY_PATH, super.getContext()) + getId() + File.separator;

			//useMetadata confiburation
			this.useMetadata = configuration.getAttributeAsBoolean(ATTRIBUTE_USE_METADATA, true);
			if(!isUseMetadata()){
				//TODO: eventually control params with useMetadata="false"
			}

		} else {
			String[] args = new String[1];
			args[0] = configuration.getName();
			//not passing a super.getLog() should be logged later up the stack
			SDXException sdxE = new SDXException(SDXExceptionCode.ERROR_UNRECOGNIZED_DOCUMENTBASE_CONFIG, args);
			throw new ConfigurationException(sdxE.getMessage(), sdxE);
		}

		//setting property : the path where this document base resides
		super.getContext().put(ContextKeys.SDX.DocumentBase.DIRECTORY_PATH, dbDirPath);
		try{
			Configuration sdxConf = (Configuration) super.getContext().get(ContextKeys.SDX.Framework.CONFIGURATION_FILE);
			//testing if we have something
			if (sdxConf != null) {
				Configuration iFConf = sdxConf.getChild(SDXDocumentBase.ELEMENT_NAME_DEFAULT_HPP, false);
				//	testing if we have something
				if (iFConf != null){
					this.defaultHitsPerPage = iFConf.getAttributeAsInteger(DBELEM_ATTRIBUTE_HPP, this.defaultHitsPerPage);
				}
				else
					this.getLog().debug("No default_hpp value in sdx's configuration file.");

				iFConf = sdxConf.getChild(SDXDocumentBase.ELEMENT_NAME_DEFAULT_MAXSORT, false);
				//	testing if we have something
				if (iFConf != null){
					this.defaultMaxSort = iFConf.getAttributeAsInteger(DBELEM_ATTRIBUTE_MAXSORT, this.defaultMaxSort);
				}
				else
					this.getLog().debug("No default_maxsort value in sdx's configuration file.");
			}
		} catch(ContextException e){
		}

		//setting the default flag, false if not specified in configuration file
		this.isDefault = configuration.getAttributeAsBoolean(DBELEM_ATTRIBUTE_DEFAULT, false);
		//setting the maxsort value, keep unchanged if not specified in configuration file
		this.defaultMaxSort = configuration.getAttributeAsInteger(DBELEM_ATTRIBUTE_MAXSORT, this.defaultMaxSort);
		//setting the hpp value, keep unchanged if not specified in configuration file
		this.defaultHitsPerPage = configuration.getAttributeAsInteger(DBELEM_ATTRIBUTE_HPP, this.defaultHitsPerPage);

	}

	/**
	 * Index splitter configuration
	 */
	protected void configureSplit(Configuration configuration) throws ConfigurationException {
		//at this point, we have a <sdx:documentBase> configuration object...
		if(configuration.getName().toLowerCase().indexOf(DocumentBase.CLASS_NAME_SUFFIX.toLowerCase()) > -1) {

			//Configure splitting condition
			Configuration splitConfig = configuration.getChild(ELEMENT_NAME_INDEX_SPLIT, false);

			//Check if exist, if not then apply unlimited values
			if(splitConfig != null)
			{
				this.splitActive = true;
				this.splitSize = splitConfig.getAttributeAsLong(ATTRIBUTE_SPLIT_SIZE, 500);
				this.splitUnit = splitConfig.getAttribute(ATTRIBUTE_SPLIT_UNIT, "m");
				this.splitDoc = splitConfig.getAttributeAsLong(ATTRIBUTE_SPLIT_DOC, 500000);

				//adjust size to unit
				if(this.splitSize < 0)
					this.splitSize = 0;
				else if(this.splitUnit.equalsIgnoreCase("k"))
					this.splitSize *= 1024;
				else if(this.splitUnit.equalsIgnoreCase("m"))
					this.splitSize *= (1024*1024);
				else if(this.splitUnit.equalsIgnoreCase("g"))
					this.splitSize *= (1024*1024*1024);

				if(this.splitDoc < 0) this.splitDoc = 0;
			} else {
				this.splitActive = false;
				this.splitSize = 500*1024*1024;
				this.splitUnit = "m";
				this.splitDoc = 500000;
			}
		}
	}

	/**
	 * Configures repositories
	 * @param configuration
	 * @return
	 * @throws ConfigurationException
	 */
	protected Configuration[] getRepositoryConfigurationList(Configuration configuration) throws ConfigurationException {
		//at this point, we should have a <sdx:repositories> element containing a list of repositories
		//intializing the array
		String elementName = Utilities.getElementName(Repository.CLASS_NAME_SUFFIX);
		Configuration[] repoConfList = new Configuration[configuration.getChild(Repository.ConfigurationNode.REPOSITORIES).getChildren(elementName).length];
		//getting an array of configuration objects from each of the <sdx:repository> subElements of <sdx:repositories> element
		repoConfList = configuration.getChild(Repository.ConfigurationNode.REPOSITORIES).getChildren(elementName);
		//testing if we have something
		if (repoConfList == null || repoConfList.length == 0) {
			//We don't want the document base fail if there isn't repositories; just warn.
			String[] args = {this.getId()};
			LoggingUtils.logWarn(_logger, SDXExceptionCode.WARN_NO_REPOSITORIES_IN_CONFIG,args);
		}

		return repoConfList;
	}


	/**
	 * Configures repositories
	 * @param configuration
	 * @throws ConfigurationException
	 */
	protected void configureRepositories(Configuration configuration) throws ConfigurationException {
		Configuration[] repoConfList = getRepositoryConfigurationList(configuration);
		//ensuring we have a hashtable to populate
		if (super.repositories == null) super.repositories = new Hashtable();
		//a pointer to the first repository
		Repository firstRepository = null;

		if(!isUseMetadata() && repoConfList.length > 1){
			throw new ConfigurationException("You must use metadata if you use more than one repository", configuration);
		}
		else {

			//iterating over the list and creating the repositories
			for (int i = 0; i < repoConfList.length; i++) {
				Repository repo = null;
				try {
					/*check for the ref attribute, if it exists, get the repository object and add it to the local hashtable
					 *if the attribute doesn't exist create the repo like below, we also need to handle DEFAULTS with refs*/
					String ref = repoConfList[i].getAttribute(Repository.ConfigurationNode.REF, null);
					if (Utilities.checkString(ref)) {
						Context appRepos = (Context) super.getContext().get(ContextKeys.SDX.Application.REPOSITORIES);
						if (appRepos != null)
							repo = (Repository) appRepos.get(ref);
						if (repo == null) {
							String[] args = new String[1];
							args[0] = ref;
							throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_LOAD_REFERENCED_REPO, args, null);
						}
						//setting the default flag for the referenced repository
						repo.setIsDefault(repoConfList[i].getAttributeAsBoolean(Repository.ConfigurationNode.DEFAULT, false));
					} else
						//creating the repository
						repo = ConfigurationUtils.createRepository(super.getLog(), super.getContext(), super.getServiceManager(), repoConfList[i]);

					if(!isUseMetadata()){
						if(repo instanceof URLRepository)
							throw new ConfigurationException("You must use metadata if you use URLrepository", configuration);
						else if(repo instanceof FSRepository)
							throw new ConfigurationException("You must use metadata if you use FSRepository", configuration);
					}

					//populating the hashtable
					Utilities.isObjectUnique(this.repositories, null, repo);
					repositories.put(repo.getId(), repo);
					//setting the default repository if applicable
					if (repo.isDefault()) defaultRepository = repo;
					//retaining first repository
					if (i == 0) firstRepository = repo;
				} catch (SDXException e) {
					//we don't want all repositories configurations to fail so we won't throw this farther out
					//the creation of the SDXException should log this message
				} catch (ConfigurationException e) {
					//we don't want all repository configurations to fail so we won't throw this farther out
					LoggingUtils.logException(super.getLog(), e);
				} catch (ContextException e) {
					throw new ConfigurationException(e.getMessage(), e);
				}
			}

			//if no default repository was provided, we set the default to the first repository
			if (defaultRepository == null) defaultRepository = firstRepository;
		}

	}


	/**
	 * Configures OAI components if relevant: repository and harvester
	 * @param configuration
	 * @throws ConfigurationException
	 */
	public void configureOAIComponents(Configuration configuration) throws ConfigurationException {
		configureOAIRepositories(configuration);
		configureOAIHarvester(configuration);
	}

	protected void configureIdGenerator(Configuration configuration) throws ConfigurationException {
		this.idGen = ConfigurationUtils.configureIDGenerator(super.getLog(), configuration);
		if(!isUseMetadata())
			this.idGen.setDatabase(super._database);
	}

	/*End Configurations*/


	/* BEGIN abstract method declarations for specific implementation overides*/
	protected abstract void configureDocumentBase(Configuration configuration) throws ConfigurationException;

	protected abstract void deleteFromSearchIndex(String id) throws SDXException;

	protected abstract void configureOAIRepositories(Configuration configuration) throws ConfigurationException;

	protected abstract void configureOAIRepository(Configuration configuration) throws ConfigurationException;

	protected abstract void configureOAIHarvester(Configuration configuration) throws ConfigurationException;

	protected abstract Object getIndexationDocument(IndexableDocument doc, String storeDocId, String repoId, IndexParameters params) throws SDXException;

	/*TODORemove, possibly remove the "batchIndex" argument below as all indexations are now done in batches
	 * this would have a corresponding affect on the LuceneIndex class
	 */
	/**Add a document to the underlying search index
	 *
	 * @param indexationDoc The document object for the specific search implementation (lucene, or other)
	 * @param batchIndex		parameter to indicate wheter a batch indexation pass is taking place so that optimization is done at the end of the batch
	 * @throws SDXException
	 */
	protected abstract void addToSearchIndex(Object indexationDoc, boolean batchIndex) throws SDXException;

	protected abstract void compactSearchIndex() throws SDXException;

	/* END abstract method declarations for specific implementation overides*/


	/**
	 * Get the specified file format for indexation
	 */
	public boolean getUseCompoundFiles()
	{
		return this.useCompoundFiles;
	}

	/**
	 * Get information about optimization method
	 */
	public boolean isAutoOptimized()
	{
		return this.autoOptimize;
	}

	/**
	 * Get the split size for indexes, in byte
	 */
	protected long getByteSplitSize()
	{
		return this.splitSize;
	}

	/**
	 * Get the split size for indexes, depending on unit
	 */
	public long getSplitSize()
	{
		if(this.splitUnit.equalsIgnoreCase("k"))
			return this.splitSize / 1024;
		else if(this.splitUnit.equalsIgnoreCase("m"))
			return this.splitSize / (1024*1024);
		else if(this.splitUnit.equalsIgnoreCase("g"))
			return this.splitSize / (1024*1024*1024);
		else return this.splitSize;
	}

	/**
	 * Get split size unit
	 */
	public String getSplitUnit()
	{
		return this.splitUnit;
	}

	/**
	 * Get the document number condition for index splitting
	 */
	public long getSplitDoc()
	{
		return this.splitDoc;
	}


	/** Gets a SDX document as SAX events.
	 *
	 * @param doc A ParsableDocument, ie XMLDocument or HTMLDocument.
	 * @param consumer A SAX content handler to feed with events.
	 * <p>
	 * The wrapped contentHandler for including events within an XSP page contentHandler should be created using
	 * <CODE>IncludeXMLConsumer stripper = new IncludeXMLConsumer(xspContentHandler);</CODE>
	 * @throws SDXException
	 */
	public void getDocument(ParsableDocument doc, XMLConsumer consumer) throws SDXException {
		//ensuring we have valid objects
		super.getDocument(doc, consumer);
		Repository repo = null;
		RepositoryConnection conn = null;

		try {
			//getting the repository in which the document resides
			repo = getRepositoryForDocument(doc);
			//getting a connection for the repository
			if (repo != null)
				conn = repo.getConnection();
			//then call toSAX from the repository
			if (repo != null && conn != null)
				repo.toSAX(doc, consumer, conn);
		} finally {
			//always close the connection
			if (repo != null && conn != null) repo.releaseConnection(conn);
		}
	}

	/** Gets a SDX document as SAX events.
	 *
	 * @param doc						A Document, ie XMLDocument or HTMLDocument, we discern the type from the document lookup.
	 * @param consumer			A SAX content handler to feed with events.
	 * @param docTypeKnown	 If the type of the ParsableDocument desired is unknown, ie. XML or HTML,
	 *											allows users to build one or the other type of document and still retrieve a document
	 * <p>
	 * The wrapped contentHandler for including events within an XSP page contentHandler should be created using
	 * <CODE>IncludeXMLConsumer stripper = new IncludeXMLConsumer(xspContentHandler);</CODE>
	 * @throws SDXException
	 */
	public void getDocument(ParsableDocument doc, XMLConsumer consumer, boolean docTypeKnown) throws SDXException {
		//ensuring we have valid objects
		super.getDocument(doc, consumer);
		if (isUseMetadata() && !docTypeKnown) {
			//looking-up for the document in repository look-up index
			DatabaseEntity ent = _database.getEntity(doc.getId());
			if (ent == null) {
				String[] args = new String[2];
				args[0] = doc.getId();
				args[1] = this.getId();
				throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_NO_DOC_EXISTS_DOCBASE, args, null);
			}
			ent.enableLogging(super.getLog());
			String doctype = ent.getProperty(PROPERTY_NAME_DOCTYPE);
			if (Utilities.checkString(doctype)) {
				if (doctype.equalsIgnoreCase(Document.DOCTYPE_HTML))
					doc = new HTMLDocument(doc.getId());
				else //if (doctype.equalsIgnoreCase(Document.DOCTYPE_XML) || doctype.equalsIgnoreCase(Document.DOCTYPE_USER) || doctype.equalsIgnoreCase(Document.DOCTYPE_GROUP))
					doc = new XMLDocument(doc.getId());
				//TODO:maybe in the future we need to add more checks for other types of parsable documents
				doc.enableLogging(super.getLog());
			}

		}
		//now we know have a good doctype so we call our original method
		this.getDocument(doc, consumer);

	}

	/** Supplies the provided output stream with the requested document
	 * @param doc The document.
	 * @param os	The output stream.
	 * @throws fr.gouv.culture.sdx.exception.SDXException
	 */
	public void getDocument(Document doc, OutputStream os) throws SDXException {
		//ensuring we have valid objects
		super.getDocument(doc, os);
		Repository repo = null;
		RepositoryConnection conn = null;
		try {
			//getting the repository in which the document resides
			repo = getRepositoryForDocument(doc);
			//getting a connection for the repository
			if (repo != null)
				conn = repo.getConnection();
			//then call toSAX from the repository
			if (repo != null && conn != null)
				repo.get(doc, os, conn);
		} finally {
			//always close the connection
			if (repo != null && conn != null) repo.releaseConnection(conn);
		}
	}

	/** Provides the requested SDX document as an InputStream
	 * @param doc The document.
	 * @return	An input stream.
	 * @throws fr.gouv.culture.sdx.exception.SDXException
	 */
	public InputStream getDocument(Document doc) throws SDXException {
		//ensuring we have valid objects
		super.getDocument(doc);
		Repository repo = null;
		RepositoryConnection conn = null;
		InputStream is = null;

		try {

			//getting the repository in which the document resides
			repo = getRepositoryForDocument(doc);
			//getting a connection for the repository
			if (repo != null)
				conn = repo.getConnection();
			//getting an InputStream from the repository with a null encoding
			//TODO?: should we support an encoding parameter in the future?-rbp
			if (repo != null && conn != null)
				is = repo.openStream(doc, null, conn);
		} finally {
			//always close the connection
			if (repo != null && conn != null) repo.releaseConnection(conn);
		}
		return is;
	}


	/**This method retrieves a repository id from a document object
	 * and then does a lookup for the corresponding repository object.
	 *
	 * @param doc		Document with desired repository storage id
	 * @param defaultRepo Default repo to utilize	 if none found
	 * @return Repository
	 * @throws SDXException
	 */
	protected Repository getRepositoryForStorage(Document doc, Repository defaultRepo) throws SDXException {
		Repository repo = null;
		if (doc != null) {
			String repoId = doc.getRepositoryForStorage();
			if (Utilities.checkString(repoId))
				repo = getRepository(repoId);
		}

		if (repo == null) {
			if (defaultRepo != null)
				repo = defaultRepo;
			else
				repo = defaultRepository;
		}
		return repo;
	}

	/**Gets the repository object in which a document resides,
	 * document should not be null and have a valid id,
	 * use Utilities.checkDocument before calling this method*/
	protected Repository getRepositoryForDocument(Document doc) throws SDXException {

		DatabaseEntity ent = null;
		String repoId = null;
		Repository repo = null;

		//if we use metadata, looking-up for the document in repository look-up index
		if(isUseMetadata()){
			ent = _database.getEntity(doc.getId());
			if (ent == null) {
				String[] args = new String[2];
				args[0] = doc.getId();
				args[1] = this.getId();
				throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_NO_DOC_EXISTS_DOCBASE, args, null);
			}
			ent.enableLogging(super.getLog());
			//getting the document's repository
			repoId = ent.getProperty(PROPERTY_NAME_REPO);
			if (Utilities.checkString(repoId) && repositories != null && repositories.containsKey(repoId))
				repo = (Repository) repositories.get(repoId);
		}
		//if we do not use metadata, get the default repository
		else{
			repo = defaultRepository;
		}

		return repo;

	}

	/**
	 * Deletes a document and any attached document(s) if not used by any other document(s).
	 *
	 * @param	doc The document to delete.
	 * @throws fr.gouv.culture.sdx.exception.SDXException
	 */
	protected synchronized void delete(Document doc, boolean isIndexable, boolean isPrimary, boolean isShared, String relation, ContentHandler handler) throws SDXException, SAXException, ProcessingException {
		//ensuring we have valid objects
		Utilities.checkDocument(super.getLog(), doc);
		Repository repository = null;
		if (handler != null)
			handler.startElement(Framework.SDXNamespaceURI, Node.Name.DELETION, Framework.SDXNamespacePrefix + ":" + Node.Name.DELETION, new AttributesImpl());
		try {

			//TODO : refactor connection management (in the calling method ? use a pool ?)
			//some set-up for accessing the repository
			repository = getRepositoryForDocument(doc);

			if (isIndexable) deleteIndexableDocumentComponents((IndexableDocument) doc, handler);
			//else {TODO Test this: removing this else{}, this would allow us to delete sub documents
			/*
			 * STEP 1 : delete the relations to any parent documents (currently,
			 *ONLY attached documents may have multiple owners)
			 */
			//TODO: use inner class when javac compiler bug is fixed-rbp
			//						super.lookupDatabase.deleteRelationsToMastersFromDatabase(doc);
			if (!isPrimary && isShared && Utilities.checkString(relation))
				deleteRelationsToMastersFromDatabase(relation, doc);
			//}

			/*
			 * STEP 6 : delete the document's entry from the look-up index
			 */
			//TODO: use inner class when javac compiler bug is fixed-rbp
//			super.lookupDatabase.deleteDocMetaDataFromRelationsIndex(doc);
			/*Getting the database entity corresponding to this document*/

			/*
			 * STEP 7 : deleteing the document itself
			 */
			if (repository != null)
				deletePhysicalDocument(doc, repository, handler);

			if(isUseMetadata())
				super._database.delete(new DatabaseEntity(doc.getId()));

			/*
			 * TODO : that should never be the case but it would be safe to destroy all other relations
			 * to and from this document, using for example :
			 * deleteFromRelationsIndex(doc, null, null, 0, batchDelete);
			 */


		} finally {

			if (handler != null) {
				handler.endElement(Framework.SDXNamespaceURI, Node.Name.DELETION, Framework.SDXNamespacePrefix + ":" + Node.Name.DELETION);
			}

		}
	}

	/**Deletes all secondary document and search index components
	 * @param doc		The parent document
	 */
	protected synchronized void deleteIndexableDocumentComponents(IndexableDocument doc, ContentHandler handler) throws SDXException, ProcessingException, SAXException {
		/*Getting the database entity corresponding to this document*/
		DatabaseEntity ent = null;
		if(isUseMetadata())
			ent = _database.getEntity(doc.getId());
		//if this is null we don't need to do anything because we shouldn't have a document
		if (isUseMetadata() && ent == null) return;

		/*
		 * this flag replaces the tests on (doc instanceof IndexableDocument)
		 * some instances of IndexableDocument (e.g. TransformedDocuments) are *not* to be indexed)
		 * TODO : find a better name and improve document -pb
		 */

		//casting the document to an indexable document
		IndexableDocument indexableDoc = (IndexableDocument) doc;

		//keeping track of deleted records if oai support is enabled
		addOaiDeletedRecord(indexableDoc);
		/*
		 * STEP 1 : deleteing the document's indexation from the search index
		 */
		//TODO : test if the deletion is relevant since the document may not be indexed *yet* -pb
		//TODO : consistency : this is now the 4th step in add -pb
		deleteFromSearchIndex(indexableDoc.getId());
		//Warning : given the behaviour of DatabaseEntity.getField, only 1 original document is assumed -pb
		//TODO: use inner class when javac compiler bug is fixed-rbp
		//DatabaseEntity[] originalDocEntry = super.lookupDatabase.getRelated(doc.getId(), RELATION_TYPE_ORIGINAL);
		DatabaseEntity originalDocEntry = null;
		String originalDocId = null;
		if(isUseMetadata()) {
			originalDocId = ent.getProperty(PROPERTY_NAME_ORIGINAL);
			if (Utilities.checkString(originalDocId))
				originalDocEntry = super._database.getEntity(ent.getProperty(PROPERTY_NAME_ORIGINAL));
		}
			
		/*
		 * STEP 2 : deleteing the original document, if relevant
		 */
		if (originalDocEntry != null) {
			Document origDoc = new BinaryDocument(); //Sic !
			origDoc.enableLogging(super.getLog());
			origDoc.setId(originalDocEntry.getId());

			//Notice that the original document look-up entry will be deleted as well
			if (handler != null) handler.startElement(Framework.SDXNamespaceURI, PROPERTY_NAME_ORIGINAL, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_ORIGINAL, new AttributesImpl());
			this.delete(origDoc, /*isIndexable*/false, /*isPrimary*/false, /*isShared*/false, PROPERTY_NAME_ORIGINAL, handler); //recursive call
			if (handler != null) handler.endElement(Framework.SDXNamespaceURI, PROPERTY_NAME_ORIGINAL, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_ORIGINAL);

			/*TODO: i don't think the below line is necessary any longer since the attached/sub document deletion calls
						deleteRelationsToMastersFromDatabase(Document doc)*/
			//deleteing the relation between the document and the original document
			//deleteFromRelationsIndex(doc, origDoc, RELATION_TYPE_ORIGINAL);
		}

		/*
		 * STEP 3 : deleteing the attached documents, if relevant
		 */
		//TODO: use inner class when javac compiler bug is fixed-rbp
//		DatabaseEntity[] attachedDocs = super.lookupDatabase.getRelated(doc.getId(), RELATION_TYPE_ATTACHED);
		String[] attachedDocs = null;
		if(isUseMetadata())
			ent.getPropertyValues(PROPERTY_NAME_ATTACHED);
		if (attachedDocs != null && attachedDocs.length > 0) {
			if (handler != null) handler.startElement(Framework.SDXNamespaceURI, PROPERTY_NAME_ATTACHED, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_ATTACHED, new AttributesImpl());
			for (int i = 0; i < attachedDocs.length; i++) {
				//getting the attached document
				if (isUseMetadata() && Utilities.checkString(attachedDocs[i])) { //How a null value could be possible ? If so, throw a recoverable exception ? -pb
					DatabaseEntity l_attachedDocEntity = _database.getEntity(attachedDocs[i]);
					//TODO: use inner class when javac compiler bug is fixed-rbp
//					DatabaseEntity owners[] = super.lookupDatabase.getOwners(l_attachedDocEntity.getId());
					String[] l_attachedDocOwners = l_attachedDocEntity.getPropertyValues(PROPERTY_NAME_PARENT);
					//Delete attached document entry if it is owned by our doc and our doc only
					if (l_attachedDocOwners == null || l_attachedDocOwners.length == 0) {
						//TODOException : the attached document is not owned ?
					} else {
						if (l_attachedDocOwners.length == 1 && l_attachedDocOwners[0] != null && l_attachedDocOwners[0].equals(indexableDoc.getId())) {
							//Notice that the attached document look-up entry will be deleted as well
							try {
								BinaryDocument l_attachedDoc = new BinaryDocument();
								l_attachedDoc.enableLogging(super.getLog());
								l_attachedDoc.setId(l_attachedDocEntity.getId());
								this.delete(l_attachedDoc, false, false, false, PROPERTY_NAME_ATTACHED, handler); //recursive call
							} catch (SDXException e) {
								if (handler != null) e.toSAX(handler);
							} catch (SAXException e) {
								//can only log here as we don't want the entire loop to fail
								LoggingUtils.logError(super.getLog(), e.getMessage(), e);
							} catch (ProcessingException e) {
								//can only log here as we don't want the entire loop to fail
								LoggingUtils.logError(super.getLog(), e.getMessage(), e);
							}
						} else if (l_attachedDocOwners.length == 1 && l_attachedDocOwners[0] != null && !l_attachedDocOwners[0].equals(indexableDoc.getId())) {
							//TODOException : the attached document is owned, but not by our document !
						} else {
							//Attached document is owned by more than 1 document : keep it
							//TODO : check that our document is among the owners
							//remove our doucment from the list of parents/owners
							_database.removeProperty(l_attachedDocEntity.getId(), PROPERTY_NAME_PARENT, indexableDoc.getId());
						}
					}
				} else if(!isUseMetadata() && Utilities.checkString(attachedDocs[i])){
					throw new SDXException("You must use metadata if you use attached documents");
				}
			}
			if (handler != null) handler.endElement(Framework.SDXNamespaceURI, PROPERTY_NAME_ATTACHED, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_ATTACHED);


		}

		/*
		 * STEP 4 : deleteing the sub-documents, if relevant
		 */
		//FIXME: Very bad things happening here at step 4 : throw a null pointer exception, not fully deleting the document. That result later in a general application error because you can no longer index anything. This occurs because there are no more subdocuments to delete. -la
		//TODO: use inner class when javac compiler bug is fixed-rbp
		//DatabaseEntity[] subDocs = super.lookupDatabase.getRelated(doc.getId(), RELATION_TYPE_SUB);
		String[] subDocs = null;
		if(isUseMetadata())
			subDocs = ent.getPropertyValues(PROPERTY_NAME_SUB);
		if(subDocs != null && subDocs.length > 0) {//TODO: find a better way to manage the subdocs. At the moment, it search for subdocs in database that just went deleted, so even if subDocs itself is not null, the database will return a null value and launch an exception. -la
			if (handler != null) handler.startElement(Framework.SDXNamespaceURI, PROPERTY_NAME_SUB, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_SUB, new AttributesImpl());
			for (int i = 0; i < subDocs.length; i++) {
				//getting the sub-document
				if (Utilities.checkString(subDocs[i])) { //How a null value could be possible ? If so, throw a recoverable exception ? -pb
					DatabaseEntity l_subDocEntity = _database.getEntity(subDocs[i]);//FIXME: Sure subDocs is not null, but the value returned by the getEntity is. So, are the subdocs already deleted as I can guess with the debug informations, or should we try another way to delete them ? -la
					//TODO: do something useful here
					if(l_subDocEntity == null)
					{
						//TODO: SubDoc already deleted in database
					}
					else
					{
						String[] l_subDocOwners = l_subDocEntity.getPropertyValues(PROPERTY_NAME_PARENT); //FIXME: here is the null pointer exception -la
						;
						if (l_subDocOwners == null || l_subDocOwners.length == 0) {
							//TODOException : the sub document is not owned ?
						} else {
							if (l_subDocOwners.length == 1 && l_subDocOwners[0] != null && l_subDocOwners[0].equals(indexableDoc.getId())) {
								//Notice that the sub document look-up entry will be deleted as well
								try {
									XMLDocument subDoc = new XMLDocument();
									subDoc.enableLogging(super.getLog());
									subDoc.setId(l_subDocEntity.getId());
									this.delete(subDoc, true, false, false, PROPERTY_NAME_SUB, handler); //recursive call
								} catch (SDXException e) {
									if (handler != null) e.toSAX(handler);
								} catch (SAXException e) {
									//can only log here as we don't want the entire loop to fail
									LoggingUtils.logError(super.getLog(), e.getMessage(), e);
								} catch (ProcessingException e) {
									//can only log here as we don't want the entire loop to fail
									LoggingUtils.logError(super.getLog(), e.getMessage(), e);
								}
							} else if (l_subDocOwners.length == 1 && l_subDocOwners[0] != null && !l_subDocOwners[0].equals(indexableDoc.getId())) {
								//TODOException : the sub document is owned, but not by our document !
							} else {
								//sub document is owned by more than 1 document : keep it
								//TODO : check that our document is among the owners
								//remove our doucment from the list of parents/owners
								_database.removeProperty(l_subDocEntity.getId(), PROPERTY_NAME_PARENT, indexableDoc.getId());
							}
						}
					}
				}
			}
			if (handler != null) handler.endElement(Framework.SDXNamespaceURI, PROPERTY_NAME_SUB, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_SUB);

		}
	}

	/**
	 * Deletes an array of documents and any attached document(s) if not used by any other document(s).
	 *
	 * @param docs	The documents to delete.
	 * @param		handler A content handler to feed with information.
	 * @throws fr.gouv.culture.sdx.exception.SDXException
	 *
	 */
	public synchronized void delete(Document[] docs, ContentHandler handler) throws SDXException, SAXException, ProcessingException {

		/* TODORefactor:
				in the future we should implement some logic to lookup the documents and group them by repository,
				get a connection for each repo and then delete using the same connection
		 */

		if (docs != null) {
			long start = System.currentTimeMillis();

			//we a batch of documents to delete so we set a true state
			AttributesImpl atts = new AttributesImpl();
			if (handler != null) {
				String appId = Utilities.getStringFromContext(ContextKeys.SDX.Application.ID, super.getContext());
				if (Utilities.checkString(appId)) atts.addAttribute("", Node.Name.APP, Node.Name.APP, Node.Type.CDATA, appId);
				atts.addAttribute("", Node.Name.BASE, Node.Name.DB_ID, Node.Type.CDATA, this.getId());
				handler.startElement(Framework.SDXNamespaceURI, Node.Name.DELETIONS, Framework.SDXNamespacePrefix + ":" + Node.Name.DELETIONS, atts);

			}
			//counters
			int deletions = 0;
			int failures = 0;

			try {
				for (int i = 0; i < docs.length; i++) {
					//if we have the last document we set the flag that the batch delete is finished
					try {
						if (docs[i] instanceof IndexableDocument) {
							delete(docs[i], /*isIndexable*/true, /*isPrimary*/false, /*isShared*/true, null, handler);
						} else if (docs[i] instanceof BinaryDocument) {
							delete(docs[i], /*isIndexable*/false, /*isPrimary*/false, /*isShared*/true, null, handler);
						}
						deletions++;
					} catch (SDXException e) {
						failures++;
						if (handler != null) e.toSAX(handler);
						else LoggingUtils.logException(super.getLog(), e);
					}
				}
			} finally {
				if (handler != null /*&& params.getSendIndexationEvents() >= IndexParameters.SEND_ALL_EVENTS*/) {
					long finish = System.currentTimeMillis();
					long elapsed = finish - start;
					AttributesImpl summaryAtts = new AttributesImpl();
					summaryAtts.addAttribute("", Node.Name.DELETIONS, Node.Name.DELETIONS, Node.Type.CDATA, Integer.toString(deletions));
					summaryAtts.addAttribute("", Node.Name.FAILURES, Node.Name.FAILURES, Node.Type.CDATA, Integer.toString(failures));
					summaryAtts.addAttribute("", Node.Name.DURATION, Node.Name.DURATION, Node.Type.CDATA, String.valueOf(elapsed / 1000));
					handler.startElement(Framework.SDXNamespaceURI, Node.Name.SUMMARY, Framework.SDXNamespacePrefix + ":" + Node.Name.SUMMARY, summaryAtts);
					handler.endElement(Framework.SDXNamespaceURI, Node.Name.SUMMARY, Framework.SDXNamespacePrefix + ":" + Node.Name.SUMMARY);

					handler.endElement(Framework.SDXNamespaceURI, Node.Name.DELETIONS, Framework.SDXNamespacePrefix + ":" + Node.Name.DELETIONS);
				}
				// if more than one document and DocumentBase is autoOptimize, optimize the DocumentBase
				if(autoOptimize) optimize();
			}
		}
	}

	/** Adds a document.
	 *
	 * @param doc		The document to add.
	 * @param repository		The repository where to store the document.
	 * @param params	The parameters for this adding action.
	 * @param handler A content handler where to send information about the process (may be null), currently we don't use it, but maybe in the future.
	 * TODO : what kind of "informations" ? -pb
	 */
	public synchronized void index(IndexableDocument doc, Repository repository, IndexParameters params, ContentHandler handler) throws SDXException, SAXException, ProcessingException {
		IndexableDocument[] docs = new IndexableDocument[1];
		docs[0] = doc;
		index(docs, repository, params, handler);
	}


	protected void rollbackIndexation(IndexableDocument doc, ContentHandler handler) throws SDXException, SAXException, ProcessingException {
		if (doc.getStoreHandler().getDocs().length > 0) {
			delete(doc.getStoreHandler().getDocs(), handler);
			deleteFromSearchIndex(doc.getId());
		}
	}



	//TODO: use inner class when javac compiler bug is fixed-rbp
	//protected class LookupDatabase {

	/**Retrieves all parent documents
	 * containing the relationType/docId
	 * entry
	 *
	 * @param relationType	The relation type (sub, attached, original)
	 * @param docId The secondary document id
	 * @param handler		Then handler to feed with events
	 * @return String[]
	 * @throws SDXException
	 * @throws SAXException
	 */
	public String[] getOwners(String relationType, String docId, ContentHandler handler) throws SDXException, SAXException {
		//below line can be used to test _database.search() method
		//params.setParameter(MIMETYPE_PROPERTY, "text/xml");
		DatabaseEntity doc = null;
		if(isUseMetadata())
			doc = _database.getEntity(docId);
		if (doc == null) {
			String[] args = new String[2];
			args[0] = docId;
			args[1] = this.getId();
			throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_NO_DOC_EXISTS_DOCBASE, args, null);
		}
		String owners[] = doc.getPropertyValues(relationType);
		if (handler != null && owners != null && owners.length > 0) {
			AttributesImpl atts = new AttributesImpl();
			atts.addAttribute("", Node.Name.NO, Node.Name.NO, Node.Type.CDATA, Integer.toString(owners.length));
			atts.addAttribute("", Node.Name.RELATION, Node.Name.RELATION, Node.Type.CDATA, relationType);
			atts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, docId);
			handler.startElement(Framework.SDXNamespaceURI, Node.Name.OWNERS, Utilities.prefixNodeNameSDX(Node.Name.OWNERS), atts);
			for (int i = 0; i < owners.length; i++) {
				String ownerId = owners[i];
				if (ownerId != null && !"".equals(ownerId)) {
					AttributesImpl atts2 = new AttributesImpl();
					atts2.addAttribute("", Node.Name.ID, Node.Name.ID, "CDATA", ownerId);
					handler.startElement(Framework.SDXNamespaceURI, Node.Name.DOCUMENT, Utilities.prefixNodeNameSDX(Node.Name.DOCUMENT), atts2);
					handler.endElement(Framework.SDXNamespaceURI, Node.Name.DOCUMENT, Utilities.prefixNodeNameSDX(Node.Name.DOCUMENT));
				}
			}
			handler.endElement(Framework.SDXNamespaceURI, Node.Name.OWNERS, Utilities.prefixNodeNameSDX(Node.Name.OWNERS));

		}

		return owners;
	}

	/**Retrieves the related documents (sub, attached, original)
	 * of the parent document for the relationType provided
	 *
	 * @param docId The parent document id
	 * @param relationType	The relation type (sub, attached, original)
	 * @param handler The handler to feed with events
	 * @return String[]
	 * @throws SDXException
	 * @throws SAXException
	 */
	public String[] getRelated(String docId, String relationType, ContentHandler handler) throws SDXException, SAXException {
		if (!Utilities.checkString(docId)) return null;
		DatabaseEntity doc = null;
		if(isUseMetadata())
			_database.getEntity(docId);
		if (doc == null) {
			String[] args = new String[2];
			args[0] = docId;
			args[1] = this.getId();
			throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_NO_DOC_EXISTS_DOCBASE, args, null);
		}
		String[] related = doc.getPropertyValues(relationType);
		if (handler != null && related != null && related.length > 0) {
			AttributesImpl atts = new AttributesImpl();
			atts.addAttribute("", Node.Name.NO, Node.Name.NO, Node.Type.CDATA, Integer.toString(related.length));
			atts.addAttribute("", Node.Name.RELATION, Node.Name.RELATION, Node.Type.CDATA, relationType);
			atts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, docId);
			handler.startElement(Framework.SDXNamespaceURI, relationType, Utilities.prefixNodeNameSDX(relationType), atts);
			for (int i = 0; i < related.length; i++) {
				String relatedId = related[i];
				if (relatedId != null && !"".equals(relatedId)) {
					AttributesImpl atts2 = new AttributesImpl();
					atts2.addAttribute("", Node.Name.ID, Node.Name.ID, "CDATA", relatedId);
					handler.startElement(Framework.SDXNamespaceURI, Node.Name.DOCUMENT, Utilities.prefixNodeNameSDX(Node.Name.DOCUMENT), atts2);
					handler.endElement(Framework.SDXNamespaceURI, Node.Name.DOCUMENT, Utilities.prefixNodeNameSDX(Node.Name.DOCUMENT));
				}
			}
			handler.endElement(Framework.SDXNamespaceURI, relationType, Utilities.prefixNodeNameSDX(relationType));

		}

		return related;

	}


	//TODO : possibly move this method to a LuceneRelationsIndex class or to LuceneDatase -pb
	protected void deleteRelationsToMastersFromDatabase(String relation, Document doc) throws SDXException {
		//deleting properties of database entities which have this document listed as an ATTACHED document
		String l_docId = doc.getId();
		if(isUseMetadata())
			super._database.removeProperty(relation, l_docId);
	}

	protected synchronized int handleParameters(Document doc, Repository repo, IndexParameters params, boolean isIndexable, boolean isPrimary, String relation, ContentHandler handler) throws SDXException, SAXException, ProcessingException {
		//checking to see if the doc exists in the repositories via the repository look-up index
		String docId = doc.getId();
		Repository repoToUse = repo;
		Repository repoFromInternalLookup = null;
		boolean documentExistsInParamRepo = false;
		String repoIDFromInternalLookup = null;
		if(isUseMetadata())
			repoIDFromInternalLookup = _database.getPropertyValue(docId, PROPERTY_NAME_REPO);
		if( Utilities.checkString(repoIDFromInternalLookup) )
			repoFromInternalLookup = this.getRepository(repoIDFromInternalLookup);
		if (repoToUse != null)//we use an repository, check the doc
			documentExistsInParamRepo = repoToUse.exists(docId, getPooledRepositoryConnection((repoToUse.getId())));
		if (!documentExistsInParamRepo && repoFromInternalLookup != null)//we do not have find the doc in the repository, check in the metadata
			repoToUse = repoFromInternalLookup;

		int status = DOC_ADD_STATUS_ADDED;
		//doc exists... handling using params
		int handleSameId = -1;
		if (isPrimary)
			handleSameId = params.handleSameId();
		else {
			//we have a sub document or an attached document
			if (isIndexable)
				handleSameId = params.handleSubDocumentSameId();
			else
				handleSameId = params.handleAttachedDocumentSameId();
		}
		String repoToUseId = "";
		if (repoToUse != null) repoToUseId = repoToUse.getId();
		switch (handleSameId) {
		case IndexParameters.SAME_ID_ERROR:
			status = DOC_ADD_STATUS_FAILURE;
			String[] args = new String[3];
			args[0] = docId;
			args[1] = super.getId();
			args[2] = repoToUseId;
			throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_DOC_EXISTS, args, null);
		case IndexParameters.SAME_ID_IGNORE:
			if(_logger.isInfoEnabled()){
				String[] args2 = {doc.getId()};
				LoggingUtils.logInfo(super.getLog(), SDXExceptionCode.INFO_IGNORING_DOCUMENT_WITH_SAME_ID,args2);
			}
			status = DOC_ADD_STATUS_IGNORED;
			break;
		case IndexParameters.SAME_ID_REFRESH_SHARED_DOCUMENT:
			if (isIndexable && Utilities.checkString(repoIDFromInternalLookup))
				deleteIndexableDocumentComponents((IndexableDocument) doc, handler);//deleting the master document components
			if (Utilities.checkString(repoIDFromInternalLookup) || documentExistsInParamRepo)
				deletePhysicalDocument(doc, repoToUse, handler);//deleting the physical document and it's entry in our lookup database
			status = DOC_ADD_STATUS_REFRESHED;
			break;
		case IndexParameters.SAME_ID_REPLACE:
		default:
			if (Utilities.checkString(repoIDFromInternalLookup)) {
				boolean isShared = !isPrimary;//a secondary document could be shared among primary documents
				delete(doc, isIndexable, isPrimary, isShared, relation, handler);
				status = DOC_ADD_STATUS_REPLACED;
			} else if (!isIndexable && !isPrimary && documentExistsInParamRepo) {
				//a binary document, which at this point exists ONLY (not internally) in the repository passed as a parameter
				deletePhysicalDocument(doc, repoToUse, handler);
				status = DOC_ADD_STATUS_ADDED;
			} else if(!isUseMetadata() || repoToUse==null){
				//the documents base don't use metadata and/or repository, so we have to check for the document, and delete it if necessary
				FieldQuery m_query=new FieldQuery();
				SearchLocations slocs = new SearchLocations();
				// slocs.enableLogging(this._logger);
				slocs.addDocumentBase(this);
				m_query.setUp(slocs, docId, LuceneDataStore.ID_FIELD);
				m_query.prepare();
				fr.gouv.culture.sdx.search.lucene.query.Results res = m_query.execute();
				if ( res != null && res.count() > 0){
					boolean isShared = !isPrimary;//a secondary document could be shared among primary documents
					delete(doc, isIndexable, isPrimary, isShared, relation, handler);
					status = DOC_ADD_STATUS_REPLACED;
				}
				m_query=null;
				slocs=null;
			}
		break;
		}

		return status;
	}

	/**Set's the default pipeline parameters and ensures the params have a pipeline
	 *
	 * @param params The params object provided by the user at indexation time
	 */
	protected IndexParameters setBaseParameters(IndexParameters params) {
		//if we get null params, we create default ones
		if (params == null)
			params = new IndexParameters();
		//providing a super.getLog() for the params
		params.enableLogging(super.getLog());

		//checking pipeline parameters,
		if (params.getPipelineParams() == null)
			params.setPipelineParams(new Parameters());

		//adding the user, if it exists
		fr.gouv.culture.sdx.user.User tmpUser = params.getUser(); //Quick patch to increase performance -pb
		if (tmpUser != null)
			params.getPipelineParams().setParameter(SDX_USER, tmpUser.getId());
		//if their is not a pipline specified we use our default pipeline from the application configuration
		//TODO?:what if this has already been done outside of our code, we reset them after adding our infos, it think it is ok?-rbp
		/*raises problems when the developer reUses the same index parameters for different document bases, once set after the below code,
				then we have no handle on its validity as the correct pipeline*/
		if (params.getPipeline() == null) params.setPipeline(this._indexationPipeline);

		return params;
	}

	/** Adds some documents.
	 *
	 * @param docs The documents to add.
	 * @param repository The repository where to store the documents. If null is passed, the default repository will be used.
	 * @param params The parameters for this adding action.
	 * @param handler A content handler where to send information about the process (may be null)
	 * TODO : what kind of "informations" ? -pb
	 */
	public synchronized void index(IndexableDocument[] docs, Repository repository, IndexParameters params, ContentHandler handler) throws SDXException, SAXException, ProcessingException {

		/** boolean indicating a batch indexation so the the search
		 * index can be optimized only once at the end of the batch,
		 * false by default to indicate single document indexation.
		 */
		//ensuring we have valid parameters
		params = setBaseParameters(params);

		if (handler != null) handler.startElement(Framework.SDXNamespaceURI, Node.Name.UPLOAD_DOCUMENTS, Framework.SDXNamespacePrefix + ":" + Node.Name.UPLOAD_DOCUMENTS, new AttributesImpl());

		long start = System.currentTimeMillis();
		boolean batchIndex = false;
		int docsAdded = 0;
		int docsFailed = 0;
		double batchMax = 0;
		int k = 0;

		// starts logging
		startsIndexationLogging(start, docs.length);
		
		//if we have a very big patch we split them into smaller batches
		if (docs != null && docs.length > 0) {
			batchMax = params.getBatchMax();
			//we reset the batch max if it is a smaller batch
			if (batchMax > docs.length) batchMax = docs.length;
			//we calculate the number of batches needed
			int batches = (int) Math.ceil((docs.length / batchMax));
			//setting the index of the document for indexing in the docs array
			int docIdx = 0;

			//Get the list of XML fields to give to each document. It is more efficient to get it once.
			HashMap xml_field_list = getXMLFieldList();
			/*	Create a new TransformerFactory
			 *	We make sure we use Saxon here, since Xalan has major bugs
			 *	with XML document fragments...*/
			// SAXTransformerFactory tf = new com.icl.saxon.TransformerFactoryImpl();	// Saxon 6
			SAXTransformerFactory tf = new net.sf.saxon.TransformerFactoryImpl();	// Saxon 8
			// Create only one transformer in order to parse the xml fields of all documents.
			TransformerHandler t;
			try{
				t = tf.newTransformerHandler();
			}catch(TransformerConfigurationException e){
				throw new SDXException(this._logger,SDXExceptionCode.ERROR_CREATE_XML_PARSER,null,e);
			}

			IndexableDocument current_doc = null; //just a bit faster

			//selecting a batch
			for (int i = 0; i < batches; i++) { // TODO (MP): Loguer les passes de batch
				//we have batch of documents to index so we set a true state
				batchIndex = true;
				if (docs.length == 1) batchIndex = false;
				//using the same connection to the store for each document
				RepositoryConnection conn = null;
				//if no repository is provided, using the default repository
				if (repository == null) {
					String[] args = new String[1];
					args[0] = super.getId();
					SDXException sdxE = new SDXException(SDXExceptionCode.ERROR_USING_DEFAULT_REPO, args);
					LoggingUtils.logInfo(super.getLog(), sdxE.getMessage());
					//we are using the default because no repository was found
					repository = defaultRepository;
				}

				//ensuring we have a valid repository id to work with
				if (repository != null && !Utilities.checkString(repository.getId()))
					throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_INVALID_REPO_ID, null, null);
				try {

					if (repository != null)
						conn = getPooledRepositoryConnection(repository.getId());

					//indexing documents upto batchMax with the same connection
					for (k = 1; docIdx < docs.length && k <= batchMax; k++) {

						current_doc = docs[docIdx];

						try {
							if(current_doc == null)
								throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_DOC_NULL, null, null);
							//let the document know what are the xml fields
							current_doc.setXMLFieldList(xml_field_list);
							//give it a transformer to handle the xml fields
							current_doc.setXMLTransformerHandler(t);
							//force the transformer to be in UTF-8
							t.getTransformer().setOutputProperty(OutputKeys.ENCODING,"UTF-8");
							// indexation log
							if(_ilevel>0 && _ilogger!=null) _ilogger.setDocRank(k);
							//indexing the next document
							index(current_doc, repository, conn, params, handler, batchIndex);
							docsAdded++;//parent document added
							docsAdded = docsAdded + current_doc.getStoreHandler().getDocs().length;//any subordinate additions
						} catch (SDXException e) {
							docsFailed++;
							// indexation log: add error message only for indexaton logging level gretter than 1
							if(_ilevel>0 && _ilogger!=null) _ilogger.addErrorDoc(k, current_doc.getId(), e.getMessage(this._locale), e);
							if (handler != null && params.getSendIndexationEvents() >= IndexParameters.SEND_ERRORS_EVENTS) {
								AttributesImpl failAtts = new AttributesImpl();
								failAtts.addAttribute("", Node.Name.STATUS, Node.Name.STATUS, Node.Type.CDATA, Node.Value.FAILURE);
								if (current_doc != null && current_doc.getId() != null)
									failAtts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, current_doc.getId());
								if (repository != null)
									failAtts.addAttribute("", Node.Name.REPO, Node.Name.REPO, Node.Type.CDATA, repository.getId());
								failAtts.addAttribute("", Node.Name.BASE, Node.Name.BASE, Node.Type.CDATA, this.getId());
								String appId = Utilities.getStringFromContext(ContextKeys.SDX.Application.ID, super.getContext());
								if (Utilities.checkString(appId))
									failAtts.addAttribute("", Node.Name.APP, Node.Name.APP, Node.Type.CDATA, appId);
								handler.startElement(Framework.SDXNamespaceURI, Node.Name.UPLOAD_DOCUMENT, Framework.SDXNamespacePrefix + ":" + Node.Name.UPLOAD_DOCUMENT, failAtts);
								e.toSAX(handler);
								handler.endElement(Framework.SDXNamespaceURI, Node.Name.UPLOAD_DOCUMENT, Framework.SDXNamespacePrefix + ":" + Node.Name.UPLOAD_DOCUMENT);
							}
							rollbackIndexation(current_doc, handler);
							//catch and then continue adding other docs, message should be already logged, but pass some info's to the handler in the future
						} finally {
							// Setting the docs array to null
							docs[docIdx] = null;
							//incrementing to the next document
							docIdx++;
							/*FIXME: it may be a good idea to optimize here as we don't know how complex a document may be
								and it could have many attached documents/subdocuments
								//optimizing our internal datastructure
								this.optimizeDatabase();
							 */
						}
						// indexation log
						if(_ilevel>0 && _ilogger!=null) _ilogger.setDocEnd(System.currentTimeMillis());
					}
					//TODOLogging:add some summary info and info for each document uploaded successfully

				} finally {
					// Split index managment
					try {
						if(this.splitCheck(true)) this.splitIndex(true);
						//if indexation use batchIndex, merges indexes ; if not, documents are writing into indexes directly
						if (batchIndex) {
							if (_ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_MERGE_BATCH_INDEX);
							mergeCurrentBatch();
						}
//						if(autoOptimize){
//							mergeCurrentBatch(); // INFO (MP): Lance la commande de fusion des index
//							optimize(); // INFO (MP): L'optimisation des index se fait dans le mergeCurrentBatch suivant le parametre autoOptimize
//						}
						//else indexModified(); // INFO (MP): Pas d'optimisation ; on voulait informer de la modification des index. Mais il vaut mieux le faire en fin d'indexation, non ?
						//if DocumentBase is autoOptimize (default) and indexation use batchIndex, optimize the index
					} catch (IOException e) {
						//TODO: handle the exception
					}
				}
			} // end batches
//				FIXME (MP): Pas besoin d'optimiser en fin de batch car la methode mergeCurrentBatch optimise a chaque appel suivant l'option autoOptimize 
			if (autoOptimize) {
				if (_ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_OPTIMIZE_DOCUMENT_BASE);
				optimize();
			}
		}
		long finish = System.currentTimeMillis();
		long elapsed = finish - start;
		indexModified();

		// Split index managment
		try {
			if(this.splitCheck(false)) this.splitIndex(false);
		} catch (IOException e) {
			//TODO: handle the exception
		}
		
		// ends logging
		// TODO (MP): verifier si on doit le faire (ie, un parametre de conf)
		endsIndexationLogging(finish);

		if (handler != null && params.getSendIndexationEvents() >= IndexParameters.SEND_STATS_EVENTS) {
			AttributesImpl summaryAtts = new AttributesImpl();
			summaryAtts.addAttribute("", Node.Name.ADDITIONS, Node.Name.ADDITIONS, Node.Type.CDATA, Integer.toString(docsAdded));
			// summaryAtts.addAttribute("", "replacements", "replacements", ConfigurationNode.Type.CDATA, Integer.toString(docsAdded));
			summaryAtts.addAttribute("", Node.Name.FAILURES, Node.Name.FAILURES, Node.Type.CDATA, Integer.toString(docsFailed));
			summaryAtts.addAttribute("", Node.Name.DURATION, Node.Name.DURATION, Node.Type.CDATA, String.valueOf(elapsed / 1000));
			handler.startElement(Framework.SDXNamespaceURI, Node.Name.SUMMARY, Framework.SDXNamespacePrefix + ":" + Node.Name.SUMMARY, summaryAtts);
			handler.endElement(Framework.SDXNamespaceURI, Node.Name.SUMMARY, Framework.SDXNamespacePrefix + ":" + Node.Name.SUMMARY);
		}

		if (handler != null) handler.endElement(Framework.SDXNamespaceURI, Node.Name.UPLOAD_DOCUMENTS, Framework.SDXNamespacePrefix + ":" + Node.Name.UPLOAD_DOCUMENTS);

	}

	/** Adds a document from an array of documents using the same connection.
	 *
	 * @param doc			The document to add.
	 * @param repository	The repository where to store the document.
	 * @param conn					The repository's connection, we want to use only one connection for all the documents
	 * @param params		 The parameters for this adding action.
	 * @param handler A content handler where to send information about the process (may be null), currently we don't use it, but maybe in the future.
	 * TODO : what kind of "informations" ? -pb
	 */
	protected synchronized void index(IndexableDocument doc, Repository repository, RepositoryConnection conn, IndexParameters params, ContentHandler handler, boolean batchIndex) throws SDXException, SAXException, ProcessingException {
		//ensuring we have a valid document object, not null, but the id can be null so we don't check it like in other cases below
		if (doc == null)
			throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_DOC_NULL, null, null);

		if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_TRANSFORMATION);
		
		if (handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS)
			handler.startElement(Framework.SDXNamespaceURI, Node.Name.UPLOAD_DOCUMENT, Framework.SDXNamespacePrefix + ":" + Node.Name.UPLOAD_DOCUMENT, new AttributesImpl());

		try {
			//we always use our document base's super.getLog() during indexation, if another has been provided it will not be used
			doc.enableLogging(super.getLog());
			// FIXME (MP): doit-on logguer dans le log d'indexation ? doc.enableLogging(_ilogger);

			//setting the id generator properties
			doc.setIdGenerator(this.idGen);
			//giving the document access to the handler for event logging
			if (handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS)
				doc.setMessageHandler(handler);
			
			/*
			 * setting the document's url as a pipe parameter
			 * we should always have pipelineParams at this point
			 * because setBaseParameters has already been called
			 */
			java.net.URL tmpURL = doc.getURL(); //Quick patch to increase performance -pb
			Parameters tmpParameters = params.getPipelineParams(); //Quick patch to increase performance -pb
			String tempUrlString = "";
			if (tmpURL != null)
				tempUrlString = tmpURL.toExternalForm();
			tmpParameters.setParameter(DOC_URL, tempUrlString);
			//adding our defaults
			//getting current date info
			Date date = fr.gouv.culture.sdx.utils.Date.getUtcIso8601Date();
			//formmating a humanReadable date
			tmpParameters.setParameter(SDX_DATE, DateFormat.getDateTimeInstance(DateFormat.FULL, DateFormat.FULL).format(date));
			//formatting an iso 8601 date
			// params.getPipelineParams().setParameter(SDX_FORMATTED_DATE, new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssz").format(date));
			// FG: to get exact system date, formatted by our date utilities
			// params.getPipelineParams().setParameter(SDX_ISO8601_DATE, fr.gouv.culture.sdx.utils.Date.formatDate(new Date()));
			//updated our date utilities to handle the ISP8601 date type
			tmpParameters.setParameter(SDX_ISO8601_DATE, fr.gouv.culture.sdx.utils.Date.formatUtcISO8601Date(date));
			tmpParameters.setParameter(SDX_DATE_MILLISECONDS, String.valueOf(date.getTime()));

			//if current doc is an OAIDocument, add oai parameters: sdxoaidate and sdxoaiid
			if( doc instanceof OAIDocument ){
				tmpParameters.setParameter( OAIDocument.INTERNAL_FIELD_NAME_SDXOAIDATE, ((OAIDocument)doc).getDateString() );
				tmpParameters.setParameter( OAIDocument.INTERNAL_FIELD_NAME_SDXOAIID, ((OAIDocument)doc).getIdentifier() );
			}

			//setting the parameters to the pipeline
			//TODO?:what if this has already been done outside of our code, we reset them after adding our infos,
			//I think it is ok for now, but maybe bad in some complex cases-rbp
			fr.gouv.culture.sdx.pipeline.Pipeline tmpPipeline = params.getPipeline();	 //Quick patch to increase performance -pb
			tmpPipeline.setParameters(tmpParameters);
			//indexing document using the pipeline from the index params
			tmpPipeline.setConsumer(doc);
			SAXParser parser = null;
			try {
				if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_TRANSFORMATION_START);
				parser = (SAXParser) super._manager.lookup(SAXParser.ROLE);
				doc.startIndexing(parser, tmpPipeline);
				//after parsing we want to retrieve any transformed documents
				byte[] tmpBytes = tmpPipeline.getTransformedBytes(); //Quick patch to increase performance -pb
				File tmpFile = tmpPipeline.getTransformedFile(); //Quick patch to increase performance -pb
				if (tmpBytes != null)
					doc.setTransformedDocument(tmpBytes);
				else if (tmpFile != null)
					doc.setTransformedDocument(tmpFile);
				//TODO : potential other case here -pb
				if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_TRANSFORMATION_END);
			} catch (ServiceException e) {
				String[] args = new String[1];
				args[0] = e.getMessage();
				// indexation log: add error message only for indexaton logging level gretter than 1
				if(_ilevel>1 && _ilogger!=null) _ilogger.addErrorDoc( this._ilogger.getDocRank() , doc.getId(), args[0], e );
				throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_ACQUIRE_PARSER, args, e);
			} finally {
				if (parser != null) super._manager.release(parser);
			}

			add(batchIndex, doc, repository, conn, params, true, true, null, null, handler, null);
		} finally {
			if (handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS)
				handler.endElement(Framework.SDXNamespaceURI, Node.Name.UPLOAD_DOCUMENT, Framework.SDXNamespacePrefix + ":" + Node.Name.UPLOAD_DOCUMENT);
		}


	}

	//SDX search architecture/Lucene specific
	/**Adds a document to the search index of Lucene
	 *
	 * @param originalDoc The document to add
	 * @param repository The repository to use
	 * @param conn	The repository connection to use
	 * @param params The document addition parameters
	 * @param handler		The content handler to feed with events
	 * @param isIndexable		is the document indexable
	 * @param isPrimary			is the document a primary or secondary document
	 * @param batchIndex		is this a batch index
	 * @throws SDXException
	 * @throws SAXException
	 * @throws ProcessingException
	 */

	protected synchronized void add(boolean batchIndex, Document originalDoc, Repository repository, RepositoryConnection conn, IndexParameters params, boolean isIndexable, boolean isPrimary, String relation, String parentId, ContentHandler handler, String attachedDocId) throws SDXException, SAXException, ProcessingException {
		//ensuring we have a valid document
		Utilities.checkDocument(super.getLog(), originalDoc);

		if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_ADD);

		//boolean handleConnection = false;
		Repository storeRepo = repository;
		RepositoryConnection storeRepoConn = conn;
		int status = -1;
		String repoId = "";
		//setting the status based upon parameters handled
		status = handleParameters(originalDoc, repository, params, isIndexable, isPrimary, relation, handler);
		if (status < DOC_ADD_STATUS_ADDED) return;//if the document exists and shouldnt be added exit
		if (handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS)
			handler.startElement(Framework.SDXNamespaceURI, Node.Name.ADDITION, Framework.SDXNamespacePrefix + ":" + Node.Name.ADDITION, new AttributesImpl());
		try {

			//some set-up for accessing the repository
			/*
			 * TODO : check if the *provided* connection if fit for the repository ?
			 * the opportunity to default the repository could result in providing an irrelevant connection -pb
			 */
			if (storeRepo == null) storeRepo = defaultRepository;

			//determining the storage repository
			storeRepo = getRepositoryForStorage(originalDoc, storeRepo);

			if (storeRepo != null)
				repoId = storeRepo.getId();

			//if the provided repo is not the desired store repo, or the connection provided was null we handle the connection
			if (storeRepo != null && (storeRepoConn == null || storeRepo != repository)) {
				storeRepoConn = getPooledRepositoryConnection(storeRepo.getId());
			}


			Document docToStore = originalDoc; //By default, we store the original document
			String docToStoreId = "";
			boolean storeTransformed = false;

			//look-up the system database to get the old parents' ids of the current document if its status is "refreshed"
			// TODO (mp): maintenir une liste des identifiants des documents ajoutes dans ce meme batch en plus des identifiants des documents ajoutes lors de precedents batch ou de precedentes publication
			DatabaseEntity docLookupEntry = null;
			//TODO: use inner class when javac compiler bug is fixed-rbp
			//super.lookupDatabase.addDocMetaDataToDatabase(docToStore, repository);
			String[] parentDocIds = new String[1];
			parentDocIds[0] = parentId;//add the new parent id
			if (status == DOC_ADD_STATUS_REFRESHED) {
				DatabaseEntity entityToRefresh = null;
				if(isUseMetadata())
					entityToRefresh = super._database.getEntity(docToStore.getId());
				if (entityToRefresh != null){
					String[] oldParents = entityToRefresh.getPropertyValues(PROPERTY_NAME_PARENT);
					parentDocIds = new String[oldParents.length + 1];
					for (int i=0 ;i < oldParents.length; i++)
						parentDocIds[i+1] = oldParents[i];
				}
			}

			if (attachedDocId!= null) docToStoreId = attachedDocId;

			if(isUseMetadata())
				docLookupEntry = createEntityForDocMetaData(docToStore, storeRepo, parentDocIds);



			/*
			 * this flag replaces the tests on (doc instanceof IndexableDocument)
			 * some instances of IndexableDocument (e.g. TransformedDocuments) are *not* to be indexed)
			 * TODO : find a better name and improve document -pb
			 */
			if (isIndexable) {

				//Code safe here : null is assumed in case we can't get a transformed document
				IndexableDocument transformedDoc = ((IndexableDocument) originalDoc).getTransformedDocument();

				//TODO : make the code below more generic by providing a isWrtiable() method to repositories
				//only writable repositories can store original documents -pb

				//determining which document to store
				if (transformedDoc != null && !(storeRepo instanceof URLRepository) && Utilities.checkString(transformedDoc.getId())) {
					//modifying the original object's id and filename
					originalDoc.setId("o_" + originalDoc.getId());
					// FIXME: I put the following comment because it doesn't work when getPreferredFilename returns null
//					originalDoc.setPreferredFilename("o_" + originalDoc.getPreferredFilename());
					docToStore = transformedDoc; //store the transformed
					storeTransformed = true;
				}

				docToStoreId = docToStore.getId();
				if(isPrimary && _ilevel>0 && _ilogger!=null) _ilogger.setDocID(docToStoreId); // on ne loggue les ID que des documents principaux, pas des sous-documents

				/*
				 * STEP 1 : adding the original document, if relevant
				 */
				if (storeTransformed && (params.isSaveOriginalDocument() && this.keepOriginalDocuments)) {
					if (handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS)
						handler.startElement(Framework.SDXNamespaceURI, PROPERTY_NAME_ORIGINAL, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_ORIGINAL, new AttributesImpl());
					try {
						//Notice that the original document look-up entry will be added as well
						if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_ADD_STORE_ORIGINAL_DOCUMENT);
						this.add(batchIndex, originalDoc, repository, conn, params, false, false, PROPERTY_NAME_ORIGINAL, null, handler, null); //Recursive call -pb
						((IndexableDocument) docToStore).getStoreHandler().addDoc(originalDoc);
						//adding the relation between the document and the original document
						//TODO: use inner class when javac compiler bug is fixed-rbp
						//super.lookupDatabase.addDocRelationToDatabase(docToStore, originalDoc, RELATION_TYPE_ORIGINAL, null);
						if(isUseMetadata())
							docLookupEntry.addProperty(PROPERTY_NAME_ORIGINAL, originalDoc.getId());
					} finally {
						if (handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS)
							handler.endElement(Framework.SDXNamespaceURI, PROPERTY_NAME_ORIGINAL, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_ORIGINAL);
					}


				}

				/*
				 * STEP 2 : adding the attached documents, if relevant
				 */
				Enumeration attachedDocs = ((IndexableDocument) originalDoc).getAttachedDocuments(); // TODO (MP): loguer le nombre de docs attaches
				if (attachedDocs != null) {
					try {
						if(_ilevel>0 && _ilogger!=null) _ilogger.setDocNbAttached( ( (AbstractIndexableDocument) originalDoc ).getAttachedDocumentsSize() );
					} catch (ClassCastException cce) {
						/*nothing todo */;
					}
					int attDocsAdded = 0;
					boolean startElemSent = false;
					try {
						while (attachedDocs.hasMoreElements()) {
							if (attDocsAdded == 0 && handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS) {
								handler.startElement(Framework.SDXNamespaceURI, PROPERTY_NAME_ATTACHED, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_ATTACHED, new AttributesImpl());
								startElemSent = true;
							}
							//getting the attached document
							BinaryDocument attachedDoc = (BinaryDocument) attachedDocs.nextElement();
							//Notice that the attached document look-up entry will be added as well
							if(_ilevel>0 && _ilogger!=null) _ilogger.setDocAttachedRank(attDocsAdded + 1);
							this.add(batchIndex, attachedDoc, repository, conn, params, false, false, PROPERTY_NAME_ATTACHED, docToStoreId, handler, attachedDoc.getId()); //Recursive call -pb
							attDocsAdded++;
							((IndexableDocument) docToStore).getStoreHandler().addDoc(attachedDoc);
							//adding the relation between the document and the attached document
							//TODO : consider *ordered* attached documents ; it could be useful -pb
							//TODO: use inner class when javac compiler bug is fixed-rbp
							//super.lookupDatabase.addDocRelationToDatabase(docToStore, attachedDoc, RELATION_TYPE_ATTACHED, null);
							if(isUseMetadata())
								docLookupEntry.addProperty(PROPERTY_NAME_ATTACHED, attachedDoc.getId());
							/*TODO: pierrick please see the handleParams() method and see if we can add
							 *any other proper behavior there
							 * WARNING :
							 * This implementation assumes that the attached document
							 * is the same than the one that is already stored.
							 * One can imagine that, for a *same* id, other properties
							 * (mimetype, data) could be different.
							 * We should be able to use IndexParamters and/or attachedDocLookupEntry
							 * to determine the different possible behaviours :
							 * always replace
							 * never replace (ignore)
							 * create new instance (in which case, the id management won't be easy)
							 * replace if "master document" (document to be implemented)
							 * ...
							 *
							 * For now, we do nothing here since the attached document
							 * is already present, probably (certainly ;-) owned by other documents -pb
							 */
							//if (_indexationLoggerLevel <= Priority.DEBUG_INT) {
							//	_ilogger.debug("Ajout du document attaché : "+attachedDoc.getId());
							//}
						}
					} finally {
						if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_ADD_ATTACHED_END);
						if (startElemSent && handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS) {
							AttributesImpl summaryAtts = new AttributesImpl();
							summaryAtts.addAttribute("", Node.Name.ADDITIONS, Node.Name.ADDITIONS, Node.Type.CDATA, Integer.toString(attDocsAdded));
							handler.startElement(Framework.SDXNamespaceURI, Node.Name.SUMMARY, Framework.SDXNamespacePrefix + ":" + Node.Name.SUMMARY, summaryAtts);
							handler.endElement(Framework.SDXNamespaceURI, Node.Name.SUMMARY, Framework.SDXNamespacePrefix + ":" + Node.Name.SUMMARY);
							handler.endElement(Framework.SDXNamespaceURI, PROPERTY_NAME_ATTACHED, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_ATTACHED);
						}
					}
				}


				/*
				 * STEP 3 : adding the sub-documents, if relevant
				 */
				Enumeration subDocs = ((IndexableDocument) originalDoc).getSubDocuments();
				if (subDocs != null && subDocs.hasMoreElements()) {
					try {
						if(_ilevel>0 && _ilogger!=null) _ilogger.setDocNbSubdocs( ( (AbstractIndexableDocument) originalDoc ).getSubDocumentsSize() );
					} catch (ClassCastException cce) {
						/*nothing todo */;
					}
					int subDocsAdded = -1;
					boolean startElemSent = false;
					try {
						while (subDocs.hasMoreElements()) {
							if (subDocsAdded == 0 && handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS) {
								handler.startElement(Framework.SDXNamespaceURI, PROPERTY_NAME_SUB, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_SUB, new AttributesImpl());
								startElemSent = true;
							}
							//getting the sub-document
							IndexableDocument subDoc = (IndexableDocument) subDocs.nextElement();
							//Notice that the sub-document look-up entry will be added as well
							if(_ilevel>0 && _ilogger!=null) _ilogger.setDocSubdocRank(subDocsAdded + 1);
							this.add(batchIndex, subDoc, repository, conn, params, true, false, PROPERTY_NAME_SUB, docToStoreId, handler, null); //Recursive call -pb
							subDocsAdded++;
							((IndexableDocument) docToStore).getStoreHandler().addDoc(subDoc);
							//adding the relation between the document and the attached document
							//TODO: use inner class when javac compiler bug is fixed-rbp
							//super.lookupDatabase.addDocRelationToDatabase(docToStore, subDoc, RELATION_TYPE_SUB, null);
							if(isUseMetadata())
								docLookupEntry.addProperty(PROPERTY_NAME_SUB, subDoc.getId());
							/*TODO: pierrick please see the handleParams() method and see if we can add
							 *any other proper behavior there
							 * WARNING :
							 * This implementation assumes that the attached document
							 * is the same than the one that is already stored.
							 * One can imagine that, for a *same* id, other properties
							 * (mimetype, data) could be different.
							 * We should be able to use IndexParamters and/or attachedDocLookupEntry
							 * to determine the different possible behaviours :
							 * always replace
							 * never replace (ignore)
							 * create new instance (in which case, the id management won't be easy)
							 * replace if "master document" (document to be implemented)
							 * ...
							 *
							 * For now, we do nothing here since the attached document
							 * is already present, probably (certainly ;-) owned by other documents -pb
							 */
						}
					} finally {
						if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_ADD_SUBDOCUMENTS_END);
						if (startElemSent && handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS) {
							AttributesImpl summaryAtts = new AttributesImpl();
							summaryAtts.addAttribute("", Node.Name.ADDITIONS, Node.Name.ADDITIONS, Node.Type.CDATA, Integer.toString(subDocsAdded));
							handler.startElement(Framework.SDXNamespaceURI, Node.Name.SUMMARY, Framework.SDXNamespacePrefix + ":" + Node.Name.SUMMARY, summaryAtts);
							handler.endElement(Framework.SDXNamespaceURI, Node.Name.SUMMARY, Framework.SDXNamespacePrefix + ":" + Node.Name.SUMMARY);
							handler.endElement(Framework.SDXNamespaceURI, PROPERTY_NAME_SUB, Framework.SDXNamespacePrefix + ":" + PROPERTY_NAME_SUB);
						}
					}
				}

				/*
				 * STEP 4 : adding the document's indexation to the search index
				 */
				/*possibly differ this step -pb
				 * yes i differed it as we don't want to store anything
				 * about the indexation document until its parts are
				 * sucessfully added
				 * Well... I mean the possibilty to delay the actual indexing of documents.
				 * Currently, this method does two things :
				 * 1) delegate document management to the relevant repositories
				 * 2) creadocToStoreIdte an indexation document and store it in the (Lucene) searchIndex
				 * As point 2 is somewhat ressource consuming, it would be nice to process it
				 * when server gets idle -pb
				 */

				/*
				 * Resolution of :
				 * a) document id
				 * b) attached documents info
				 * c) search index document
				 */
				Object indexationDoc = getIndexationDocument((IndexableDocument) originalDoc, docToStoreId, repoId, params);
				//TOTOpb : understand how ce could have gotten a valid id before this :-)
				//ANSWER: auto id generation or id set by user during pipeline (ex. XsltTransformatin)-rbp

				if (indexationDoc != null) {
					if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_ADD_TO_SEARCH_INDEX);
					addToSearchIndex(indexationDoc, batchIndex);//adding the current doc to the search index
//					removeOaiDeletedRecord((IndexableDocument) originalDoc);//removing the current doc from the harvester repo
					if (this.oaiRepositories != null && !this.oaiRepositories.isEmpty()) {
						if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_ADD_OAI_DELETED_RECORD);
						removeOaiDeletedRecord((IndexableDocument) docToStore);
					}
				}
			}


			/*
			 * STEP 5 : adding the document itself
			 */
			if (storeRepo != null) {
				if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_ADD_TO_REPOSITORY);
				storeRepo.add(docToStore, storeRepoConn);
			}


			/*
			 * STEP 6 : adding the document's entry to the look-up index
			 */
			if(isUseMetadata()) {
				if(_ilevel>0 && _ilogger!=null) _ilogger.setStep(LoggingIndexation.STEP_DOCUMENT_ADD_UPDATE_METADATA);
				super._database.update(docLookupEntry);
			}


			/*
			 * STEP 7: sending our info events
			 */


			if (handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS) {
				AttributesImpl addAtts = new AttributesImpl();
				addAtts.addAttribute("", Node.Name.STATUS, Node.Name.STATUS, Node.Type.CDATA, String.valueOf(_documentAdditionStatus[status]));
				addAtts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, docToStoreId);
				addAtts.addAttribute("", Node.Name.REPO, Node.Name.REPO, Node.Type.CDATA, repoId);
				addAtts.addAttribute("", Node.Name.BASE, Node.Name.BASE, Node.Type.CDATA, this.getId());
				String appId = Utilities.getStringFromContext(ContextKeys.SDX.Application.ID, super.getContext());
				if (Utilities.checkString(appId))
					addAtts.addAttribute("", Node.Name.APP, Node.Name.APP, Node.Type.CDATA, appId);
				String mimeType = docToStore.getMimeType();
				if (Utilities.checkString(mimeType))
					addAtts.addAttribute("", Node.Name.MIMETYPE, Node.Name.MIMETYPE, Node.Type.CDATA, mimeType);

				String length = Integer.toString(docToStore.getLength());
				if (Utilities.checkString(length))
					addAtts.addAttribute("", Node.Name.BYTE_LENGTH, Node.Name.BYTE_LENGTH, Node.Type.CDATA, length);

				handler.startElement(Framework.SDXNamespaceURI, Node.Name.DOCUMENT, Framework.SDXNamespacePrefix + ":" + Node.Name.DOCUMENT, addAtts);
				handler.endElement(Framework.SDXNamespaceURI, Node.Name.DOCUMENT, Framework.SDXNamespacePrefix + ":" + Node.Name.DOCUMENT);
			}
		} finally {
			if (handler != null && params.getSendIndexationEvents() == IndexParameters.SEND_ALL_EVENTS)
				handler.endElement(Framework.SDXNamespaceURI, Node.Name.ADDITION, Framework.SDXNamespacePrefix + ":" + Node.Name.ADDITION);
		}
	}

	public void init() throws SDXException {
		super.init();
		if(isUseMetadata())
			addParentRelationsToChildren();
	}

	/**
	 * Returns the list of XML type fields
	 */
	public abstract HashMap getXMLFieldList();

	/**This method adds information about each parent document to the child document(s)
	 * data structure instance. As a result deletion of documents becomes faster as
	 * an extra search does not have to be performed to determine if a child document is
	 * used by multiple parent documents.
	 * @since 2.3
	 * @see #init
	 */
	private void addParentRelationsToChildren() throws SDXException {
		final Logger l_finalLogger = super.getLog();
		//work only for document bases wich use metadata (default)
		if(isUseMetadata()){
			new Thread() {
				public void run() {
					try {

						DatabaseEntity dbFormat = _database.getEntity(SDX_DATABASE_FORMAT);
						if (dbFormat != null && dbFormat.containsValue(SDX_DATABASE_VERSION, SDX_DATABASE_VERSION_2_3))
							return;//the database is already compliant we exit

						//searching for all entities that reference a child "attached" document
						Parameters l_attachedDocParams = new Parameters();
						l_attachedDocParams.setParameter(PROPERTY_NAME_ATTACHED, _database.getWildcardSearchToken());
						String[] l_attachedDocParents = _database.search(l_attachedDocParams);

						for (int i = 0; i < l_attachedDocParents.length; i++) {
							DatabaseEntity l_entity = _database.getEntity(l_attachedDocParents[i]);
							String l_entityId = l_entity.getId();
							String[] l_attachedDocsIds = l_entity.getPropertyValues(PROPERTY_NAME_ATTACHED);
							for (int j = 0; j < l_attachedDocsIds.length; j++) {
								String l_attachedDocId = l_attachedDocsIds[j];
								DatabaseEntity l_attachedDoc = _database.getEntity(l_attachedDocId);
								if (l_attachedDoc != null && l_attachedDoc.containsValue(PROPERTY_NAME_PARENT, l_entityId))
									_database.addProperty(l_attachedDocId, PROPERTY_NAME_PARENT, l_entityId);
							}
						}

						//searching for all entities that reference a child "sub" document
						Parameters l_subDocParams = new Parameters();
						l_attachedDocParams.setParameter(PROPERTY_NAME_SUB, _database.getWildcardSearchToken());
						String[] l_subDocParents = _database.search(l_subDocParams);

						for (int i = 0; i < l_subDocParents.length; i++) {
							DatabaseEntity l_entity = _database.getEntity(l_subDocParents[i]);
							String l_entityId = l_entity.getId();
							String[] l_subDocsIds = l_entity.getPropertyValues(PROPERTY_NAME_SUB);
							for (int j = 0; j < l_subDocsIds.length; j++) {
								String l_subDocId = l_subDocsIds[j];
								DatabaseEntity l_subDoc = _database.getEntity(l_subDocId);
								if (l_subDoc != null && l_subDoc.containsValue(PROPERTY_NAME_PARENT, l_entityId))
									_database.addProperty(l_subDocId, PROPERTY_NAME_PARENT, l_entityId);
							}
						}

						//adding an entry to indicate that the database is updated to version 2.3
						DatabaseEntity dbe = new DatabaseEntity(SDX_DATABASE_FORMAT);
						dbe.addProperty(SDX_DATABASE_VERSION, SDX_DATABASE_VERSION_2_3);
						_database.save(dbe);


					} catch (SDXException e) {
						l_finalLogger.error(e.getMessage(), e);
					}
				}
			}.start();
		}
	}

	protected boolean initToSax(){
		if(!super.initToSax())
			return false;
		else{
			this._xmlizable_objects.put("Auto_Optimize",String.valueOf(this.autoOptimize));
			this._xmlizable_objects.put("Split_Size", String.valueOf(this.getSplitSize()));
			this._xmlizable_objects.put("Split_Unit", String.valueOf(this.splitUnit));
			this._xmlizable_objects.put("Split_Doc", String.valueOf(this.splitDoc));
			this._xmlizable_objects.put("Keep_Original_Documents",String.valueOf(this.keepOriginalDocuments));
			if(isUseMetadata())
				this._xmlizable_objects.put("SDX_DocumentBase_DataBase",this._database);
			this._xmlizable_objects.put("SDX_DocumentBase_Repositories",this.repositories);
			this._xmlizable_objects.put("SDX_OAI_Harvester",this._oaiHarv);

			return true;
		}
	}

	/**Init the LinkedHashMap _xmlizable_volatile_objects with the objects in order to describ them in XML
	 * Some objects need to be refresh each time a toSAX is called*/
	protected void initVolatileObjectsToSax() {
		super.initVolatileObjectsToSax();
		this._xmlizable_objects.put("Auto_Optimize",String.valueOf(this.autoOptimize));
		this._xmlizable_objects.put("Split_Size", String.valueOf(this.getSplitSize()));
		this._xmlizable_objects.put("Split_Unit", String.valueOf(this.splitUnit));
		this._xmlizable_objects.put("Split_Doc", String.valueOf(this.splitDoc));
		this._xmlizable_objects.put("Keep_Original_Documents",String.valueOf(this.keepOriginalDocuments));
		this._xmlizable_objects.put("UseMetadata",String.valueOf(isUseMetadata()));
	}

	/**
	 * Process an optimization of the Documentbase
	 */
	public abstract void optimize();

	/**
	 * Merges a batch of documents (in memory) into the physical index on the file system.
	 * @deprecated This method is deprecated since SDX v. 2.3. Use mergeCurrentBatch() instead.
	 */
	public abstract void mergeBatch() throws SDXException;

	/**
	 * Merges a batch of documents (in memory) into the physical index on the file system and optimize this one if necessary (depends of the autoOptimize attribute for the current Document Base)
	 */
	public abstract void mergeCurrentBatch();

	public abstract void indexModified();

	/**
	 * Return true when splitting condition are reached
	 * if true, should be followed by a splitIndex() call
	 */
	public abstract boolean splitCheck(boolean currentIndex) throws SDXException;

	/**
	 * Split the current big index into 2 smaller one
	 */
	public abstract void splitIndex(boolean currentIndex) throws IOException, SDXException;

	/**
	 * @return boolean	tell us if the documentBase index is optimized or not
	 */
	final public boolean isIndexOptimized()
	{
		return _isIndexOptimized;
	}

	/**Configures time triggers for optimizations
	 *
	 * @param optimizeConf		The configuration for optimizations
	 * @throws ConfigurationException
	 */
	protected void configureOptimizeTriggers(Configuration optimizeConf) throws ConfigurationException {
		if(optimizeConf == null || autoOptimize == true) return;

		TimeTrigger trigger = new TimeTriggerFactory().createTimeTrigger(optimizeConf);
		if(trigger == null) return;

		String[] args = {this._id, trigger.toString()};
		LoggingUtils.logInfo(_logger, SDXExceptionCode.INFO_CONFIGURE_DOCUMENTBASE_OPTIMIZATION,args);

		if(this.scheduler == null) this.scheduler = new SimpleTimeScheduler();
		if(this.scheduler != null)
		{
			this.scheduler.addTrigger("optimization", trigger, this);
			this.scheduler.start();
		}
	}


	/**
	 * Optimization by the cron
	 */
	public void targetTriggered(String triggeredString)
	{
		if(triggeredString.equalsIgnoreCase("optimization")){
			String[] args = {this._id};
			LoggingUtils.logInfo(_logger, SDXExceptionCode.INFO_LAUNCH_DOCUMENTBASE_OPTIMIZATION, args);
			this.optimize();
			LoggingUtils.logInfo(_logger, SDXExceptionCode.INFO_FINISHED_DOCUMENTBASE_OPTIMIZATION, args);
		}
	}

	/**
	 * Check the integrity of the documentBase
	 */
	public void checkIntegrity()
	{
		//TODO: implement
	}

	/** Save the DocumentBase data objects
	 * @see fr.gouv.culture.sdx.utils.save.Saveable#backup(fr.gouv.culture.sdx.utils.save.SaveParameters)
	 */
	public void backup(SaveParameters save_config) throws SDXException {
		super.backup(save_config);
		if(save_config != null)
		{
			String db_path = save_config.getUniqueIDString() + "_db";
			if(save_config.isAllElementSelected())
			{
				// Create the documentBases/id_db/ directory
				File db_dir = new File(save_config.getStoreCompletePath() + File.separator + db_path);
				if(!db_dir.exists())
					db_dir.mkdir();
				//add the relative path to the config file
				save_config.savePathInConfig(db_path);

				//add the database to the config file and save it
				SaveParameters saveparams = new SaveParameters(Utilities.getElementName(Database.CLASS_NAME_SUFFIX),save_config, db_path);
				if(isUseMetadata())
					_database.backup(saveparams);

				if((repositories!=null) && (repositories.size()>0))
				{
					//add the repositories to the configfile
					saveparams = new SaveParameters(Utilities.getElementName(Repository.ConfigurationNode.REPOSITORIES),save_config,db_path);
					backupRepositories(saveparams);
				}

				//save the indexes
				SaveParameters indexes_save = new SaveParameters(Utilities.getElementName(fr.gouv.culture.sdx.search.lucene.query.Index.CLASS_NAME_SUFFIX)+"es", save_config,db_path);
				this.backupIndexes(indexes_save);

				//save the timestamps
				SaveParameters timeStamp_save = new SaveParameters("timestamps", indexes_save);
				this.backupTimeStamp(timeStamp_save);
			}
		}
	}

	protected abstract void backupIndexes(SaveParameters save_config) throws SDXException;

	protected abstract void backupTimeStamp(SaveParameters save_config) throws SDXException;

	private void backupRepositories(SaveParameters save_config) throws SDXException{
		//create the documentBases/id_db/repositories/ directory
		if(repositories != null && repositories.size()>0)
		{
			String repos_path = Repository.ConfigurationNode.REPOSITORIES;
			File repos_dir = new File(save_config.getStoreCompletePath() + File.separator + repos_path);
			if(!repos_dir.exists())
				repos_dir.mkdir();

			for(Enumeration enumeration = repositories.elements(); enumeration.hasMoreElements();)
			{
				Repository repo = (Repository)enumeration.nextElement();
				if(repo != null){
					//save each repository
					SaveParameters reposave = new SaveParameters(Utilities.getElementName(Repository.CLASS_NAME_SUFFIX),save_config, repos_path);
					reposave.setUniqueID(save_config.getUniqueID());
					repo.backup(reposave);
				}
			}
		}
	}

	/** Restore the DocumentBase data objects
	 * @see fr.gouv.culture.sdx.utils.save.Saveable#restore(fr.gouv.culture.sdx.utils.save.SaveParameters)
	 */
	public void restore(SaveParameters save_config) throws SDXException {
		super.restore(save_config);
		if(save_config != null)
		{

			if(save_config.isAllElementSelected())
			{

				//add the database to the config file and save it
				SaveParameters saveparams = (SaveParameters)save_config.getChild(Utilities.getElementName(Database.CLASS_NAME_SUFFIX));
				if(isUseMetadata())
					_database.restore(saveparams);

				if((repositories!=null) && (repositories.size()>0))
				{
					//restore the repositories from the configfile
					saveparams = (SaveParameters)save_config.getChild(Utilities.getElementName(Repository.ConfigurationNode.REPOSITORIES));
					restoreRepositories(saveparams);
				}

				//restore the indexes
				SaveParameters indexes_save = (SaveParameters)save_config.getChild(Utilities.getElementName(fr.gouv.culture.sdx.search.lucene.query.Index.CLASS_NAME_SUFFIX)+"es");
				this.restoreIndexes(indexes_save);

				//restore the timestamps
				SaveParameters timeStamp_save = (SaveParameters)indexes_save.getChild("timestamps");
				this.restoreTimeStamp(timeStamp_save);
			}
		}
	}

	private void restoreRepositories(SaveParameters save_config) throws SDXException{
		if(repositories != null && repositories.size()>0)
		{

			for(Enumeration enumeration = repositories.elements(); enumeration.hasMoreElements();)
			{
				Repository repo = (Repository)enumeration.nextElement();
				if(repo != null){
					//save each repository
					SaveParameters reposave = save_config.getSaveParametersById(repo.getId());
					repo.restore(reposave);
				}
			}
		}
	}
	
	public LoggingIndexation getIndexationLogger() {
		return _ilogger;
	}
	
	public void getIndexationInformations() {
		if(_ilevel>0 && _ilogger!=null) {
			_ilogger.getXMLInformation();
		}
	}
	
	private void startsIndexationLogging(long start, int nbdocs) {
		if (_ilevel <= 0) return;
		try {
			// configurer le logger
			_ilogger = new LoggingIndexation();
			//setting the super.getLog()
            Utilities.setUpSdxObject(_ilogger, _logger, _context, _manager);
            _ilogger.configure(_configuration, _id, start, nbdocs, null, null);
//			_ilogger = new LoggingIndexation(this._configuration, _id, start, nbdocs);
//			_ilogger.enableLogging(_logger);
            // lancer le log
			_ilogger.log();
            // _ilogger.setStart(start);//TODO (MP): logguer le depart et la fin dans SDXlog
		} catch (ConfigurationException ce) {
			if (_logger!=null && _logger.isWarnEnabled()) {
				_logger.warn(ce.getMessage(), ce.fillInStackTrace());
			}
		}
	}

	private void endsIndexationLogging(long end) {
		if(_ilevel>0 && _ilogger!=null) {
			_ilogger.setEnd(end);
		}
	}
	

	protected abstract void restoreIndexes(SaveParameters save_config) throws SDXException;

	protected abstract void restoreTimeStamp(SaveParameters save_config) throws SDXException;
	
}
