/*
Copyright (C) 2000-2010  Ministere de la culture et de la communication (France), AJLSM
See LICENCE file
 */
package fr.gouv.culture.sdx.search.lucene.query;

import fr.gouv.culture.sdx.documentbase.LuceneDocumentBase;
import fr.gouv.culture.sdx.exception.SDXException;
import fr.gouv.culture.sdx.exception.SDXExceptionCode;
import fr.gouv.culture.sdx.framework.Framework;
import fr.gouv.culture.sdx.search.lucene.DateField;
import fr.gouv.culture.sdx.search.lucene.Field;
import fr.gouv.culture.sdx.search.lucene.filter.Criteria;
import fr.gouv.culture.sdx.search.lucene.filter.Filter;
import fr.gouv.culture.sdx.utils.Bits;
import fr.gouv.culture.sdx.utils.Date;
import fr.gouv.culture.sdx.utils.Utilities;
import fr.gouv.culture.sdx.utils.constants.Node;
import fr.gouv.culture.sdx.utils.logging.LoggingUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.QueryWrapperFilter;
import org.apache.lucene.search.regex.JavaUtilRegexCapabilities;
import org.apache.lucene.search.regex.RegexCapabilities;
import org.apache.lucene.search.regex.RegexTermEnum;
import org.apache.regexp.RE;
import org.apache.regexp.RESyntaxException;
import org.xml.sax.ContentHandler;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

import java.io.IOException;
import java.text.Collator;
import java.util.*;

/*
 *	Une liste de termes de la base de documents.
 *
 *	Les termes sont des valeurs uniques de champs. On peut construire une
 *	liste de tous les termes, une liste de termes qui respectent une
 *	expression reguliere (dans le but de gerer la troncature), une
 *	liste de termes pour un champ, ainsi qu'une liste de termes pour
 *	un champ en fonction d'une requete de recherche.
 */

/** A list of terms for a document base.
 *
 * The terms are single values of fields. One can build one
 * list of all the terms, a list of terms which respect one
 * regular expression (with an aim of managing truncation), one
 * list of terms for a field, as well as a list of terms for
 * a field according to a search acceptRequest.
 *
 */
/* TODO (MP) : Brancher le mécanisme de RegexCapabilities pour augmenter la rapidité
 *             de construction des listes de termes avec troncature.
 */
public class Terms extends AbstractResponse implements fr.gouv.culture.sdx.search.Terms {

    /** The ordered list of terms. */
    protected TreeMap termList;

    /** The fields of filters. */
    private String[] fieldFilters;

    /** The values of filters. */
//    FIXME : private field never used locally [MP]
    private String[] valueFilters;

    /** THe filter used for the document base. */
    private Filter filter;

    /** The object of comparison for sorting  */
    protected static Collator sortCollator;

    /**
     *	Builds a term list.
     *
     *  @param	sLocs	The SearchLocations object (indices to be searched).
     *	@param	field	The field name.
     */
    public void setUp(SearchLocations sLocs, String field) throws SDXException {
        if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);

        if (!Utilities.checkString(field)) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);
        setFieldFilter(field);
        //interning the field name for better performance with lucene
        String internedfieldName = field.intern();

        super.setSearchLocations(sLocs);

        initCollator(super._searchLocations.getField(field));
        termList = new TreeMap(sortCollator);
        IndexReader reader = null;
        TermEnum ts = null;
        try {
            reader = super._searchLocations.getReader();
            if (reader != null) {
                ts = reader.terms(new Term(internedfieldName, ""));
                boolean collecting = false;
                if (ts != null) {
                    do {
                        if (ts.term() != null && ts.term().field() == internedfieldName) {
                            collecting = true;
                            TermInfo ti = (TermInfo) termList.get(ts.term().text());
                            if (ti != null)
                                ti.update(reader, ts.term());
                            else {
                                ti = new TermInfo();
                                ti.enableLogging(super.getLog());
                                ti.setUp(reader, ts.term());
                                termList.put(ts.term().text(), ti);
                            }
                        } else if (collecting) break;
                    } while (ts.next());
                }
            }
        } catch (IOException e) {
            String[] args = new String[1];
            args[0] = e.getMessage();
            throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SETUP_TERMS, args, e);
        } finally {
            //freeing resources
            if (ts != null) {
                try {
                    ts.close();
                } catch (IOException e) {
                    String[] args = new String[1];
                    args[0] = e.getMessage();
                    throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e);
                }
            }
            if (reader != null) {
                try {
                    reader.close();
                } catch (IOException e) {
                    String[] args = new String[1];
                    args[0] = e.getMessage();
                    throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e);
                }
            }
        }

        super.setNbPages(countPages());
    }

    /**
     *	Builds a term list.
     *
     *  @param	sLocs	The SearchLocations object (indices to be searched).
     *	@param	field	The field name.
     *  @param  str     The term
     */
    public void setUp(SearchLocations sLocs, String field, String str) throws SDXException {
        if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);

        if (!Utilities.checkString(field)) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);
        setFieldFilter(field);
        //interning the field name for better performance with lucene
        String internedfieldName = field.intern();

        if (str == null)
            setUp(sLocs, field);
        else {
            super.setSearchLocations(sLocs);

            initCollator(super._searchLocations.getField(field));
            termList = new TreeMap(sortCollator);
            IndexReader reader = null;
            TermEnum ts = null;
            RE re = null;
            try {
                re = getRE(str);
            } catch (RESyntaxException e) {
                String[] args = new String[1];
                args[0] = e.getMessage();
                throw new SDXException(null, SDXExceptionCode.ERROR_SETUP_TERMS, args, e);
            }
            reader = super._searchLocations.getReader();
            if (reader != null) {
                try {
                    ts = reader.terms(new Term(internedfieldName, ""));
                    Term t = null;
                    String text;
                    boolean collecting = false;
                    if (ts != null) {
                        do {
                            if (ts.term() != null)
                                t = ts.term();
                            if (t != null && t.field() == internedfieldName) {
                                collecting = true;
                                text = t.text();
                                if (re.match(text)) {
                                    TermInfo ti = (TermInfo) termList.get(text);
                                    if (ti != null)
                                        ti.update(reader, ts.term());
                                    else {
                                        ti = new TermInfo();
                                        ti.enableLogging(super.getLog());
                                        ti.setUp(reader, ts.term());
                                        termList.put(text, ti);
                                    }
                                }
                            } else if (collecting) break;
                        } while (ts.next());
                    }
                } catch (IOException e) {
                    String[] args = new String[1];
                    args[0] = e.getMessage();
                    throw new SDXException(null, SDXExceptionCode.ERROR_SETUP_TERMS, args, e);
                } finally {
                    //freeing resources
                    if (ts != null) {
                        try {
                            ts.close();
                        } catch (IOException e) {
                            String[] args = new String[1];
                            args[0] = e.getMessage();
                            throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e);
                        }
                    }
                    if (reader != null) {
                        try {
                            reader.close();
                        } catch (IOException e) {
                            String[] args = new String[1];
                            args[0] = e.getMessage();
                            throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e);
                        }
                    }
                }
            }
            super.setNbPages(countPages());
        }
    }

    protected void setFieldFilter(String field) {
        String[] fields = {field};
        fieldFilters = fields;
    }

    /**
     *	Builds a term list.
     *
     *  @param	results	The Results object (indices to be searched).
     *	@param	field	The field name.
     */
    public void setUp(Results results, String field, String value) throws SDXException {
        setFieldFilter(field);
        String[] fields = new String[1];
        fields[0] = field;
        String[] values = new String[1];
        values[0] = value;
        setUp(results, fields, values);
    }

    public void setUp(Results results, String[] fields, String[] values) throws SDXException {
        if (results != null) {
            super.setSearchLocations(results.getSearchLocations());
            if (super._searchLocations == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);
            if (fields == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);
            // If there is more than one Reader in the searchlocation, then log a warning
            if (super._searchLocations.size() > 1) LoggingUtils.logWarn(super.getLog(), SDXExceptionCode.WARN_USE_FIRST_SEARCHLOCATION, null);
            // We store the fields and values
            fieldFilters = fields;
            termList = getTerms(results.getHits(), fields, values);
            super.setNbPages(countPages());
        }
    }

    /**
     *	Builds a term list filtred by a query
     *
     *  @param	searchLocations	The SDX Search Locations object
     *  @param	results	The Results object to extract the Lucene Query
     *	@param	field	The field name.
     *  @param	value	The value (may be null)
     */
    public void setUp(SearchLocations sLocs, Results sdxResults, String field, String value) 
    		throws SDXException 
	{
    	setUp(sLocs, sdxResults.getQuery(), field, value);
    }
    /**
     *	Builds a term list filtred by a query
     *
     *  @param	searchLocations	The SDX Search Locations object
     *  @param	sdxQuery	The SDX Query object to extract the Lucene Query
     *	@param	field	The field name.
     *  @param	value	The value (may be null)
     */
    public void setUp(SearchLocations sLocs, Query sdxQuery, String field, String value) 
    throws SDXException 
    {
    	if (sdxQuery==null || sdxQuery.getLuceneQuery()==null) {
    		throw new SDXException(null, SDXExceptionCode.ERROR_QUERY_NULL, null, null);
    	}
    	setUp(sLocs, sdxQuery.getLuceneQuery(), field, value);
    }
    /**
     *	Builds a term list filtred by a query
     *
     *  @param	searchLocations	The SDX Search Locations object
     *  @param	query	The Lucene Query object
     *	@param	field	The field name.
     *  @param	value	The value (may be null)
     */
    public void setUp(SearchLocations sLocs, org.apache.lucene.search.Query luceneQuery, String field, String value) 
    throws SDXException 
    {

    	if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);
        if (!Utilities.checkString(field)) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);
        setFieldFilter(field);
        //interning the field name for better performance with lucene
        String internedfieldName = field.intern();
        
        super.setSearchLocations(sLocs);
        // If there is more than one Reader in the searchlocation, then log a warning
        if (super._searchLocations.size() > 1) LoggingUtils.logWarn(super.getLog(), SDXExceptionCode.WARN_USE_FIRST_SEARCHLOCATION, null);

        initCollator(super._searchLocations.getField(field));
        termList = new TreeMap(sortCollator);
        
    	QueryWrapperFilter qwf;
    	try {
    		qwf = new QueryWrapperFilter( luceneQuery );
    	} catch (Exception e) {
    		String[] args = new String[1];
    		args[0] = e.getMessage();
    		throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_QUERY_NULL, args, e);
    	} finally {
    		luceneQuery = null;
    	}

    	IndexReader r = sLocs.getReader();

    	try {
    		termList = getTerms( sLocs.getReader(), sLocs.getField(field), qwf.bits(r), value );
    	} catch (Exception e) {
    		String[] args = new String[2];
    		args[0] = sLocs.getReader().directory().toString();
    		args[1] = e.getMessage();
    		throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_READ, args, e);
    	} finally {
    		if (r != null) {
    			try {
    				r.close(); r = null;
    			} catch (IOException e) {
    				String[] args = new String[1];
    				args[0] = e.getMessage();
    				throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e);
    			}
    		}
    		qwf = null;
    	}

    	super.setNbPages(countPages());

    }


    /**
     * Builds a lits of terms from multiple criterias.
     *
     * <p>IMPORTANT : only one search location is actually used.
     * TODO : fix this...
     *
     * <p>Criterias are field=value pairs. This method lets a
     * developer creates hierarchical lists.
     *
     * <p>There should be one value less than fields; the last
     * field will then be the field to return values from.
     *
     * <p>For instance, if the first field is 'region', the
     * second is 'department' and the third is 'city', and
     * the first value is 'Aquitaine' and the second value is
     * 'Gironde', you will get a list of cities that appear in
     * documents that also have region=Aquitaine and
     * department=Gironde.
     *
     * @param  sLocs       Where to find the terms (may not bu null).
     * @param  fields      The list of fields (may not be null).
     * @param  values      The list of values (there should be one less value than fields).
     */
    public void setUp(SearchLocations sLocs, String[] fields, String[] values) throws SDXException {
    	setUp(sLocs, fields, values, Filter.BOOLEAN_OPERATOR_AND); // par defaut le ET
    }
    public void setUp(SearchLocations sLocs, String[] fields, String[] values, int operator) throws SDXException {
        if (sLocs == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_SEARCHLOCATIONS_NULL, null, null);
        if (fields == null) throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_FIELD_NAME_NULL, null, null);

        super.setSearchLocations(sLocs);

        // If there is more than one Reader in the searchlocation, then log a warning
        if (sLocs.size() > 1) LoggingUtils.logWarn(super.getLog(), SDXExceptionCode.WARN_USE_FIRST_SEARCHLOCATION, null);

        // We store the fields and values
        fieldFilters = fields;
        valueFilters = values;

        // Consider a null values as an array of length 0, or no filter...
        if (values == null) values = new String[0];

        // We only build the terms if there is one more field than values
        if (fields.length == (values.length + 1)) {

            // We'll build a simple filter, by ANDing all field=value
            // pairs except the last field.

            BitSet documentSet = null;
            IndexReader r = sLocs.getReader();
            if (fields.length > 1) {
            	
            	if(operator != Filter.BOOLEAN_OPERATOR_AND
            			&& operator != Filter.BOOLEAN_OPERATOR_OR
            			&& operator != Filter.BOOLEAN_OPERATOR_NOT) {
            		operator = Filter.BOOLEAN_OPERATOR_AND; // Le ET par defaut
            	}
            	
                filter = new Filter(operator);

                for (int i = 0; i < values.length; i++) {
                    Criteria c = new Criteria();
                    c.enableLogging(super.getLog());
                    c.setUp(sLocs.getField(fields[i]), values[i]);
                    filter.add(c);
                }
                documentSet = filter.bits(r);
            }


            // Then we build the list of values of the last field
            // with at least one document in the bitset.
            try {
                termList = getTerms(r, sLocs.getField(fields[fields.length - 1]), documentSet);
            } finally {
                if (r != null) {
                    try {
                        r.close();
                    } catch (IOException e) {
                        String[] args = new String[1];
                        args[0] = e.getMessage();
                        throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_READER_CLOSE, args, e);
                    }
                }
            }
            super.setNbPages(countPages());

        } else {
            // Here, we don't meet the criteria that there must be one more field than value
            // TODO : log a warning...
        }
    }

    /**Creates a list of terms according to a chain with truncation
     *
     *  <p>
     * A super.getLog() must be set and then the Terms must be setUp.
     *
     * @see #enableLogging
     * @see #setUp
     */
    public Terms() {
    }

    /**
     *	Returns an XML representation of this term list.
     *
     *	@param	hdl	    The ContentHandler to feed with SAX events.
     */
    public void toSAX(ContentHandler hdl) throws SAXException {
        toSAX(hdl, -1);
    }
    /**
     *	Returns an XML representation of this term list.
     *
     *	@param	hdl	    The ContentHandler to feed with SAX events.
     *	@param	page	The number of the page to show.
     */
    public void toSAX(ContentHandler hdl, int page) throws SAXException {
        String sdxNsUri = Framework.SDXNamespaceURI;
        String sdxNsPrefix = Framework.SDXNamespacePrefix;

        if (page < 1) page = 1;
        if (page > super.getNbPages()) page = super.getNbPages();
        int sIdx = ((page - 1) * super.getHitsPerPage());
        if (sIdx < 0) sIdx = 0;

        int eIdx = (page * super.getHitsPerPage()) - 1;
        if (termList != null && eIdx >= termList.size()) eIdx = termList.size() - 1;

        //Creation of local variables which are later passed into startElement() and endElement() methods-rbp13/03/02
        String localName = Node.Name.TERMS;
        String qName = sdxNsPrefix + ":" + localName;
        AttributesImpl atts = new AttributesImpl();


        if (fieldFilters != null && fieldFilters.length > 0) {
            atts.addAttribute("", Node.Name.NAME, Node.Name.NAME, Node.Type.CDATA, fieldFilters[fieldFilters.length - 1]);
            atts.addAttribute("", Node.Name.FIELD, Node.Name.FIELD, Node.Type.CDATA, fieldFilters[fieldFilters.length - 1]);
        }

        // Les informations sur le nombre de pages
        //TODO?:should we also add the field name here or other information?-rbp
        atts.addAttribute("", Node.Name.QID, Node.Name.QID, Node.Type.CDATA, String.valueOf(super.getId()));
        atts.addAttribute("", Node.Name.PAGE, Node.Name.PAGE, Node.Type.CDATA, String.valueOf(page));
        atts.addAttribute("", Node.Name.HPP, Node.Name.HPP, Node.Type.CDATA, String.valueOf(super.getHitsPerPage()));
        atts.addAttribute("", Node.Name.PAGES, Node.Name.PAGES, Node.Type.CDATA, String.valueOf(super.getNbPages()));
        if (termList != null)
            atts.addAttribute("", Node.Name.NB, Node.Name.NB, Node.Type.CDATA, String.valueOf(termList.size()));
        atts.addAttribute("", Node.Name.START, Node.Name.START, Node.Type.CDATA, String.valueOf(sIdx + 1));
        atts.addAttribute("", Node.Name.END, Node.Name.END, Node.Type.CDATA, String.valueOf(eIdx + 1));

        // for backward compatibility but not documented
        atts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, String.valueOf(super.getId()));
        atts.addAttribute("", Node.Name.CURRENT_PAGE, Node.Name.CURRENT_PAGE, Node.Type.CDATA, String.valueOf(page));
        atts.addAttribute("", Node.Name.NB_PAGES, Node.Name.NB_PAGES, Node.Type.CDATA, String.valueOf(super.getNbPages()));

        //startElement() method is called for "terms" and local variables are passed-rbp12/03/02
        hdl.startElement(sdxNsUri, localName, qName, atts);

        // On va ajouter des informations sur les filtres utilises.
        if (filter != null) filter.toSAX(hdl);

        if (super.getHitsPerPage() != 0) {
            if (termList != null) {
                Iterator it = termList.keySet().iterator();
                int i = 0;
                while (it.hasNext()) {
                    i++;

                    if (i > sIdx && i <= eIdx + 1) {
                        TermInfo term = (TermInfo) termList.get(it.next());
                        //Creation of local variables which are later passed into startElement() and endElement() methods-rbp12/03/02
                        String subLocalName = Node.Name.TERM;
                        String subQName = sdxNsPrefix + ":" + subLocalName;
                        AttributesImpl subAtts = new AttributesImpl();
                        subAtts.addAttribute("", Node.Name.NO, Node.Name.NO, Node.Type.CDATA, String.valueOf(i));

                        String fieldname = term.getField();
                        String content;

                        int fieldType = -1;
                        try {
                            fieldType = super._searchLocations.getFieldType(fieldname);
                        } catch (SDXException sdxE) {
                            throw new SAXException(sdxE.getMessage(), sdxE);
                        }
                        if (super._searchLocations != null && fieldType == Field.DATE)
                            content = Date.formatDate(DateField.stringToDate(term.getContent()));
                        else
                            content = term.getContent();

                        String esc = null;
                        esc = Utilities.encodeURL(content, super.getEncoding());
                        subAtts.addAttribute("", Node.Name.VALUE, Node.Name.VALUE, Node.Type.CDATA, content);
                        subAtts.addAttribute("", Node.Name.ESCAPED_VALUE, Node.Name.ESCAPED_VALUE, Node.Type.CDATA, esc);
                        subAtts.addAttribute("", Node.Name.FIELD, Node.Name.FIELD, Node.Type.CDATA, fieldname);


                        subAtts.addAttribute("", Node.Name.DOCS, Node.Name.DOCS, Node.Type.CDATA, String.valueOf(term.getDocFrequency()));
                        // S'il y a un seul document, alors on retourne aussi son identificateur.
                        if (term.getDocFrequency() == 1) {
                            subAtts.addAttribute("", Node.Name.ID, Node.Name.ID, Node.Type.CDATA, term.getDocId());
                            subAtts.addAttribute("", Node.Name.DB_ID, Node.Name.DB_ID, Node.Type.CDATA, term.getDbId());
                            subAtts.addAttribute("", Node.Name.APPID, Node.Name.APPID, Node.Type.CDATA, term.getAppId());
                            subAtts.addAttribute("", Node.Name.DOC_ID, Node.Name.DOC_ID, Node.Type.CDATA, term.getDocId());
                        }
                        // not documented but preserved for backward compatibility
                        subAtts.addAttribute("", Node.Name.DOC_FREQ, Node.Name.DOC_FREQ, Node.Type.CDATA, String.valueOf(term.getDocFrequency()));
                        //startElement() method is called for "term" and local variables are passed-rbp12/03/02
                        hdl.startElement(sdxNsUri, subLocalName, subQName, subAtts);

                        //endElement() method is called for "term" and local variables are passed-rbp12/03/02
                        hdl.endElement(sdxNsUri, subLocalName, subQName);

                    } else if (i > eIdx + 1) break; else it.next();
                }
            }
        }
        //endElement() method is called for "terms" and local variables are passed-rbp12/03/02
        hdl.endElement(sdxNsUri, localName, qName);
    }

    /**
     *	Returns a list of the terms.
     */
    public TreeMap getList() {
        return termList;
    }

    /**
     *	Constructs a regular expression from a term of the acceptRequest.
     *
     *	@param	str		The term.
     */
    private static RE getRE(String str) throws RESyntaxException {
        if (str == null) return null;
        StringBuffer st = new StringBuffer("^");
        for (int i = 0; i < str.length(); i++) {
            char c = str.charAt(i);
            switch (c) {
                case '*':
                    st.append("(.*)");
                    break;

                case '?':
                    st.append("(.?)");
                    break;

                case '(':
                case ')':
                case '.':
                case '\\':
                case '[':
                case ']':
                case '^':
                case '$':
                case '+':
                case '{':
                case '}':
                case '|':
                    st.append('\\');
                    st.append(c);
                    break;

                default:
                    st.append(c);
            }
        }
        st.append("$");
        return new RE(st.toString());
    }

    /*
     *	Retourne une liste de termes en fonction d'un champ et d'une valeur.
     *
     *	La valeur peut contenir des masques * et ?. Tous les termes de ce champ
     *	qui respectent la valeur masquee seront retournes. Si aucun terme ne
     *	satisfait le critere, on retourne une liste vide.
     *
     *	@param	r		Un objet pour lire l'index.
     *	@param	f		Le nom du champ
     *	@param	pattern	La valeur masquee.
     */
    /**
     * Returns a list of terms according to a field and a value.
     *
     * The value can contain masks and?. All terms of this field
     * which respects the masked value will be turned over. If no term
     * satisfied the criterion, an empty list is returned.
     *
     * @param r     The index reader
     * @param f     The field
     * @param pattern   The pattern
     * @throws SDXException
     * @deprecated Use {@link RegexTerms#getRegexTerms(BitSet)} after a {@link RegexTerms#setUp(SearchLocations, String, String)}
     */
    public static TreeMap getTerms(IndexReader r, Field f, String pattern) throws SDXException {
        TreeMap ret = null;

        if (r == null || f == null || pattern == null) return ret;
        String internedfieldName = "";
        if (Utilities.checkString(f.getCode()))
        //interning the field name for better performance with lucene
            internedfieldName = f.getCode().intern();

        if (f.getLocale() != null)
            ret = new TreeMap(Collator.getInstance(f.getLocale()));
        else
        //TODO?:is this acceptable default behavior?-rbp
            ret = new TreeMap(Collator.getInstance(new Locale("fr", "FR")));

        TermEnum ts = null;
        try {
            RE re = getRE(pattern);
            ts = r.terms(new Term(internedfieldName, ""));
            boolean collecting = false;
            if (ts != null) {
                do {
                    if (ts.term() != null && ts.term().field() == internedfieldName) {
                        collecting = true;
                        if (ts.term() != null && re != null && re.match(ts.term().text())) {
                            // First check if the term is already in the list
                            TermInfo ti = (TermInfo) ret.get(ts.term().text());
                            if (ti != null)
                                ti.update(r, ts.term());
                            else {
                                ti = new TermInfo();
                                //ti.enableLogging(_logger);
                                ti.setUp(r, ts.term());
                                ret.put(ts.term().text(), ti);
                            }
                        }
                    } else if (collecting) break;
                } while (ts.next());
            }
        } catch (RESyntaxException e) {
            String[] args = new String[1];
            args[0] = e.getMessage();
            throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e);
        } catch (IOException e) {
            String[] args = new String[1];
            args[0] = e.getMessage();
            throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e);
        } finally {
            try {
                //freeing resources
                if (ts != null) ts.close();
            } catch (IOException e) {
                String[] args = new String[2];
                args[0] = f.getCode();
                args[1] = e.getMessage();
                throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e);
            }
        }

        return ret;
    }

    /**
     * Builds a list of values for a filtered f.
     *
     *	@param	hits			The results.
     *	@param	fields	        The fields in question.
     *	@param	values	        The values for the fields in question.
     */
    private TreeMap getTerms(Hits hits, String[] fields, String[] values) throws SDXException {

        TreeMap ret = new TreeMap();
        if (hits == null || hits.length() == 0) return ret; // TODO : is the the better solution? Harmonize with other lists of terms...
        Hashtable fieldMaps = new Hashtable();
        for (int i = 0; i < hits.length(); i++) {
            try {
                Document hitDoc = hits.doc(i);
                if (hitDoc != null) {
                    for (int j = 0; j < fields.length; j++) {
                        Field f = super._searchLocations.getField(fields[j]);
                        String lfname = f.getCode();
                        if (f != null) {
                            TreeMap map = null;
                            if (fieldMaps.size() > 0 && fieldMaps.containsKey(lfname))
                                map = (TreeMap) fieldMaps.get(lfname);
                            else {
                                if (f.getCollator() == null)
                                    map = new TreeMap(Collator.getInstance(new Locale("fr", "FR"))); // TODO : is it OK?
                                else
                                    map = new TreeMap(f.getCollator());
                            }
                            String[] fieldsValues = hitDoc.getValues(lfname);
                            if (fieldsValues != null) {
                                HashSet termsCollected = new HashSet();
                                for (int k = 0; k < fieldsValues.length; k++) {
                                    String lftext = fieldsValues[k];
                                    if (Utilities.checkString(lftext)) {
                                        if (values == null || values.length == 0 || (j < values.length && (!Utilities.checkString(values[j]) || lftext.equals(values[j])))) {
                                            if (!termsCollected.contains(lftext)) {
                                                TermInfo ti = (TermInfo) map.get(lftext);
                                                if (ti != null) {
                                                    ti.update(lfname, lftext);
                                                } else {
                                                    ti = new TermInfo();
                                                    ti.enableLogging(super.getLog());
                                                    ti.setUp(lfname, lftext);
                                                    ti.update(lfname, lftext);
                                                    ti.setDocId(hitDoc.getField(LuceneIndex.ID_FIELD).stringValue());
                                                    ti.setDbId(hitDoc.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXDBID).stringValue());
                                                    ti.setAppId(hitDoc.getField(LuceneDocumentBase.INTERNAL_FIELD_NAME_SDXAPPID).stringValue());
                                                    map.put(lftext, ti);
                                                }
                                                termsCollected.add(lftext);
                                            }
                                        }
                                    }
                                }
                            }
                            if (map != null) fieldMaps.put(lfname, map);
                        }
                    }
                }
            } catch (IOException e) {
                String[] args = new String[1];
                args[0] = e.getMessage();
                new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e);
            }
        }


        Enumeration maps = fieldMaps.elements();
        if (maps != null) {
            while (maps.hasMoreElements()) {
                TreeMap outmap = (TreeMap) maps.nextElement();
                ret.putAll(outmap);
            }
            maps = null;
        }
        return ret;
    }


    /**
     * Builds a list of values for a filtered field
     *
     *	@param	r		The reader.
     *	@param	f		The f.
     *	@param	docs	A list of documents as a filter.
     */
    public TreeMap getTerms(IndexReader r, Field f, BitSet docs) throws SDXException {
    	return getTerms(r, f, docs, null);
    }
    /**
     * Builds a list of values for a filtered field, according to a value (may contains wildcards)
     *
     *	@param	r		The reader.
     *	@param	f		The field
     *	@param	docs	A list of documents as a filter.
     *	@params str		The value to match (may be null)
     */
    public TreeMap getTerms(IndexReader r, Field f, BitSet docs, String str) 
    	throws SDXException {
    	if (f == null) return new TreeMap(); // TODO : is the the better solution? Harmonize with other lists of terms...

        //interning the f name for better performance with lucene
        String internedfieldName = "";
        if (Utilities.checkString(f.getCode())) internedfieldName = f.getCode().intern();

        TreeMap ret;
        if (f.getCollator() == null)
            ret = new TreeMap(Collator.getInstance(new Locale("fr", "FR"))); // TODO : is it OK?
        else
            ret = new TreeMap(f.getCollator());

        RE re = null;
        if (Utilities.checkString(str)) {
            try {
                re = getRE(str);
            } catch (RESyntaxException e) {
                String[] args = new String[1];
                args[0] = e.getMessage();
                throw new SDXException(null, SDXExceptionCode.ERROR_SETUP_TERMS, args, e);
            }
        }

        TermEnum ts = null;
        try {
            ts = r.terms(new Term(internedfieldName, ""));
            boolean collecting = false;
            BitSet termDocumentSet;
            int nbDocs;
            String text;
            do {
                if (ts.term() != null && ts.term().field() == internedfieldName) {
                    collecting = true;

                    // We build the bitset for this term and compare
                    // it with the one we have, and then count what's
                    // left.

                    termDocumentSet = getDocumentSet(r, ts.term());
                    if (docs != null) termDocumentSet.and(docs);
                    nbDocs = Bits.countBits(termDocumentSet);
                    if (nbDocs > 0) {
                    	text = ts.term().text();
                    	if (re==null || re.match(text)) { // On ajoute le terme courant dans la liste finale s'il n'y a pas de contrainte sur la valeur elle-meme ou si celle-ci est correcte
	                        TermInfo ti = (TermInfo) ret.get(ts.term().text());
	                        if (ti != null)
	                            ti.update(r, ts.term());
	                        else {
	                            ti = new TermInfo();
	                            ti.enableLogging(super.getLog());
	                            ti.setUp(r, ts.term());
	                            ret.put(ts.term().text(), ti);
	                        }
                    	}
                    }
                } else if (collecting) break;
            } while (ts.next());
            return ret;
        } catch (IOException e) {
            String[] args = new String[1];
            args[0] = e.getMessage();
            throw new SDXException(null, SDXExceptionCode.ERROR_GET_TERMS, args, e);
        } finally {
            try {
                //freeing resources
                if (ts != null) ts.close();
            } catch (IOException e) {
                String[] args = new String[2];
                args[0] = f.getCode();
                args[1] = e.getMessage();
                throw new SDXException(null, SDXExceptionCode.ERROR_LUCENE_TERMENUM_CLOSE, args, e);
            }
        }
    }
    
    /**
     * Returns a list of the documents associated with a term within the index.
     *
     *	@param	r		The index reader.
     *	@param	term	The term for searching.
     */
    public static BitSet getDocumentSet(IndexReader r, Term term) throws IOException {
        BitSet bits = new BitSet(r.numDocs());
        TermDocs docs = r.termDocs(term);
        while (docs.next()) bits.set(docs.doc());
        return bits;
    }

    /**
     *	Counts the number of pages for these terms.
     */
    public int countPages() {
        if (termList == null)
            return 0;
        else if (termList.size() == 0)
            return 0;
        else {
            if (super.getHitsPerPage() == 0) return 1;
            float temp = ((float) termList.size() / (float) super.getHitsPerPage());
            if (temp != Math.round(temp))
                return (int) (Math.round(Math.floor(temp) + 1));
            else
                return Math.round(temp);
        }
    }

    /**
     *	Set the number of hits per page.
     *
     *	@param	nb		The number of hits.
     */
    public void setHitsPerPage(int nb) {
        if (nb < 0)
            setAllHits();
        else {
            super.setHitsPerPage(nb);
            super.setNbPages(countPages());
        }
    }

    /**
     *	Initializes the collator used for sorting .
     *
     *	@param	field  	The field from the document base.
     */
    protected void initCollator(Field field) {
        if (field != null) sortCollator = field.getCollator();
    }

    /**
     *	Indicate to return all the results on only one page.
     */
    public void setAllHits() {
        if (termList != null)
            super.setHitsPerPage(termList.size());
        super.setNbPages(1);
    }

    protected String getClassNameSuffix() {
        return Terms.CLASS_NAME_SUFFIX;
    }


}
