/*
Copyright (C) 2000-2010  Ministere de la culture et de la communication (France), AJLSM
See LICENCE file
*/
package fr.gouv.culture.sdx.search.lucene.highlight;

import java.io.IOException;
import java.io.StringReader;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.TreeMap;
import java.util.Vector;

import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.xml.sax.Attributes;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

import fr.gouv.culture.sdx.exception.SDXException;
import fr.gouv.culture.sdx.exception.SDXExceptionCode;
import fr.gouv.culture.sdx.framework.Framework;
import fr.gouv.culture.sdx.pipeline.GenericPipeline;
import fr.gouv.culture.sdx.search.Query;
import fr.gouv.culture.sdx.search.TermHighlighter;
import fr.gouv.culture.sdx.search.Terms;
import fr.gouv.culture.sdx.search.lucene.Field;
import fr.gouv.culture.sdx.search.lucene.analysis.Analyzer;
import fr.gouv.culture.sdx.search.lucene.query.SearchLocations;
import fr.gouv.culture.sdx.search.lucene.query.TermInfo;
import fr.gouv.culture.sdx.utils.Utilities;
import fr.gouv.culture.sdx.utils.constants.Node;
import fr.gouv.culture.sdx.utils.logging.LoggingUtils;
import fr.gouv.culture.sdx.utils.lucene.LuceneTools;

public class LuceneTermHighlighter extends GenericPipeline implements TermHighlighter {

    private int hliteIdx = 0;
    private String uri = Framework.SDXNamespaceURI;
    private String local = Node.Name.HILITE;
    private String qName = Framework.SDXNamespacePrefix + ":" + local;
    private Query query = null;
    private SearchLocations sLocs = null;
    /**A hashtable which contains as values the terms to be highlighted in the text*/
    private Hashtable terms = null;
    private boolean saxStreamStarted = false;
    private String termsSt = null; // la liste des termes recherches sous forme de String
    private Vector texps = null; // liste des expressions presentes dans les termes recherches
    private String texpsSt = null;

    /**Set's up the pipeline
     *
     * @param query The sdx query object on which the results are based
     */
    public void setQuery(Query query) throws SDXException {
        this.query = query;
        if (this.query != null) {
            this.sLocs = ((fr.gouv.culture.sdx.search.lucene.query.Query) this.query).getSearchLocations();
            org.apache.lucene.search.Query lQuery = ((fr.gouv.culture.sdx.search.lucene.query.Query) this.query).getCachedQuery();
            if (lQuery != null) {
                // get terms in query
                try {
                    this.terms = new Hashtable();
                    LuceneTools.getTerms(lQuery, this.terms, false);
                } catch (IOException e) {
                    throw new SDXException(super.getLog(), SDXExceptionCode.ERROR_GET_QUERY_TERMS, null, e);
                }

            }
        }

    }

    public void setTerms(Terms terms) {
        if (terms != null) {//TODO: better null checks, etc.
            this.sLocs = ((fr.gouv.culture.sdx.search.lucene.query.Terms) terms).getSearchLocations();
            TreeMap termList = terms.getList();
            Iterator termValues = termList.keySet().iterator();
            if (termValues != null) {
                this.terms = new Hashtable();
                while (termValues.hasNext()) {
                    String termText = (String) termValues.next();
                    TermInfo ti = (TermInfo) termList.get(termText);
                    String fieldName = ti.getField();
                    this.terms.put(fieldName + "_" + termText, termText);
                }
            }
        }

    }


    /**
     * Highlight an arbitrary term. For example, an HTML TermHighlighter could simply do:
     *
     * <p><dl><dt></dt><dd><code>return "&lt;b&gt;" + term + "&lt;/b&gt;";</code></dd></dl>
     *
     * @param term term text to highlight
     * @param termText the string value of the term on which the comparison was made
     *
     */
    public void highlightTerm(String term, String termText) throws SAXException {
        if (this.xmlConsumer != null && Utilities.checkString(term)) {
            this.hliteIdx++;

            AttributesImpl locAttr = null;
            locAttr = new AttributesImpl();
            locAttr.addAttribute("", Node.Name.NO, Node.Name.NO, Node.Type.CDATA, Integer.toString(this.hliteIdx));
            locAttr.addAttribute("", Node.Name.TERM, Node.Name.TERM, Node.Type.CDATA, termText);

            this.xmlConsumer.startElement(this.uri, this.local, this.qName, locAttr);

            char[] chars = term.toCharArray();
            if (chars != null)
                this.xmlConsumer.characters(chars, 0, chars.length);

            this.xmlConsumer.endElement(this.uri, this.local, this.qName);
        }
    }

    /**
     * Highlights a text in accordance to a given query.
     * This text is analyzed according to the field and the analyzer of the query.
     * So, empty words are not hilited.
     *
     * @param text     Text to highlight terms in
     * @param field    Name of the query's field
     * @param analyzer Analyzer used to construct the Query
     *
     */
    private void highlightTerms(String field, String text, Analyzer analyzer) 
    		throws SAXException, IOException 
		{

        TokenStream stream = null;
        String lastHilitedTermFromText = ""; //the last term for which higlighting was last done
        boolean isTestExp=false; // marqueur du test de la presence des expressions dans termsSt

        try {

            stream = analyzer.tokenStream(field, new StringReader(text)); // analyse du texte a mettre eventuellement en surbrillance
            Vector tokens = new Vector(); // liste des termes a mettre eventuellement en surbrillance
            StringBuffer tbriefs = null;// le texte original sortie de l'analyseur

            Token tmpToken;
            int tmpStartOffset = -1;
            int tmpEndOffset = -1;
            int lastEndOffset = 0;

            // Boucle sur les termes
            while ((tmpToken = stream.next()) != null || tokens.size() > 0) {

                if (tmpStartOffset == -1)//init
                    tmpStartOffset = tmpToken.startOffset();
                if (tmpEndOffset == -1)//init
                    tmpEndOffset = tmpToken.endOffset();
                if (tmpToken != null && tmpToken.startOffset() == tmpStartOffset && tmpToken.endOffset() == tmpEndOffset)
                    tokens.add(tmpToken);//token for same text position

                else if (tokens.size() > 0) {
                  // no longer token for same text position, so process all past tokens for same text position stored in 'tokens'

                    //processing tokens stored in array list
                    Enumeration tokensForSameTextPosition = tokens.elements();

                    // Boucle sur les termes a mettre eventuellement en surbrillance
                    while (tokensForSameTextPosition.hasMoreElements()) {

                        org.apache.lucene.analysis.Token token = (Token) tokensForSameTextPosition.nextElement();
                        String originalTextAtTokenPosition;
                        int startOffset;
                        int endOffset;

                        if (token != null) {

                            startOffset = token.startOffset();
                            endOffset = token.endOffset();
                            originalTextAtTokenPosition = text.substring(startOffset, endOffset);

                            if (startOffset > lastEndOffset) {
                              // append text between end of last token (or beginning of text) and start of current token
                              char[] chars = text.substring(lastEndOffset, startOffset).toCharArray();
                              if (chars != null && chars.length > 0)
                                  this.xmlConsumer.characters(chars, 0, chars.length);
                              lastHilitedTermFromText = "";
                            }

                            //analye the next portion of text
                            String termText = token.termText();

                            // has the termText already been highlighted, if not: does query contain current termText?
                            if ( Utilities.checkString(termText)
                                && !Utilities.checkString(lastHilitedTermFromText)
                                && this.terms != null)
                            {
                                if( termsSt==null ) termsSt = this.terms.toString();
                                if( termsSt!=null
                                  && ( termsSt.matches(".*\\b"+termText+"\\b.*")
                                  		|| termsSt.matches(".*\\b"+originalTextAtTokenPosition+"\\b.*") ) )
                                {
                                  // le terme courant est dans la liste des termes recherches

                                if( this.terms.containsValue(termText)
                                		|| this.terms.containsValue(originalTextAtTokenPosition) )
                                {
                                  // cas habituel : le terme courant est present dans la liste des termes recherches. on active la surbrillance
                                  highlightTerm(originalTextAtTokenPosition, termText);
                                  lastHilitedTermFromText = termText;
                                }

                                else {

                                  // nouveau : mise en surbrillance des expressions

                                  if(!isTestExp){
                                    if (texps == null) texps = new Vector();
                                    // rechercher les expressions dans les termes recherches
                                    String tsearch=null;
                                    Enumeration tsearchs = this.terms.elements(); // recupere la liste des termes recherches
                                    while(tsearchs.hasMoreElements()){
                                      tsearch = (String) tsearchs.nextElement();
                                      if(Utilities.checkString(tsearch) && tsearch.indexOf(" ")>0) {
                                        texps.add(tsearch);
                                      }
                                    }
                                    texpsSt = texps.toString();
                                    isTestExp=true;
                                  }

                                  if( isTestExp && texps!=null
                                      //&& texpsSt.indexOf(termText)!=-1
                                      && ( texpsSt.matches(".*\\b"+termText+"\\b.*") 
                                      		|| texpsSt.matches(".*\\b"+originalTextAtTokenPosition+"\\b.*") ) )
                                  {
                                    // on a des expressions dans les termes recherches et le terme courant semble y etre
                                    String texp=null;

                                    // boucle sur les expressions
                                    for(Enumeration mE = texps.elements(); mE.hasMoreElements();){
                                      /* nouveau : on analyse la requete pour permettre la surbrillance de bout de texte (expression et autre)
                                       * issus d'une requete field ou word.
                                       * FIXME (MP) : Le danger est de mettre en surbrillance des choses que l'utilisateur n'a pas exactement recherche : 
                                       * l'utilisateur recherche "Torpilleur Escadre" sur un critere "field" et le bout de texte en surbrillance
                                       * est "torpilleur en escadre".
                                       */
                                      // TODO OPTIMIZE (MP) : stocker les expressions analysees dans un tableau pour ne pas le faire a chaque fois.
                                      texp = (String) mE.nextElement();
                                      if(Utilities.checkString(texp)){
                                    	  TokenStream mStrm=null;
                                          Token mTk=null;
                                          int nb=0;
                                          try{
                                            mStrm = analyzer.tokenStream(field,
                                                          new StringReader(texp));
                                            StringBuffer mSb = new StringBuffer();
                                            while ( (mTk=mStrm.next()) != null )
                                            {
                                              if(nb>0) mSb.append(" ");
                                              mSb.append(mTk.termText());
                                              nb++;
                                            }
                                            texp = mSb.toString();
                                          } finally{
                                            if(mStrm!=null){
                                              mStrm.close();
                                            }
                                          }
                                      }
                                      
                                      if(Utilities.checkString(texp)
                                        && texp.indexOf(termText)!=-1 // le mot courant est dans l'expression recherchee courante
                                        && texp.startsWith(termText)  // le mot courant est le premier mot de l'expression recherchee courante
                                      ){

                                        /* Le mot courant est le premier mot de l'expression, mais on n'est pas encore sur
                                         * qu'il fasse bien parti de l'expression recherchee.
                                         * Le but maintenant est de confirmer cela et, le cas echant, de mettre l'expression
                                         * en surbrillance.
                                         * Pour cela, on va compter le nombre de mot dans l'expression et construire
                                         * une chaine de meme taille a partir du texte analyse en debutant au mot
                                         * courant.
                                         */
                                    	// TODO OPTIMIZE (MP) : stocker l'information dans un tableau pour ne pas le faire a chaque fois.
                                        TokenStream mStrm=null;
                                        Token mTk=null;
                                        int nb=0; // le nombre de mots du texte original analyse correspondant a l'expression recherchee
                                        try{
                                          mStrm = analyzer.tokenStream(field,
                                                        new StringReader(text.substring(startOffset)));
                                          tbriefs = new StringBuffer();
                                          int l = texp.split(" ").length;
                                          while ( nb < l
                                              && (mTk=mStrm.next()) != null )
                                          {
                                            if(nb>0) tbriefs.append(" ");
                                            tbriefs.append(mTk.termText());
                                            nb++;
                                          }
                                        } finally{
                                          if(mStrm!=null){
                                            mStrm.close();
                                          }
                                        }

                                        if(tbriefs!=null && tbriefs.toString().equals(texp) )
                                        {
                                          // maintenant, on sait que le texte original contient l'expression recherchee
                                          if(nb==1){ // Dans le cas ou on n'a un seul terme a traiter, le dernier terme mis en surbrillance est toujours le terme courant, pas le suivant !
                                        	  lastHilitedTermFromText = termText;
                                          }
                                          else {
	                                          // On avance le curseur jusqu'au mot suivant le dernier mot de l'expression recherchee
	                                          int t=2; // on demarre a 2 parce que le curseur est deja sur le mot suivant le premier de l'expression, ie on a deja lu 2 mots de l'expression
	                                          while(t<=nb && tmpToken!=null){
	                                            if(t==nb){
	                                              /* On se trouve sur le dernier mot de l'expression dans
	                                               * le texte original analyse. On y deplace le curseur.
	                                               * On recupere le texte original correspondant a l'expression
	                                               * et on l'a met en surbrillance. Enfin, on reinitialise
	                                               * le dernier mot mis en surbrillance sur
	                                               * le dernier mot de l'expression.
	                                               */
	                                              endOffset=tmpToken.endOffset();
	                                              lastHilitedTermFromText = tmpToken.termText();
	                                            }
	                                            // on avance le curseur
	                                            tmpToken=stream.next();
	                                            t++;
	                                          }
                                          }

                                          originalTextAtTokenPosition = text.substring(startOffset, endOffset); // recupere le texte original correspondant a l'expression
                                          highlightTerm(originalTextAtTokenPosition, tbriefs.toString()); // commande le mise en surbrillance du texte
                                          break; // on en a fini avec les expressions pour cette portion de texte

                                        }
                                      }
                                    }
                                  }

                                }
                              }
                            }

                            if (!tokensForSameTextPosition.hasMoreElements() && !Utilities.checkString(lastHilitedTermFromText)) {
                                //text has not been hilited and we are at the end of the loop so send the text without hilite
                                char[] chars2 = originalTextAtTokenPosition.toCharArray();
                                if (chars2 != null && chars2.length > 0)
                                    this.xmlConsumer.characters(chars2, 0, chars2.length);
                            }

                            //setting the end of the analyzed text
                            lastEndOffset = endOffset;
                        }

                    }
                    //empty array list
                    tokens.clear();
                    //adding nextToken
                    if (tmpToken != null) {
                        tokens.add(tmpToken);//token for same text position
                        //resetting loop vars
                        tmpStartOffset = tmpToken.startOffset();
                        tmpEndOffset = tmpToken.endOffset();
                    }
                    //let the loop continue

                }
            }

            // append text after end of last token
            if (lastEndOffset < text.length()) {
                char[] chars = text.substring(lastEndOffset).toCharArray();
                if (chars != null)
                    this.xmlConsumer.characters(chars, 0, chars.length);
            }


        } finally {
            if (stream != null) {
                stream.close();
            }
        }
    }

    public void characters(char c[], int start, int len)
            throws SAXException {

        StringBuffer sBuff = null;
        // if (newCharsExpected){
        sBuff = new StringBuffer();
        // buffers.push(sBuff);
        sBuff.append(c, start, len);
        try {
            if (sBuff != null) {
                String text = sBuff.toString();
                if (this.sLocs != null) {
                    fr.gouv.culture.sdx.search.lucene.analysis.Analyzer analyzer = null;
                    Field df = this.sLocs.getDefaultField();
                    if (df != null) analyzer = df.getAnalyzer();
                	//Analyzer analyzer = new fr.gouv.culture.sdx.search.lucene.analysis.WhitespaceAnalyzer();
                	//Analyzer analyzer = new fr.gouv.culture.sdx.search.lucene.analysis.StandardAnalyzer();
                    //TODO?:always using the analyzer from the default field here, i think it is the best we can do, without getting ridiculously complicated?
                    if (analyzer != null && this.terms != null && this.terms.size() > 0) {
                        try {
                          // this.highlightTerms(df.getCode(), text, analyzer);
                        	this.highlightTerms(null, text, analyzer);
                        } catch (IOException e) {
                            throw new SAXException(e.getMessage(), e);
                        }
                    } else {
                        char[] chars = text.toCharArray();
                        this.xmlConsumer.characters(chars, 0, chars.length);
                    }
                } else {
                    char[] chars = text.toCharArray();
                    this.xmlConsumer.characters(chars, 0, chars.length);
                }
            }
        } catch (SDXException e) {
            this.xmlConsumer.characters(c, start, len);
        }

        //}
        /* else
             this.xmlConsumer.characters(c, start, len);  */
    }


    public void endElement(String uri, String loc, String raw)
            throws SAXException {
        /* try {
             StringBuffer sBuff = null;
             if (buffers.size() > 0)
                 sBuff = (StringBuffer)buffers.pop();
             if (sBuff != null && sBuff.length() > 0) {
                 String text = sBuff.toString();
                 if (this.sLocs != null) {
                     fr.gouv.culture.sdx.search.lucene.analysis.Analyzer analyzer = null;
                     Field df = this.sLocs.getDefaultField();
                     if (df != null) analyzer = df.getAnalyzer();
                     //TODO?:always using the analyzer from the default field here, i think it is the best we can do, without getting ridiculously complicated?
                     if (analyzer != null && this.terms != null && this.terms.size() > 0) {
                         try {
                             this.highlightTerms(text, this.terms, analyzer);
                         } catch (IOException e) {
                             throw new SAXException(e.getMessage(), e);
                         }
                     } else {
                         char[] chars = text.toCharArray();
                         this.xmlConsumer.characters(chars, 0, chars.length);
                     }
                 } else {
                     char[] chars = text.toCharArray();
                     this.xmlConsumer.characters(chars, 0, chars.length);
                 }
             }
         } catch (SDXException e) {
         } finally {
             //closing our element
             super.endElement(uri, loc, raw);
             //resetting our field for expecting character data
             this.newCharsExpected = false;
         }   */

        super.endElement(uri, loc, raw);


    }


    public void startElement(String uri, String loc, String raw, Attributes a)
            throws SAXException {

        if (!this.saxStreamStarted) {
            if (this.terms != null) {

                AttributesImpl topAtts = new AttributesImpl();
                //TODO : factorize analyzer instanciation in this class -pb
                //This could be done by getting the default field's analyzer in a constructor
                //And to provide a setAnalyzer method that could be used by the highlighting pipeline
                try {
                    if (this.sLocs != null) {
                        fr.gouv.culture.sdx.search.lucene.analysis.Analyzer analyzer = null;
                        Field df = this.sLocs.getDefaultField();
                        if (df != null) analyzer = df.getAnalyzer();
                        if (analyzer != null)
                            topAtts.addAttribute("", Node.Name.ANALYZER_CLASS, Node.Name.ANALYZER_CLASS, Node.Type.CDATA, analyzer.getClass().getName());
                    }
                } catch (SDXException e) {
                    topAtts.addAttribute("", Node.Name.ANALYZER_CLASS, Node.Name.ANALYZER_CLASS, Node.Type.CDATA, "unable to get an analyzer");
                }

                this.xmlConsumer.startElement(this.uri, Node.Name.HIGHLIGHT_TERMS, Framework.SDXNamespacePrefix + ":" + Node.Name.HIGHLIGHT_TERMS, topAtts);

                Enumeration list = this.terms.elements();
                while (list.hasMoreElements()) {
                    String localName = Node.Name.TERM;
                    String localQName = Framework.SDXNamespacePrefix + ":" + localName;
                    AttributesImpl localAtts = new AttributesImpl();
                    localAtts.addAttribute("", Node.Name.VALUE, Node.Name.VALUE, Node.Type.CDATA, ((String) list.nextElement()));
                    this.xmlConsumer.startElement(this.uri, localName, localQName, localAtts);
                    this.xmlConsumer.endElement(this.uri, localName, localQName);

                }


                this.xmlConsumer.endElement(this.uri, Node.Name.HIGHLIGHT_TERMS, Framework.SDXNamespacePrefix + ":" + Node.Name.HIGHLIGHT_TERMS);
                this.saxStreamStarted = true;
            }
        }

        if (a.getLength() == 0)
            super.startElement(uri, loc, raw, a);
        else {
            Attributes hilitedAtts = a;
            try {
                hilitedAtts = getHilitedAttributes(a);
            } catch (SDXException e) {
                LoggingUtils.logException(super.getLog(), e);
            } catch (IOException e) {
                LoggingUtils.logException(super.getLog(), e);
            } finally {
                super.startElement(uri, loc, raw, hilitedAtts);
            }
        }


    }

    protected Attributes getHilitedAttributes(Attributes a) throws SDXException, IOException {
        AttributesImpl hilitedAtts = new AttributesImpl(a);

        if (this.sLocs != null) {
            fr.gouv.culture.sdx.search.lucene.analysis.Analyzer analyzer = null;
            Field df = this.sLocs.getDefaultField();
            if (df != null) analyzer = df.getAnalyzer();
            if (analyzer != null && this.terms != null && this.terms.size() > 0) {
                for (int i = 0; i < a.getLength(); i++) {
                    String attName = a.getLocalName(i);
                    String attValue = a.getValue(i);
                    String text = attValue;
                    String hiliteVal = "";
                    //hiliteVal = originalTextAtTokenPosition;
                    String suffix = "-" + attName;
                    String locName = Node.Name.HILITE + suffix;
                    String qualName = Framework.SDXNamespacePrefix + ":" + Node.Name.HILITE + suffix;

                    //TODO?:always using the analyzer from the default field here, i think it is the best we can do, without getting ridiculously complicated?
                    TokenStream stream = analyzer.tokenStream(null, new StringReader(text));
                    Vector tokens = new Vector();
                    Token tmpToken;
                    int tmpStartOffset = -1;
                    int tmpEndOffset = -1;
                    int lastEndOffset = 0;
                    while ((tmpToken = stream.next()) != null || tokens.size() > 0) {
                        if (tmpStartOffset == -1)//init
                            tmpStartOffset = tmpToken.startOffset();
                        if (tmpEndOffset == -1)//init
                            tmpEndOffset = tmpToken.endOffset();
                        if (tmpToken != null && tmpToken.startOffset() == tmpStartOffset && tmpToken.endOffset() == tmpEndOffset)
                            tokens.add(tmpToken);//token for same text position
                        else if (tokens.size() > 0) {//no longer token for same text position, so process all past tokens for same text position stored in 'tokens'
                            //processing tokens stored in array list
                            Enumeration tokensForSameTextPosition = tokens.elements();
                            //the last term for which higlighting was last done
                            String lastHilitedTermFromText = "";
                            while (tokensForSameTextPosition.hasMoreElements()) {
                                org.apache.lucene.analysis.Token token = (Token) tokensForSameTextPosition.nextElement();
                                //String originalTextAtTokenPosition;
                                int startOffset;
                                int endOffset;
                                if (token != null) {
                                    startOffset = token.startOffset();
                                    endOffset = token.endOffset();
                                    //originalTextAtTokenPosition = text.substring(startOffset, endOffset);

                                    if (startOffset > lastEndOffset)
                                        lastHilitedTermFromText = "";

                                    //analye the next portion of text
                                    String termText = token.termText();
                                    // has the termText already been highlighted, if not does query contain current termText ?
                                    if (Utilities.checkString(termText) && !Utilities.checkString(lastHilitedTermFromText) && this.terms != null && this.terms.contains(termText)) {
                                        int adjustOffsetForXslStringFunctions = 1;
                                        if (Utilities.checkString(hiliteVal))
                                            hiliteVal += ";";
                                        hiliteVal += Integer.toString(startOffset + adjustOffsetForXslStringFunctions) + ":" + Integer.toString(endOffset);
                                        //hiliteVal = originalTextAtTokenPosition;
                                        lastHilitedTermFromText = termText;
                                    }

                                    //setting the end of the analyzed text
                                    lastEndOffset = endOffset;
                                }

                            }
                            //empty array list
                            tokens.clear();
                            //adding nextToken
                            if (tmpToken != null) {
                                tokens.add(tmpToken);//token for same text position
                                //resetting loop vars
                                tmpStartOffset = tmpToken.startOffset();
                                tmpEndOffset = tmpToken.endOffset();
                            }
                            //let the loop continue

                        }
                    }

                    if (Utilities.checkString(hiliteVal))
                        hilitedAtts.addAttribute(Framework.SDXNamespaceURI, locName, qualName, Node.Type.CDATA, hiliteVal);

                }
            }
        }

        return hilitedAtts;

    }


    /**
     * Recycle the producer by removing references
     */
    public void recycle() {
        //resetting some veriables
        this.saxStreamStarted = false;
        this.hliteIdx = 0;
        this.query = null;
        this.sLocs = null;
        this.terms = null;
    }

}
