package org.wltea.analyzer.lucene; import java.io.IOException; import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.TypeAttribute;import org.wltea.analyzer.core.IKSegmenter;import org.wltea.analyzer.core.Lexeme; public class IKTokenizer extends Tokenizer {     //IK分詞器實(shí)現(xiàn)    PRivate IKSegmenter _IKImplement;          //詞元文本屬性    private final CharTermAttribute termAtt;    //詞元位移屬性    private final OffsetAttribute offsetAtt;    //詞元分類屬性（該屬性分類參考o(jì)rg.wltea.analyzer.core.Lexeme中的分類常量）    private final TypeAttribute typeAtt;    //記錄最后一個詞元的結(jié)束位置    private int endPosition;                /**    public IKTokenizer(Reader in , boolean useSmart){        super(in);        offsetAtt = addAttribute(OffsetAttribute.class);        termAtt = addAttribute(CharTermAttribute.class);        typeAtt = addAttribute(TypeAttribute.class);        _IKImplement = new IKSegmenter(input , useSmart);    }**/          /**     * Lucene 5.x Tokenizer適配器類構(gòu)造函數(shù)     * 實(shí)現(xiàn)最新的Tokenizer接口     * @param useSmart     */    public IKTokenizer(boolean useSmart){        super();        offsetAtt = addAttribute(OffsetAttribute.class);        termAtt = addAttribute(CharTermAttribute.class);        typeAtt = addAttribute(TypeAttribute.class);        _IKImplement = new IKSegmenter(input , useSmart);    }      /* (non-Javadoc)     * @see org.apache.lucene.analysis.TokenStream#incrementToken()     */    @Override    public boolean incrementToken() throws IOException {        //清除所有的詞元屬性        clearAttributes();        Lexeme nextLexeme = _IKImplement.next();        if(nextLexeme != null){            //將Lexeme轉(zhuǎn)成Attributes            //設(shè)置詞元文本            termAtt.append(nextLexeme.getLexemeText());            //設(shè)置詞元長度            termAtt.setLength(nextLexeme.getLength());            //設(shè)置詞元位移            offsetAtt.setOffset(nextLexeme.getBeginPosition(), nextLexeme.getEndPosition());            //記錄分詞的最后位置            endPosition = nextLexeme.getEndPosition();            //記錄詞元分類            typeAtt.setType(nextLexeme.getLexemeTypeString());                      //返會true告知還有下個詞元            return true;        }        //返會false告知詞元輸出完畢        return false;    }          @Override    public void reset() throws IOException {        super.reset();        _IKImplement.reset(input);    }             @Override    public final void end() {        // set final offset        int finalOffset = correctOffset(this.endPosition);        offsetAtt.setOffset(finalOffset, finalOffset);    } }
package org.wltea.analyzer.lucene; import java.io.IOException; import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;import org.apache.lucene.analysis.tokenattributes.TypeAttribute;import org.wltea.analyzer.core.IKSegmenter;import org.wltea.analyzer.core.Lexeme; public class IKTokenizer extends Tokenizer {     //IK分詞器實(shí)現(xiàn)    private IKSegmenter _IKImplement;          //詞元文本屬性    private final CharTermAttribute termAtt;    //詞元位移屬性    private final OffsetAttribute offsetAtt;    //詞元分類屬性（該屬性分類參考o(jì)rg.wltea.analyzer.core.Lexeme中的分類常量）    private final TypeAttribute typeAtt;    //記錄最后一個詞元的結(jié)束位置    private int endPosition;                /**    public IKTokenizer(Reader in , boolean useSmart){        super(in);        offsetAtt = addAttribute(OffsetAttribute.class);        termAtt = addAttribute(CharTermAttribute.class);        typeAtt = addAttribute(TypeAttribute.class);        _IKImplement = new IKSegmenter(input , useSmart);    }**/          /**     * Lucene 5.x Tokenizer適配器類構(gòu)造函數(shù)     * 實(shí)現(xiàn)最新的Tokenizer接口     * @param useSmart     */    public IKTokenizer(boolean useSmart){        super();        offsetAtt = addAttribute(OffsetAttribute.class);        termAtt = addAttribute(CharTermAttribute.class);        typeAtt = addAttribute(TypeAttribute.class);        _IKImplement = new IKSegmenter(input , useSmart);    }      /* (non-Javadoc)     * @see org.apache.lucene.analysis.TokenStream#incrementToken()     */    @Override    public boolean incrementToken() throws IOException {        //清除所有的詞元屬性        clearAttributes();        Lexeme nextLexeme = _IKImplement.next();        if(nextLexeme != null){            //將Lexeme轉(zhuǎn)成Attributes            //設(shè)置詞元文本            termAtt.append(nextLexeme.getLexemeText());            //設(shè)置詞元長度            termAtt.setLength(nextLexeme.getLength());            //設(shè)置詞元位移            offsetAtt.setOffset(nextLexeme.getBeginPosition(), nextLexeme.getEndPosition());            //記錄分詞的最后位置            endPosition = nextLexeme.getEndPosition();            //記錄詞元分類            typeAtt.setType(nextLexeme.getLexemeTypeString());                      //返會true告知還有下個詞元            return true;        }        //返會false告知詞元輸出完畢        return false;    }          @Override    public void reset() throws IOException {        super.reset();        _IKImplement.reset(input);    }             @Override    public final void end() {        // set final offset        int finalOffset = correctOffset(this.endPosition);        offsetAtt.setOffset(finalOffset, finalOffset);    } }