* Note: this does a decent job for most European languages, but does a terrible * job for some Asian languages, where words are not separated by spaces. */ class CLUCENE_EXPORT LowerCaseTokenizer:public LetterTokenizer { public: /** Construct a new LowerCaseTokenizer. */ LowerCaseTokenizer(CL_NS(util)::Reader* in); virtual ~LowerCaseTokenizer(); protected: /** Collects only characters which satisfy _totlower. */ TCHAR normalize(const TCHAR chr) const; }; /** A WhitespaceTokenizer is a tokenizer that divides text at whitespace. * Adjacent sequences of non-Whitespace characters form tokens. */ class CLUCENE_EXPORT WhitespaceTokenizer: public CharTokenizer { public: /** Construct a new WhitespaceTokenizer. */ WhitespaceTokenizer(CL_NS(util)::Reader* in); virtual ~WhitespaceTokenizer(); protected: /** Collects only characters which do not satisfy _istspace.*/ bool isTokenChar(const TCHAR c) const; }; /** An Analyzer that uses WhitespaceTokenizer. */ class CLUCENE_EXPORT WhitespaceAnalyzer: public Analyzer { public: WhitespaceAnalyzer(); TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); TokenStream* reusableTokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); virtual ~WhitespaceAnalyzer(); }; /** An Analyzer that filters LetterTokenizer with LowerCaseFilter. */ class CLUCENE_EXPORT SimpleAnalyzer: public Analyzer { public: SimpleAnalyzer(); TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); TokenStream* reusableTokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); virtual ~SimpleAnalyzer(); }; /** * Normalizes token text to lower case. */ class CLUCENE_EXPORT LowerCaseFilter: public TokenFilter { public: LowerCaseFilter(TokenStream* in, bool deleteTokenStream); virtual ~LowerCaseFilter(); Token* next(Token* token); }; /** * Removes stop words from a token stream. */ class CLUCENE_EXPORT StopFilter: public TokenFilter { private: //bvk: i found this to work faster with a non-hash table. the number of items //in the stop table is not like to make it worth having hashing. //ish: implement a radix/patricia tree for this? CLTCSetList* stopWords; bool deleteStopTable; bool enablePositionIncrements; const bool ignoreCase; public: static bool ENABLE_POSITION_INCREMENTS_DEFAULT; // Constructs a filter which removes words from the input // TokenStream that are named in the array of words. StopFilter(TokenStream* in, bool deleteTokenStream, const TCHAR** _stopWords, const bool _ignoreCase = false); virtual ~StopFilter(); /** Constructs a filter which removes words from the input * TokenStream that are named in the CLSetList. */ StopFilter(TokenStream* in, bool deleteTokenStream, CLTCSetList* stopTable, bool _deleteStopTable=false); /** * Builds a Hashtable from an array of stop words, appropriate for passing * into the StopFilter constructor. This permits this table construction to * be cached once when an Analyzer is constructed. * Note: the stopWords list must be a static list because the strings are not copied */ static void fillStopTable(CLTCSetList* stopTable, const TCHAR** stopWords, const bool _ignoreCase = false); /** * Returns the next input Token whose termText() is not a stop word. */ Token* next(Token* token); /** * @see #setEnablePositionIncrementsDefault(boolean). */ static bool getEnablePositionIncrementsDefault(); /** * Set the default position increments behavior of every StopFilter created from now on. *
* Note: behavior of a single StopFilter instance can be modified * with {@link #setEnablePositionIncrements(boolean)}. * This static method allows control over behavior of classes using StopFilters internally, * for example {@link lucene::analysis::standard::StandardAnalyzer StandardAnalyzer}. *
* Default : false.
* @see #setEnablePositionIncrements(boolean).
*/
static void setEnablePositionIncrementsDefault(const bool defaultValue);
/**
* @see #setEnablePositionIncrements(boolean).
*/
bool getEnablePositionIncrements() const;
/**
* Set to true to make this StopFilter enable position increments to result tokens.
*
* When set, when a token is stopped (omitted), the position increment of * the following token is incremented. *
* Default: see {@link #setEnablePositionIncrementsDefault(boolean)}. */ void setEnablePositionIncrements(const bool enable); }; /** * Loader for text files that represent a list of stopwords. * */ class CLUCENE_EXPORT WordlistLoader { public: /** * Loads a text file and adds every line as an entry to a HashSet (omitting * leading and trailing whitespace). Every line of the file should contain only * one word. The words need to be in lowercase if you make use of an * Analyzer which uses LowerCaseFilter (like StandardAnalyzer). * * @param wordfile File containing the wordlist * @return A HashSet with the file's words */ static CLTCSetList* getWordSet(const char* wordfilePath, const char* enc = NULL, CLTCSetList* stopTable = NULL); /** * Reads lines from a Reader and adds every line as an entry to a HashSet (omitting * leading and trailing whitespace). Every line of the Reader should contain only * one word. The words need to be in lowercase if you make use of an * Analyzer which uses LowerCaseFilter (like StandardAnalyzer). * * @param reader Reader containing the wordlist * @return A HashSet with the reader's words */ static CLTCSetList* getWordSet(CL_NS(util)::Reader* reader, CLTCSetList* stopTable = NULL, const bool bDeleteReader = false); }; /** Filters LetterTokenizer with LowerCaseFilter and StopFilter. */ class CLUCENE_EXPORT StopAnalyzer: public Analyzer { CLTCSetList* stopTable; class SavedStreams; public: /** Builds an analyzer which removes words in ENGLISH_STOP_WORDS. */ StopAnalyzer(); virtual ~StopAnalyzer(); /** Builds an analyzer which removes words in the provided array. */ StopAnalyzer( const TCHAR** stopWords ); /** Builds an analyzer with the stop words from the given file. * @see WordlistLoader#getWordSet(File) */ StopAnalyzer(const char* stopwordsFile, const char* enc = NULL); /** Builds an analyzer with the stop words from the given reader. * @see WordlistLoader#getWordSet(Reader) */ StopAnalyzer(CL_NS(util)::Reader* stopwordsReader, const bool _bDeleteReader = false); /** Filters LowerCaseTokenizer with StopFilter. */ TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); TokenStream* reusableTokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader); /** An array containing some common English words that are not usually useful for searching. */ static const TCHAR* ENGLISH_STOP_WORDS[]; }; /** * This analyzer is used to facilitate scenarios where different * fields require different analysis techniques. Use {@link #addAnalyzer} * to add a non-default analyzer on a field name basis. * *
Example usage: * *
* PerFieldAnalyzerWrapper* aWrapper =
* new PerFieldAnalyzerWrapper(new StandardAnalyzer());
* aWrapper.addAnalyzer("firstname", new KeywordAnalyzer());
* aWrapper.addAnalyzer("lastname", new KeywordAnalyzer());
*
*
* In this example, StandardAnalyzer will be used for all fields except "firstname" * and "lastname", for which KeywordAnalyzer will be used. * *
A PerFieldAnalyzerWrapper can be used like any other analyzer, for both indexing
* and query parsing.
*/
class CLUCENE_EXPORT PerFieldAnalyzerWrapper : public Analyzer {
private:
Analyzer* defaultAnalyzer;
typedef CL_NS(util)::CLHashMap
* For instance, 'à' will be replaced by 'a'.
*
*/
class CLUCENE_EXPORT ISOLatin1AccentFilter: public TokenFilter {
public:
ISOLatin1AccentFilter(TokenStream* input, bool deleteTs);
/**
* To replace accented characters in a String by unaccented equivalents.
*/
Token* next(Token* token);
virtual ~ISOLatin1AccentFilter();
};
/**
* Emits the entire input as a single token.
*/
class CLUCENE_EXPORT KeywordTokenizer: public Tokenizer {
private:
LUCENE_STATIC_CONSTANT(int, DEFAULT_BUFFER_SIZE = 256);
bool done;
int bufferSize;
public:
KeywordTokenizer(CL_NS(util)::Reader* input, int bufferSize=-1);
Token* next(Token* token);
void reset(CL_NS(util)::Reader* input);
virtual ~KeywordTokenizer();
};
/**
* "Tokenizes" the entire stream as a single token. This is useful
* for data like zip codes, ids, and some product names.
*/
class CLUCENE_EXPORT KeywordAnalyzer: public Analyzer {
public:
KeywordAnalyzer();
virtual ~KeywordAnalyzer();
TokenStream* tokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
TokenStream* reusableTokenStream(const TCHAR* fieldName, CL_NS(util)::Reader* reader);
};
/**
* Removes words that are too long and too short from the stream.
*
*/
class CLUCENE_EXPORT LengthFilter: public TokenFilter {
private:
size_t _min;
size_t _max;
public:
/**
* Build a filter that removes words that are too long or too
* short from the text.
*/
LengthFilter(TokenStream* in, const size_t _min, const size_t _max);
/**
* Returns the next input Token whose termText() is the right len
*/
Token* next(Token* token);
};
CL_NS_END
#endif