package test; /** * * 分词每个词语，并且统计词频 * */
import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import jeasy.analysis.MMAnalyzer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream;
public class Segment {
public static void main(String args[]) throws IOException { Segment s = new Segment(); String text = s.ReadFileByBufferdeReader("F://test.txt"); // System.out.println(text); s.getWordByReader(text); }
public String ReadFileByBufferdeReader(String readFileName) { String temp = ""; File f = new File(readFileName);
InputStreamReader read; try { read = new InputStreamReader(new FileInputStream(f), "gb2312"); BufferedReader reader = new BufferedReader(read); String line; while ((line = reader.readLine()) != null) { temp += line + "\n"; } } catch (UnsupportedEncodingException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (FileNotFoundException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } return temp; }
public Map getWordByReader(String text) { // 采用正向最大匹配的中文分词算法 MMAnalyzer analyzer = new MMAnalyzer(); analyzer.addWord("任家坪"); Map<String, Integer> map = new HashMap<String, Integer>(); try { System.out.println("Length = " + text.length()); Reader r = new StringReader(text); TokenStream ts = analyzer.tokenStream(null, r); //Creates a TokenStream which tokenizes all the text in the provided Reader.
System.out.println("开始分词...\n"); long begin = System.currentTimeMillis(); //返回以毫秒为单位的当前时间 for (Token t = ts.next(); t != null; t = ts.next()) {
String str = t.termText(); // Returns the Token's term text. // System.out.println(str); Object o = map.get(str); // 返回指定键所映射的值；如果此映射不包含该键的映射关系，则返回 null if (o == null) { map.put(str, new Integer(1)); } else { Integer I = new Integer(((Integer) o).intValue() + 1); map.put(str, I); //将指定的值与此映射中的指定键关联 } }
for (Iterator iter = map.entrySet().iterator(); iter.hasNext();) { Map.Entry entry = (Map.Entry) iter.next(); //映射项（键-值对）。 System.out.println(entry.getKey() + ":" + entry.getValue()); //输出词汇及频率 } long end = System.currentTimeMillis(); // System.out.println("分词数量: " + map.size() + " 耗时 : " + (end - begin) // + "ms");
return map; } catch (IOException e) { e.printStackTrace(); } return map; }
}