分析ngnix日志文件的开发程序记录
package com.school.statistics;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.RandomAccessFile;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.lang3.StringUtils;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import com.school.dao.CourseContentMapper;
import com.school.dao.StatisticsDurationCuMapper;
import com.school.dao.StatisticsDurationMapper;
import com.school.domain.CourseContent;
import com.school.domain.CourseContentJson;
import com.school.domain.CourseContentJson.Lecture;
import com.school.domain.CourseContentJson.Section;
import com.school.domain.StatisticsDuration;
import com.school.domain.StatisticsDurationCu;
public class AnalysisDurationLogs implements Runnable {
private static ApplicationContext applicationContext = null;
public static StatisticsDurationMapper statisticsDurationMapper;// 明细统计
public static StatisticsDurationCuMapper statisticsDurationCuMapper;// 根据课程统计
public static CourseContentMapper courseContentMapper;// 课时内容
public static Integer sleepTime = 1 * 60 * 1000;// 时间(多少时间执行一次)
public static String logFilePath = "/data/nginx/logs/vlms_access.log";
public static String seekFilePath = "/data/nginx/logs/";
// 构造函数
public AnalysisDurationLogs() {
if (applicationContext == null) {
applicationContext = new ClassPathXmlApplicationContext(new String[] { "applicationContext-resources.xml",
"classpath:applicationContext-dao.xml" });
}
statisticsDurationMapper = (StatisticsDurationMapper) applicationContext.getBean("statisticsDurationMapper");
statisticsDurationCuMapper = (StatisticsDurationCuMapper) applicationContext
.getBean("statisticsDurationCuMapper");
courseContentMapper = (CourseContentMapper) applicationContext.getBean("courseContentMapper");
}
// 实现run方法
@Override
public void run() {
System.out.println(Thread.currentThread().getName());
try {
startFunction(sleepTime);// 启动10秒执行一次
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public static void main(String[] args) throws IOException {
System.out.println("-------统计分析----------");
AnalysisDurationLogs a1 = new AnalysisDurationLogs();
Thread t1 = new Thread(a1);
t1.start();
// startFunction();
}
/**
* 取出读取标记，进行文件读取分析-记录读取标记
*/
private static void startFunction(int stopTime) throws IOException, FileNotFoundException {
while (true) {
try {
Thread.sleep(stopTime);
int row = 0;
System.out.println("--------------------------------kai----------------------------------");
System.out.println("---------------------------------------------------------------------");
SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
String datefile = sdf.format(new Date());
String lineMarkFilePath = seekFilePath + datefile + "seek.txt";
File file = new File(lineMarkFilePath);
if (!file.exists()) {
file.createNewFile();
}
BufferedReader bufr = new BufferedReader(new InputStreamReader(new FileInputStream(lineMarkFilePath)));
String rowStr = null;
if ((rowStr = bufr.readLine()) != null) {
if (StringUtils.isNotBlank(rowStr) || "".equals(rowStr)) {
// row=Integer.valueOf(rowStr).intValue();
row = Integer.parseInt(rowStr.trim());
// row = bufr.read();
}
}
bufr.close();
System.out.println("--------r1--------" + row);
// 每隔10秒去分析一下日志文件
Integer markline = getLogs(row);
// FileOutputStream(lineMarkFilePath,false)//false为覆盖文本，true为追加文本
BufferedWriter bufw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(lineMarkFilePath,
false), "UTF-8"));
System.out.println("--------r2--------" + markline);
String lineStr = null;
lineStr = markline.toString();
if ((lineStr) != null) {
bufw.write(lineStr);
// bufw.newLine();///换行
bufw.flush();// 刷新
}
bufw.close();
// 筛选数据
System.out.println("--------综合表--------");
selectDurationData();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
/**
* 打开分析nginx文件
*/
public static int getLogs(Integer row) throws FileNotFoundException, IOException {
RandomAccessFile raf = new RandomAccessFile(logFilePath, "rw");
raf.seek(row);// 移动文件指针位置
String line = null;
int i = row;
Map<String, StatisticsDuration> map = new HashMap<String, StatisticsDuration>();
while ((line = raf.readLine()) != null) {
// i++;
// System.out.println("----count------"+line);
Pattern p = Pattern.compile("((2[0-4]\\d|25[0-5]|[01]?\\d\\d?)\\.){3}(2[0-4]\\d|25[0-5]|[01]?\\d\\d?)");
Matcher m = p.matcher(line);
String ipv4 = "";
if (m.find()) {
// 找到ipv4//使用if而不使用while母的是只找第一个就结束寻找，因为ip放在前面，
ipv4 = m.group();
// System.out.println("get-----" +m.group());
}
String timeStr = "";
timeStr = line.substring(line.indexOf("[") + 1, line.lastIndexOf("]"));
String time = "";
Pattern pt = Pattern.compile("(\\d{2})/([^&]*)/\\d{4}:\\d{2}:\\d{2}:\\d{2}");
Matcher mt = pt.matcher(timeStr);
while (mt.find()) {
// 找到时间串
time = mt.group();
SimpleDateFormat sdf = new SimpleDateFormat("d/MMM/yyyy:K:m:s", Locale.ENGLISH);
SimpleDateFormat sdfnew = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date d2 = null;
try {
d2 = sdf.parse(time.trim());
} catch (ParseException e) {
e.printStackTrace();
}
time = sdfnew.format(d2);
}
i = (int) raf.getFilePointer();// 记录当前读取位置
// 获取参数
String pid = getParameterRex(line, "pid");
String schoolID = getParameterRex(line, "schoolID");
String courseID = getParameterRex(line, "courseID");
String lectureID = getParameterRex(line, "lectureID");
// String sectionID=getParameterRex(line, "sectionID");//参数正则获取法
String sectionID = getSectionId(lectureID, 2);// 截取获取法(基于传过来的参数不正确)
String userID = getParameterRex(line, "userID");
String duration = getParameterRex(line, "duration");
String Browser = getParameterRex(line, "Browser");
String terminal = getParameterRex(line, "terminal");
System.out.println(i + "---t---" + time + "---ip---" + ipv4 + "---pid----" + pid + "----s---" + schoolID
+ "---c---" + courseID + "---s---" + sectionID + "---l---" + lectureID + "---u---" + userID
+ "---b---" + Browser + "---t---terminal+----d----" + duration);
StatisticsDuration statisticsDuration = new StatisticsDuration();
if (StringUtils.isNotBlank(pid.trim())) {
statisticsDuration.setPid(pid);
statisticsDuration.setSchoolId(schoolID);
statisticsDuration.setCourseId(courseID);
statisticsDuration.setSectionId(sectionID);
statisticsDuration.setLectureId(lectureID);
statisticsDuration.setUserId(userID);
statisticsDuration.setIpv4(ipv4);
if (StringUtils.isNotBlank(duration.trim())) {
statisticsDuration.setDuration(Integer.parseInt(duration.trim()));
}
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date = null;
try {
date = sdf.parse(time);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
statisticsDuration.setDateEnd(date);
statisticsDuration.setBrowser(Browser);
statisticsDuration.setTerminal(terminal);
map.put(pid, statisticsDuration);// 存进map集合
}
}
raf.close();
// 调用存进数据库
System.out.println("-------insert明细表--------");
insertData(map);
map.clear();
return i;
}
/**
* 存进数据库 //入库操作
*/
private static void insertData(Map<String, StatisticsDuration> map) {
if (map.size() > 0) {
Set<String> keyi = map.keySet();
java.util.Iterator<String> it = keyi.iterator();
while (it.hasNext()) {
String key = it.next();
// System.out.println(dbv.getPid());
StatisticsDuration statisticsDuration = map.get(key);
StatisticsDuration selectSD = statisticsDurationMapper.selectByPrimaryKey(statisticsDuration.getPid());// /查询是否存在
if (selectSD != null) {
// 理论上不会执行
statisticsDurationMapper.updateByPrimaryKeySelective(statisticsDuration);// 更新操作
} else {
statisticsDurationMapper.insertSelective(statisticsDuration);// 入库操作
}
}
}
}
/**
* 提取请求里的参数
*/
public static String getParameterRex(String str, String name) {
String rex = "(?:^|\\?|&)" + name + "=([^&]*)(?:&|$)";
Pattern p = Pattern.compile(rex);
Matcher m = p.matcher(str);
String param = "";
while (m.find()) {
// 找到相应的参数
param = m.group();
// System.out.println("value-----" +m.group());
if (StringUtils.isNotBlank(param)) {
try {
param = param.substring(param.indexOf("=") + 1, param.lastIndexOf("&"));
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
param = "";
}
}
}
return param;
}
/**
* 从lectureId里截取sectionId
*/
private static String getSectionId(String lectureId, int index) {
try {
String[] ss = lectureId.split("\\.");
StringBuffer sb = new StringBuffer();
String result = "";
for (int i = 0; i < 2; i++) {
sb.append(ss[i]).append(".");
}
result = sb.toString();
result = result.substring(0, result.length() - 2);
return result;
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
return "";
}
}
/**
* 筛选数据统计
*/
public static void selectDurationData() {
// 今天的日期
// SimpleDateFormat sdfOne = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat sdfTwo = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
String today = sdfTwo.format(new Date());
// System.out.println("--------------"+today);
// 明天的日期
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DAY_OF_YEAR, 1);
Date date = calendar.getTime();
String tomorrow = sdfTwo.format(date);
// System.out.println("------m-------"+tomorrow);
// 筛选当天的数据(根据传过去的时间)，筛选明细表的数据，去插入综合表
List<StatisticsDurationCu> list = statisticsDurationMapper.selectDurationCu(today, tomorrow);
// System.out.println("------zzzsi------"+list.size());
if (list.size() > 0) {
for (int i = 0; i < list.size(); i++) {
String schoolId = list.get(i).getSchoolId();
String courseId = list.get(i).getCourseId();
String lectureId = list.get(i).getLectureId();
String userId = list.get(i).getUserId();
String sectionId = list.get(i).getSectionId();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
String onlyDate = sdf.format(list.get(i).getDate());
String sectionTitle = "";
String lectureTitle = "";
boolean insertMark = false;// 预防lecture_id不存在的情况，存在可以insert
// 获取章节标题、与课时标题
// CourseContentExample exampleC = new CourseContentExample();
// exampleC.createCriteria().andSchoolIdEqualTo(schoolId).andCourseIdEqualTo(courseId);
// List<CourseContent> courseContentList = courseContentMapper.selectByExample(exampleC);
CourseContent courseContent = courseContentMapper.selectByPrimaryKey(courseId);
if (courseContent != null) {
CourseContentJson ccj = courseContent.getCourseContentJson();
List<Section> sections = ccj.getSections();// 获取章节列表
if (sections.size() > 0) {
// 遍历章节
for (int j = 0; j < sections.size(); j++) {
if (insertMark) {
break;// 找到了课时title就没必要继续遍历其他的section
}
Section s = sections.get(j);
System.out.println("------secIdccc-------" + s.getSectionId());
// 因为拖动排序，没有章节概念，所以找课时标题，只能从所有的章节去找
List<Lecture> lecList = sections.get(j).getLectures();// 获取到课时列表
if (lecList.size() > 0) {
for (int k = 0; k < lecList.size(); k++) {
if (lectureId.equals(lecList.get(k).getLectureId())) {
sectionTitle = sections.get(j).getTitle();// 找到不一定是所需要的章节，并获取章节的标题
lectureTitle = lecList.get(k).getTitle();
insertMark = true;
System.out.print("-------找到le-------" + lectureTitle);
break;
}
}
System.out.println("--------lecList结束--------");
}
}
}
}
// /insertMark为true表示在课程内容表(course_content)存在相应的lecture_id，
if (insertMark) {
// 先去查询，该记录存不存在
List<StatisticsDurationCu> sdcu = statisticsDurationCuMapper.selectExistDurationCu(onlyDate,
schoolId, courseId, userId, lectureId);
if (sdcu.size() > 0) {
// 存在该条数据，去更新该条数据
StatisticsDurationCu statisticsDurationCu = sdcu.get(0);
float durationS = list.get(i).getDuration();
float durationH = durationS / 3600;
statisticsDurationCu.setDuration(durationH);// 新的时长替换旧的时长
statisticsDurationCu.setCount(list.get(i).getCount());
statisticsDurationCu.setSectionTitle(sectionTitle);
statisticsDurationCu.setLectureTitle(lectureTitle);
// 根据Id是唯一的，去更新时长、次数
statisticsDurationCuMapper.updateDurationCountToday(statisticsDurationCu);
System.out.println("-----updatezzzzz---------");
} else {
// 不存在该条数据,
StatisticsDurationCu sdc = new StatisticsDurationCu();
sdc = list.get(i);
float durationS = sdc.getDuration();
float durationH = durationS / 3600;
sdc.setDuration(durationH);
sdc.setSectionTitle(sectionTitle);
sdc.setLectureTitle(lectureTitle);
statisticsDurationCuMapper.insertSelective(sdc);
System.out.println("-----insertzzzz---------");
}
}
}
}
}
}
遇到故障重新找数据（根据时间）
package com.school.statistics;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import com.school.dao.CourseContentMapper;
import com.school.dao.StatisticsDurationCuMapper;
import com.school.dao.StatisticsDurationMapper;
import com.school.domain.CourseContent;
import com.school.domain.CourseContentJson;
import com.school.domain.CourseContentJson.Lecture;
import com.school.domain.CourseContentJson.Section;
import com.school.domain.StatisticsDurationCu;
/**
* 遇到故障了，根据传过来的日期去筛选数据，insert到综合表
*/
public class FailureAndInsert {
private static ApplicationContext applicationContext = null;
public static StatisticsDurationMapper statisticsDurationMapper;// 明细统计
public static StatisticsDurationCuMapper statisticsDurationCuMapper;// 根据课程统计
public static CourseContentMapper courseContentMapper;// 课时内容
public static String breakDate = "2014-10-10 00:00:00";// 故障日期
// 构造函数
public FailureAndInsert() {
}
public static void main(String[] args) {
BreakDownInsert(breakDate);
System.out.println("-------------------程序运行结束可以终止------------------------------");
}
/**
* 遇到故障了，根据传过来的日期去筛选数据，insert到综合表
*/
public static void BreakDownInsert(String breakDate) {
if (applicationContext == null) {
applicationContext = new ClassPathXmlApplicationContext(new String[] { "applicationContext-resources.xml",
"applicationContext-business.xml", "classpath:applicationContext-dao.xml" });
}
statisticsDurationMapper = (StatisticsDurationMapper) applicationContext.getBean("statisticsDurationMapper");
statisticsDurationCuMapper = (StatisticsDurationCuMapper) applicationContext
.getBean("statisticsDurationCuMapper");
courseContentMapper = (CourseContentMapper) applicationContext.getBean("courseContentMapper");
// 今天的日期
// SimpleDateFormat sdfOne = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat sdfTwo = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
Date todayStr = null;
try {
todayStr = sdfTwo.parse(breakDate);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
String today = sdfTwo.format(todayStr);
System.out.println("------t--------" + today);
// 明天的日期
Calendar calendar = Calendar.getInstance();
calendar.setTime(todayStr);
int day = calendar.get(Calendar.DATE);
calendar.set(Calendar.DATE, day + 1);
String tomorrow = new SimpleDateFormat("yyyy-MM-dd 00:00:00").format(calendar.getTime());
System.out.println("------m-------" + tomorrow);
// 筛选当天的数据(根据传过去的时间)，筛选明细表的数据，去插入综合表
List<StatisticsDurationCu> list = statisticsDurationMapper.selectDurationCu(today, tomorrow);
System.out.println("------zzzsi------" + list.size());
if (list.size() > 0) {
for (Integer i = 0; i < list.size(); i++) {
String schoolId = list.get(i).getSchoolId();
String courseId = list.get(i).getCourseId();
String lectureId = list.get(i).getLectureId();
String userId = list.get(i).getUserId();
String sectionId = list.get(i).getSectionId();
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
String onlyDate = sdf.format(list.get(i).getDate());
String sectionTitle = "";
String lectureTitle = "";
boolean insertMark = false;// 预防lecture_id不存在的情况，存在可以insert
System.out.print("-----m-------" + i);
System.out.print("-----secid-------" + sectionId);
System.out.println("-----lecid-------" + lectureId);
// 获取章节标题、与课时标题
// CourseContentExample exampleC = new CourseContentExample();
// exampleC.createCriteria().andSchoolIdEqualTo(schoolId).andCourseIdEqualTo(courseId);
// List<CourseContent> courseContentList = courseContentMapper.selectByExample(exampleC);
CourseContent courseContent = courseContentMapper.selectByPrimaryKey(courseId);
if (courseContent != null) {
CourseContentJson ccj = courseContent.getCourseContentJson();
List<Section> sections = ccj.getSections();// 获取章节列表
if (sections.size() > 0) {
// 遍历章节
for (int j = 0; j < sections.size(); j++) {
if (insertMark) {
break;// 找到了课时title就没必要继续遍历其他的section
}
Section s = sections.get(j);
System.out.println("------secIdccc-------" + s.getSectionId());
// 因为拖动排序，没有章节概念，所以找课时标题，只能从所有的章节去找
List<Lecture> lecList = sections.get(j).getLectures();// 获取到课时列表
if (lecList.size() > 0) {
for (int k = 0; k < lecList.size(); k++) {
if (lectureId.equals(lecList.get(k).getLectureId())) {
sectionTitle = sections.get(j).getTitle();// 找到不一定是所需要的章节，并获取章节的标题
lectureTitle = lecList.get(k).getTitle();
insertMark = true;
System.out.print("-------le找到-------" + lectureTitle);
break;
}
}
System.out.println("--------lecList结束--------");
}
}
}
}
// ///--end
System.out.println("---only------" + onlyDate);
// /insertMark为true表示在课程内容表(course_content)存在相应的lecture_id，
if (insertMark) {
// 先去查询，该记录存不存在
List<StatisticsDurationCu> sdcu = statisticsDurationCuMapper.selectExistDurationCu(onlyDate,
schoolId, courseId, userId, lectureId);
if (sdcu.size() > 0) {
// 存在该条数据，去更新该条数据
StatisticsDurationCu statisticsDurationCu = sdcu.get(0);
float durationS = list.get(i).getDuration();
float durationH = durationS / 3600;
statisticsDurationCu.setDuration(durationH);// 新的时长替换旧的时长
statisticsDurationCu.setCount(list.get(i).getCount());
statisticsDurationCu.setSectionTitle(sectionTitle);
statisticsDurationCu.setLectureTitle(lectureTitle);
// 根据Id是唯一的，去更新时长、次数
statisticsDurationCuMapper.updateDurationCountToday(statisticsDurationCu);
System.out.println("-----updatezzzzz---------");
} else {
// 不存在该条数据,
StatisticsDurationCu sdc = new StatisticsDurationCu();
sdc = list.get(i);
float durationS = sdc.getDuration();
float durationH = durationS / 3600;
sdc.setDuration(durationH);
sdc.setSectionTitle(sectionTitle);
sdc.setLectureTitle(lectureTitle);
statisticsDurationCuMapper.insertSelective(sdc);
System.out.println("-----insertzzzz---------");
}
}
}
}
}
}
package com.school.statistics;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import com.school.dao.StatisticsDurationCuMapper;
import com.school.dao.StatisticsDurationMapper;
import com.school.domain.StatisticsDurationExample;
/*
* 删除明细表的数据
*/
public class DeleteDetailData {
private static ApplicationContext applicationContext = null;
public static StatisticsDurationMapper statisticsDurationMapper;// 明细统计
public static StatisticsDurationCuMapper statisticsDurationCuMapper;// 根据课程统计
public static String breakDate = "2012-09-18 00:00:00";// 故障日期
// 构造函数
public DeleteDetailData() {
}
public static void main(String[] args) {
deleteData(breakDate);
}
/**
* 删除明细表的数据
*/
public static void deleteData(String breakDate) {
if (applicationContext == null) {
applicationContext = new ClassPathXmlApplicationContext(new String[] { "applicationContext-resources.xml",
"classpath:applicationContext-dao.xml" });
}
statisticsDurationMapper = (StatisticsDurationMapper) applicationContext.getBean("statisticsDurationMapper");
statisticsDurationCuMapper = (StatisticsDurationCuMapper) applicationContext
.getBean("statisticsDurationCuMapper");
// 把用户输入的日期转成 java 日期类
DateFormat df = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
Date deleteTime = null;
try {
deleteTime = df.parse(breakDate);
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
// 删除小于这个时间的记录---明细表
StatisticsDurationExample example = new StatisticsDurationExample();
example.createCriteria().andDateEndLessThanOrEqualTo(deleteTime);
statisticsDurationMapper.deleteByExample(example);
}
}
原文：http://blog.csdn.net/kenhins/article/details/40184317