ctripcorp/zeus
1
1
Fork 0
mirror of https://github.com/ctripcorp/zeus.git synced 2024-11-11 01:24:27 +08:00
Code Issues Projects Releases Packages Wiki Activity

add test and fix bugs

Browse source
This commit is contained in:
Mengyi Zhou 2015-11-19 13:32:51 +08:00
parent 2f3ad51423
commit 79f202ec0e
5 changed files with 99 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/main/java/com/ctrip/zeus/logstats/analyzer/AccessLogStatsAnalyzer.java
View file

@ -28,11 +28,15 @@ public class AccessLogStatsAnalyzer implements LogStatsAnalyzer {
private final LogParser logParser;
public AccessLogStatsAnalyzer() throws IOException {
config = new LogStatsAnalyzerConfigBuilder()
this(new LogStatsAnalyzerConfigBuilder()
.setLogFormat(AccessLogFormat)
.setLogFilename("/opt/app/nginx/access.log")
.setTrackerReadSize(TrackerReadSize)
.build();
.build());
}
public AccessLogStatsAnalyzer(LogStatsAnalyzerConfig config) {
this.config = config;
logTracker = config.getLogTracker();
logParser = new AccessLogParser(config.getLineFormats());
}
@ -42,6 +46,16 @@ public class AccessLogStatsAnalyzer implements LogStatsAnalyzer {
return config;
}
@Override
public void start() throws IOException {
logTracker.start();
}
@Override
public void stop() throws IOException {
logTracker.stop();
}
@Override
public String analyze() throws IOException {
String raw = logTracker.move();
@ -49,16 +63,16 @@ public class AccessLogStatsAnalyzer implements LogStatsAnalyzer {
}
@Override
public void analyze(final StatsDelegate<String> delegate) throws IOException {
public void analyze(final StatsDelegate<String> delegator) throws IOException {
logTracker.fastMove(new StatsDelegate<String>() {
@Override
public void delegate(String input) {
delegate(JsonStringWriter.write(logParser.parse(input)));
delegator.delegate(JsonStringWriter.write(logParser.parse(input)));
}
});
}
private class LogStatsAnalyzerConfigBuilder {
public static class LogStatsAnalyzerConfigBuilder {
private String logFormat;
private String logFilename;
private int trackerReadSize;

4
src/main/java/com/ctrip/zeus/logstats/analyzer/LogStatsAnalyzer.java
View file

@ -11,6 +11,10 @@ public interface LogStatsAnalyzer {
LogStatsAnalyzerConfig getConfig();
void start() throws IOException;
void stop() throws IOException;
String analyze() throws IOException;
void analyze(StatsDelegate<String> delegate) throws IOException;

19
src/main/java/com/ctrip/zeus/logstats/tracker/AccessLogTracker.java
View file

@ -17,6 +17,7 @@ public class AccessLogTracker implements LogTracker {
private final ByteBuffer buffer;
private RandomAccessFile raf;
private FileChannel fileChannel;
private int offset = 0;
public AccessLogTracker(LogTrackerStrategy strategy) {
this.strategy = strategy;
@ -62,7 +63,7 @@ public class AccessLogTracker implements LogTracker {
}
@Override
public void fastMove(final StatsDelegate<String> delegate) throws IOException {
public void fastMove(final StatsDelegate<String> delegator) throws IOException {
buffer.clear();
try {
if (fileChannel.read(buffer) == -1)
@ -73,18 +74,16 @@ public class AccessLogTracker implements LogTracker {
buffer.flip();
boolean eol = false;
int colOffset = 0;
int lnOffset = 0;
byte[] line = new byte[size];
while (buffer.hasRemaining()) {
while (!eol) {
while (!eol && buffer.hasRemaining()) {
byte b;
switch (b = buffer.get()) {
case -1:
case '\n':
eol = true;
delegate.delegate(new String(line, 0, colOffset));
lnOffset = ++colOffset;
delegator.delegate(new String(line, 0, colOffset));
offset += ++colOffset;
break;
case '\r':
eol = true;
@ -92,8 +91,8 @@ public class AccessLogTracker implements LogTracker {
buffer.position(colOffset);
else
colOffset++;
delegate.delegate(new String(line, 0, colOffset));
lnOffset = ++colOffset;
delegator.delegate(new String(line, 0, colOffset));
offset += ++colOffset;
break;
default:
line[colOffset] = b;
@ -101,7 +100,9 @@ public class AccessLogTracker implements LogTracker {
break;
} // end of switch
}// end of while !eol
colOffset = 0;
eol = false;
}
fileChannel.position(lnOffset);
fileChannel.position(offset);
}
}

62
src/test/java/com/ctrip/zeus/logstats/AnalyzerTest.java Normal file
View file

@ -0,0 +1,62 @@
package com.ctrip.zeus.logstats;
import com.ctrip.zeus.logstats.analyzer.AccessLogStatsAnalyzer;
import com.ctrip.zeus.logstats.analyzer.LogStatsAnalyzer;
import com.ctrip.zeus.logstats.analyzer.LogStatsAnalyzerConfig;
import com.sun.el.stream.Stream;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Created by zhoumy on 2015/11/18.
*/
public class AnalyzerTest {
private static final String AccessLogFormat =
"[$time_local] $host $hostname $server_addr $request_method $uri " +
"\"$query_string\" $server_port $remote_user $remote_addr $http_x_forwarded_for " +
"$server_protocol \"$http_user_agent\" \"$cookie_COOKIE\" \"$http_referer\" " +
"$host $status $body_bytes_sent $request_time $upstream_response_time " +
"$upstream_addr $upstream_status";
private static final int TrackerReadSize = 2048;
private final URL accessLogUrl = this.getClass().getClassLoader().getResource("com.ctrip.zeus.service/access.log");
@Test
public void testAnalyzer() throws IOException {
final LogStatsAnalyzerConfig config =
new AccessLogStatsAnalyzer.LogStatsAnalyzerConfigBuilder()
.setLogFormat(AccessLogFormat)
.setLogFilename(accessLogUrl.getFile())
.setTrackerReadSize(TrackerReadSize)
.build();
final AtomicInteger count = new AtomicInteger();
LogStatsAnalyzer analyzer = new AccessLogStatsAnalyzer(config);
StatsDelegate reporter = new StatsDelegate<String>() {
@Override
public void delegate(String input) {
Assert.assertNotNull(input);
count.incrementAndGet();
System.out.println(input);
}
};
InputStream s = null;
try {
s = accessLogUrl.openStream();
int total = s.available();
analyzer.start();
for (int i = 0; i < total / TrackerReadSize + 1; i++) {
analyzer.analyze(reporter);
}
analyzer.stop();
} finally {
if (s != null)
s.close();
}
Assert.assertEquals(14, count.get());
}
}

9
src/test/java/com/ctrip/zeus/logstats/LogParsingTest.java
View file

@ -17,14 +17,13 @@ import java.util.List;
*/
public class LogParsingTest {
private final String accessLogFormat =
private static final String AccessLogFormat =
"[$time_local] $host $hostname $server_addr $request_method $uri " +
"\"$query_string\" $server_port $remote_user $remote_addr $http_x_forwarded_for " +
"$server_protocol \"$http_user_agent\" \"$cookie_COOKIE\" \"$http_referer\" " +
"$host $status $body_bytes_sent $request_time $upstream_response_time " +
"$upstream_addr $upstream_status";
private final String log = "[17/Nov/2015:15:10:44 +0800] ws.you.ctripcorp.com vms09191 10.8.95.27 POST /gsapi/api/xml/GetRecmdProduct \"-\" 80 - 10.8.106.66 - HTTP/1.1 \"-\" \"-\" \"-\" ws.you.ctripcorp.com 200 521 0.042 0.039 10.8.168.228:80 200";
@Test
public void testFormatParsing() {
String[] expectedKeys = {"time_local", "host", "hostname", "server_addr", "request_method", "uri",
@ -35,7 +34,7 @@ public class LogParsingTest {
String expectedPatternString = "\\[(.+)\\]\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+\\\"(.+)\\\"\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+\\\"(.+)\\\"\\s+\\\"(.+)\\\"\\s+\\\"(.+)\\\"\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)";
LineFormat lineFormat = new AccessLogLineFormat();
lineFormat.setFormat(accessLogFormat);
lineFormat.setFormat(AccessLogFormat);
String[] actualKeys = lineFormat.getKeys();
Assert.assertArrayEquals(expectedKeys, actualKeys);
Assert.assertEquals(expectedPatternString, lineFormat.getPatternString());
@ -46,7 +45,7 @@ public class LogParsingTest {
String[] expectedValues = {"17/Nov/2015:15:10:44 +0800", "ws.you.ctripcorp.com", "vms09191", "10.8.95.27", "POST", "/gsapi/api/xml/GetRecmdProduct", "-", "80", "-", "10.8.106.66", "-", "HTTP/1.1", "-", "-", "-", "ws.you.ctripcorp.com", "200", "521", "0.042", "0.039", "10.8.168.228:80", "200"};
LineFormat lineFormat = new AccessLogLineFormat();
lineFormat.setFormat(accessLogFormat);
lineFormat.setFormat(AccessLogFormat);
List<LineFormat> formats = new ArrayList<>();
formats.add(lineFormat);
final LogParser parser = new AccessLogParser(formats);
@ -61,7 +60,7 @@ public class LogParsingTest {
public void testJsonSerializer() {
String expectedJsonValue = "{\"time_local\":\"17/Nov/2015:15:10:44 +0800\",\"host\":\"ws.you.ctripcorp.com\",\"hostname\":\"vms09191\",\"server_addr\":\"10.8.95.27\",\"request_method\":\"POST\",\"uri\":\"/gsapi/api/xml/GetRecmdProduct\",\"query_string\":\"-\",\"server_port\":\"80\",\"remote_user\":\"-\",\"remote_addr\":\"10.8.106.66\",\"http_x_forwarded_for\":\"-\",\"server_protocol\":\"HTTP/1.1\",\"http_user_agent\":\"-\",\"cookie_COOKIE\":\"-\",\"http_referer\":\"-\",\"status\":\"200\",\"body_bytes_sent\":\"521\",\"request_time\":\"0.042\",\"upstream_response_time\":\"0.039\",\"upstream_addr\":\"10.8.168.228:80\",\"upstream_status\":\"200\"}";
LineFormat lineFormat = new AccessLogLineFormat();
lineFormat.setFormat(accessLogFormat);
lineFormat.setFormat(AccessLogFormat);
List<LineFormat> formats = new ArrayList<>();
formats.add(lineFormat);
final LogParser parser = new AccessLogParser(formats);