add state machine parsing option

src/main/java/com/ctrip/zeus/logstats/common/AccessLogFormat.java

@@ -0,0 +1,51 @@
+package com.ctrip.zeus.logstats.common;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+/**
+ * Created by zhoumy on 2016/6/6.
+ */
+public class AccessLogFormat implements LineFormat {
+    private String format;
+    private final Map<String, String> patternRegistry = new HashMap<>();
+
+    @Override
+    public String getFormat() {
+        return null;
+    }
+
+    @Override
+    public String getPatternString() {
+        return null;
+    }
+
+    @Override
+    public Pattern getPattern() {
+        return null;
+    }
+
+    @Override
+    public String[] getKeys() {
+        return new String[0];
+    }
+
+    @Override
+    public LineFormat setFormat(String format) {
+        return null;
+    }
+
+    @Override
+    public LineFormat registerPatternForKey(String key, String pattern) {
+        return null;
+    }
+
+    @Override
+    public LineFormat generate() {
+        return null;
+    }
+
+}

src/main/java/com/ctrip/zeus/logstats/parser/AccessLogParser.java

@@ -0,0 +1,24 @@
+package com.ctrip.zeus.logstats.parser;
+
+import com.ctrip.zeus.logstats.parser.state.AccessLogStateMachine;
+import com.ctrip.zeus.logstats.parser.state.LogStatsStateMachine;
+
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/6/6.
+ */
+public class AccessLogParser implements LogParser {
+    private LogStatsStateMachine stateMachine = new AccessLogStateMachine();
+
+    @Override
+    public List<KeyValue> parse(String line) {
+
+        return null;
+    }
+
+    @Override
+    public String parseToJsonString(String line) {
+        return null;
+    }
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/AccessLogStateContext.java

@@ -0,0 +1,69 @@
+package com.ctrip.zeus.logstats.parser.state;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class AccessLogStateContext implements StateContext<String> {
+    private String value;
+    private char[] valueArray;
+    private int idx;
+    private List<String> parsedValues = new ArrayList<>();
+
+    @Override
+    public void setSourceValue(String value) {
+        this.value = value;
+        valueArray = this.value.toCharArray();
+    }
+
+    @Override
+    public int getCurrentIndex() {
+        return idx;
+    }
+
+    @Override
+    public int getStateHistoryCount() {
+        return parsedValues.size();
+    }
+
+    @Override
+    public void proceed(int length) {
+        idx += length;
+    }
+
+    @Override
+    public char[] delay(int length) {
+        char[] result = new char[length];
+        for (int i = 0; i < length && idx + i < valueArray.length; i++) {
+            result[i] = valueArray[idx + i];
+        }
+        return result;
+    }
+
+    @Override
+    public char[] getSource() {
+        return valueArray;
+    }
+
+    @Override
+    public void addResult(String result) {
+        parsedValues.add(result);
+    }
+
+    @Override
+    public String getLastStateValue() {
+        return parsedValues.get(parsedValues.size() - 1);
+    }
+
+    @Override
+    public List<String> getParsedValue() {
+        return parsedValues;
+    }
+
+    @Override
+    public boolean shouldProceed() {
+        return idx < value.length() - 1;
+    }
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/AccessLogStateMachine.java

@@ -0,0 +1,60 @@
+package com.ctrip.zeus.logstats.parser.state;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class AccessLogStateMachine implements LogStatsStateMachine {
+    private LogStatsState defaultState = new StringState();
+    private final LogStatsState initState = new WrappedStringState();
+    private LogStatsState currentState;
+
+    private Map<Integer, LogStatsState> stateInterceptor = new HashMap<>();
+    private Map<String, LogStatsState> stateRegitry = new HashMap<>();
+
+    public AccessLogStateMachine() {
+        currentState = initState;
+        stateRegitry.put("stringState", defaultState);
+    }
+
+    @Override
+    public LogStatsState getStartState() {
+        return initState;
+    }
+
+    @Override
+    public LogStatsState getNextState(LogStatsState current, StateContext ctxt) {
+        LogStatsState next = stateInterceptor.get(ctxt.getStateHistoryCount());
+        if (next == null) {
+            return stateRegitry.get("stringState");
+        } else {
+            return next;
+        }
+    }
+
+    @Override
+    public void transduce(StateContext ctxt) {
+        if (currentState.shouldDeplay()) {
+            currentState.getSubMachine().transduce(ctxt);
+        } else {
+            currentState.getTranstition().execute(ctxt);
+        }
+        ctxt.proceed(1);
+        while (ctxt.shouldProceed()) {
+            currentState = getNextState(currentState, ctxt);
+            if (currentState.shouldDeplay()) {
+                currentState.getSubMachine().transduce(ctxt);
+            } else {
+                currentState.getTranstition().execute(ctxt);
+                ctxt.proceed(1);
+            }
+        }
+    }
+
+    @Override
+    public void registerState(int idx, LogStatsState state) {
+        stateInterceptor.put(idx, state);
+    }
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/LogStatsState.java

@@ -0,0 +1,19 @@
+package com.ctrip.zeus.logstats.parser.state;
+
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public interface LogStatsState<V> {
+
+    V getOutput(StateContext ctxt);
+
+    boolean shouldDeplay();
+
+    List<Transition> getDelayedTransition();
+
+    LogStatsStateMachine getSubMachine();
+
+    Transition getTranstition();
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/LogStatsStateMachine.java

@@ -0,0 +1,14 @@
+package com.ctrip.zeus.logstats.parser.state;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public interface LogStatsStateMachine {
+    LogStatsState getStartState();
+
+    LogStatsState getNextState(LogStatsState current, StateContext ctxt);
+
+    void transduce(StateContext ctxt);
+
+    void registerState(int idx, LogStatsState state);
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/StateContext.java

@@ -0,0 +1,29 @@
+package com.ctrip.zeus.logstats.parser.state;
+
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public interface StateContext<V> {
+
+    void setSourceValue(V value);
+
+    int getCurrentIndex();
+
+    int getStateHistoryCount();
+
+    void proceed(int length);
+
+    char[] delay(int length);
+
+    char[] getSource();
+
+    void addResult(V result);
+
+    String getLastStateValue();
+
+    List<String> getParsedValue();
+
+    boolean shouldProceed();
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/StringState.java

@@ -0,0 +1,62 @@
+package com.ctrip.zeus.logstats.parser.state;
+
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class StringState implements LogStatsState<String> {
+    private Transition transition = new StringTransition();
+
+    @Override
+    public String getOutput(StateContext ctxt) {
+        transition.execute(ctxt);
+        return null;
+    }
+
+    @Override
+    public boolean shouldDeplay() {
+        return false;
+    }
+
+    @Override
+    public List<Transition> getDelayedTransition() {
+        return null;
+    }
+
+    @Override
+    public LogStatsStateMachine getSubMachine() {
+        return null;
+    }
+
+    @Override
+    public Transition getTranstition() {
+        return transition;
+    }
+
+    private class StringTransition implements Transition {
+
+        @Override
+        public void execute(StateContext ctxt) {
+            StringBuilder sb = new StringBuilder();
+            char c;
+            char[] source = ctxt.getSource();
+            for (int i = ctxt.getCurrentIndex(); i < source.length; i++) {
+                c = source[i];
+                switch (c) {
+                    case ' ': {
+                        ctxt.proceed(i - ctxt.getCurrentIndex());
+                        ctxt.addResult(sb.toString());
+                        System.out.println(sb.toString());
+                        return;
+                    }
+                    default:
+                        sb.append(c);
+                }
+            }
+            ctxt.proceed(source.length - ctxt.getCurrentIndex());
+            ctxt.addResult(sb.toString());
+            System.out.println(sb.toString());
+        }
+    }
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/Transition.java

@@ -0,0 +1,9 @@
+package com.ctrip.zeus.logstats.parser.state;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public interface Transition {
+
+    void execute(StateContext ctxt);
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/WrappedStringState.java

@@ -0,0 +1,101 @@
+package com.ctrip.zeus.logstats.parser.state;
+
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class WrappedStringState implements LogStatsState<String> {
+    private Transition transition = new WrappedStringTransition();
+
+    @Override
+    public String getOutput(StateContext ctxt) {
+        transition.execute(ctxt);
+        return null;
+    }
+
+    @Override
+    public boolean shouldDeplay() {
+        return false;
+    }
+
+    @Override
+    public List<Transition> getDelayedTransition() {
+        return null;
+    }
+
+    @Override
+    public LogStatsStateMachine getSubMachine() {
+        return null;
+    }
+
+    @Override
+    public Transition getTranstition() {
+        return transition;
+    }
+
+    private class WrappedStringTransition implements Transition {
+
+        @Override
+        public void execute(StateContext ctxt) {
+            char[] matcher = new char[]{Character.MIN_VALUE, Character.MIN_VALUE};
+            StringBuilder sb = new StringBuilder();
+            char c;
+            boolean _continue = false;
+            boolean exit = false;
+            char[] source = ctxt.getSource();
+            for (int i = ctxt.getCurrentIndex(); i < source.length; i++) {
+                c = source[i];
+                switch (c) {
+                    case '[': {
+                        if (matcher[0] == Character.MIN_VALUE) {
+                            matcher[0] = c;
+                            break;
+                        } // otherwise fall through
+                    }
+                    case ']': {
+                        if (matcher[0] == '[') {
+                            matcher[1] = ']';
+                            exit = true;
+                            break;
+                        }
+                    }
+                    case ' ': {
+                        if (matcher[0] == Character.MIN_VALUE) {
+                            matcher[0] = c;
+                            break;
+                        } else {
+                            if (matcher[0] == c) {
+                                matcher[1] = c;
+                                exit = true;
+                                break;
+                            }
+                        }
+                    }
+                    case '\"': {
+                        if (matcher[0] == Character.MIN_VALUE) {
+                            matcher[0] = c;
+                            break;
+                        } else {
+                            if (!_continue && matcher[0] == c) {
+                                matcher[1] = c;
+                                exit = true;
+                                break;
+                            }
+                        }
+                    }
+                    case '\\':
+                        _continue = !_continue;
+                    default:
+                        sb.append(c);
+                }
+                if (exit) {
+                    ctxt.proceed(i - ctxt.getCurrentIndex() + 1);
+                    ctxt.addResult(sb.toString());
+                    System.out.println(sb.toString());
+                    break;
+                }
+            }
+        }
+    }
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/extended/RequestUriState.java

@@ -0,0 +1,67 @@
+package com.ctrip.zeus.logstats.parser.state.extended;
+
+import com.ctrip.zeus.logstats.parser.state.LogStatsState;
+import com.ctrip.zeus.logstats.parser.state.LogStatsStateMachine;
+import com.ctrip.zeus.logstats.parser.state.StateContext;
+import com.ctrip.zeus.logstats.parser.state.Transition;
+
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class RequestUriState implements LogStatsState<String> {
+    private Transition transition = new RequestUriTransition();
+
+    @Override
+    public String getOutput(StateContext ctxt) {
+        return null;
+    }
+
+    @Override
+    public boolean shouldDeplay() {
+        return false;
+    }
+
+    @Override
+    public List<Transition> getDelayedTransition() {
+        return null;
+    }
+
+    @Override
+    public LogStatsStateMachine getSubMachine() {
+        return null;
+    }
+
+    @Override
+    public Transition getTranstition() {
+        return transition;
+    }
+
+    private class RequestUriTransition implements Transition {
+
+        @Override
+        public void execute(StateContext ctxt) {
+            char[] source = ctxt.getSource();
+            boolean _ignore = false;
+            StringBuilder sb = new StringBuilder();
+            for (int i = ctxt.getCurrentIndex(); i < source.length; i++) {
+                char c = source[i];
+                switch (c) {
+                    case '?': {
+                        _ignore = true;
+                    }
+                    case ' ': {
+                        ctxt.proceed(i - ctxt.getCurrentIndex());
+                        System.out.println(sb.toString());
+                        ctxt.addResult(sb.toString());
+                    }
+                    default:
+                        if (!_ignore) {
+                            sb.append(c);
+                        }
+                }
+            }
+        }
+    }
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/extended/UpstreamStateMachine.java

@@ -0,0 +1,51 @@
+package com.ctrip.zeus.logstats.parser.state.extended;
+
+import com.ctrip.zeus.logstats.parser.state.LogStatsState;
+import com.ctrip.zeus.logstats.parser.state.LogStatsStateMachine;
+import com.ctrip.zeus.logstats.parser.state.StateContext;
+import com.ctrip.zeus.logstats.parser.state.StringState;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class UpstreamStateMachine implements LogStatsStateMachine {
+    private LogStatsState stringState = new StringState();
+
+    @Override
+    public LogStatsState getStartState() {
+        return stringState;
+    }
+
+    @Override
+    public LogStatsState getNextState(LogStatsState current, StateContext ctxt) {
+        if (current == stringState) {
+            String v = ctxt.getLastStateValue();
+            if (!"".equals(v)) {
+                char[] transitId = ctxt.delay(3);
+                if (transitId[0] == ' ' && transitId[1] == ':' && transitId[2] == ' ') {
+                    ctxt.proceed(3);
+                    return stringState;
+                } else {
+                    return null;
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public void transduce(StateContext ctxt) {
+        stringState.getTranstition().execute(ctxt);
+        while (ctxt.shouldProceed()) {
+            LogStatsState next = getNextState(stringState, ctxt);
+            if (next == null)
+                return;
+            next.getTranstition().execute(ctxt);
+        }
+    }
+
+    @Override
+    public void registerState(int idx, LogStatsState state) {
+
+    }
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/extended/XForwardForState.java

@@ -0,0 +1,62 @@
+package com.ctrip.zeus.logstats.parser.state.extended;
+
+import com.ctrip.zeus.logstats.parser.state.LogStatsState;
+import com.ctrip.zeus.logstats.parser.state.LogStatsStateMachine;
+import com.ctrip.zeus.logstats.parser.state.StateContext;
+import com.ctrip.zeus.logstats.parser.state.Transition;
+
+import java.util.List;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class XForwardForState implements LogStatsState {
+    private Transition transition = new XForwardForTransition();
+
+    @Override
+    public Object getOutput(StateContext ctxt) {
+        return null;
+    }
+
+    @Override
+    public boolean shouldDeplay() {
+        return true;
+    }
+
+    @Override
+    public List<Transition> getDelayedTransition() {
+        return null;
+    }
+
+    @Override
+    public LogStatsStateMachine getSubMachine() {
+        return null;
+    }
+
+    @Override
+    public Transition getTranstition() {
+        return transition;
+    }
+
+    private class XForwardForTransition implements Transition {
+
+        @Override
+        public void execute(StateContext ctxt) {
+            StringBuilder sb = new StringBuilder();
+            char[] source = ctxt.getSource();
+            int idx = ctxt.getCurrentIndex();
+            for (; idx < source.length; idx++) {
+                char c = source[idx];
+                if (('0' <= c && c <= '9' ) || c == '.' || c == '-') {
+                    sb.append(c);
+                } else {
+                    idx--;
+                    break;
+                }
+            }
+            ctxt.addResult(sb.toString());
+            System.out.println(sb.toString());
+            ctxt.proceed(idx - ctxt.getCurrentIndex() + 1);
+        }
+    }
+}

src/main/java/com/ctrip/zeus/logstats/parser/state/extended/XForwardForStateMachine.java

@@ -0,0 +1,51 @@
+package com.ctrip.zeus.logstats.parser.state.extended;
+
+import com.ctrip.zeus.logstats.parser.state.LogStatsState;
+import com.ctrip.zeus.logstats.parser.state.LogStatsStateMachine;
+import com.ctrip.zeus.logstats.parser.state.StateContext;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class XForwardForStateMachine implements LogStatsStateMachine {
+
+    private LogStatsState xforwardState = new XForwardForState();
+
+    @Override
+    public LogStatsState getStartState() {
+        return xforwardState;
+    }
+
+    @Override
+    public LogStatsState getNextState(LogStatsState current, StateContext ctxt) {
+        if (current == xforwardState) {
+            String v = ctxt.getLastStateValue();
+            if (!"-".equals(v) && !"".equals(v)) {
+                char[] transitId = ctxt.delay(2);
+                if (transitId[0] == ',' && transitId[1] == ' ') {
+                    ctxt.proceed(2);
+                    return xforwardState;
+                } else {
+                    return null;
+                }
+            }
+        }
+        return null;
+    }
+
+    @Override
+    public void transduce(StateContext ctxt) {
+        xforwardState.getTranstition().execute(ctxt);
+        while (ctxt.shouldProceed()) {
+            LogStatsState next = getNextState(xforwardState, ctxt);
+            if (next == null)
+                return;
+            next.getTranstition().execute(ctxt);
+        }
+    }
+
+    @Override
+    public void registerState(int idx, LogStatsState state) {
+
+    }
+}

src/test/java/com/ctrip/zeus/logstats/StateMachineTest.java

@@ -0,0 +1,54 @@
+package com.ctrip.zeus.logstats;
+
+import com.ctrip.zeus.logstats.parser.state.*;
+import com.ctrip.zeus.logstats.parser.state.extended.RequestUriState;
+import com.ctrip.zeus.logstats.parser.state.extended.UpstreamStateMachine;
+import com.ctrip.zeus.logstats.parser.state.extended.XForwardForStateMachine;
+import org.junit.Test;
+
+/**
+ * Created by zhoumy on 2016/6/7.
+ */
+public class StateMachineTest {
+    @Test
+    public void testWrappedStringState() {
+        StateContext context = new AccessLogStateContext();
+        context.setSourceValue("[01/Jun/2016:09:00:15 +0800] ws.mip.hotel.ctripcorp.com svr5153hw1288 10.8.208.7 POST ");
+
+        LogStatsStateMachine accessLogStateMachine = new AccessLogStateMachine();
+        accessLogStateMachine.transduce(context);
+    }
+
+    @Test
+    public void testXForwardForState() {
+        StateContext context = new AccessLogStateContext();
+        context.setSourceValue("10.32.65.134, 10.15.202.207, 10.15.202.207, 10.15.202.207, 10.15.202.207, 10.15.202.207");
+        LogStatsStateMachine xForwardForStateMachine = new XForwardForStateMachine();
+        xForwardForStateMachine.transduce(context);
+
+        context = new AccessLogStateContext();
+        context.setSourceValue("10.32.65.134");
+        xForwardForStateMachine.transduce(context);
+
+        context = new AccessLogStateContext();
+        context.setSourceValue("-");
+        xForwardForStateMachine.transduce(context);
+    }
+
+    @Test
+    public void testRequestUriState() {
+        StateContext context = new AccessLogStateContext();
+        context.setSourceValue("[dummy] /app/index.php?param=/api/home&method=config.getAppConfig&_fxpcqlniredt=09031130410105805720");
+        LogStatsStateMachine accessLogStateMachine = new AccessLogStateMachine();
+        accessLogStateMachine.registerState(1, new RequestUriState());
+        accessLogStateMachine.transduce(context);
+    }
+
+    @Test
+    public void testUpstreamState() {
+        StateContext context = new AccessLogStateContext();
+        context.setSourceValue("- : 0.006");
+        LogStatsStateMachine upstreamStateMachine = new UpstreamStateMachine();
+        upstreamStateMachine.transduce(context);
+    }
+}