merge feature-bhdownload

cmd/root.go

@@ -34,13 +34,10 @@ var cfgFile string
 
 // rootCmd represents the base command when called without any subcommands
 var (
-	noWarning bool
+	rootCmd = &cobra.Command{
-	rootCmd   = &cobra.Command{
 		Run: func(cmd *cobra.Command, args []string) {
 			cmd.Flags().Parse(args)
-			if !noWarning {
+			checkBhEnv()
-				checkBhEnv()
-			}
 			startupMessage()
 			internal.Run()
 		},
@@ -60,7 +57,6 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&internal.LogFile, "log", "", `Set filepath for HTTP log. "" logs to stderr`)
 	rootCmd.PersistentFlags().StringVar(&internal.DbPath, "db", dbPath(), "DB location (sqlite or postgres)")
 	rootCmd.PersistentFlags().StringVarP(&internal.Addr, "addr", "a", listenAddr(), "Ip and port to listen and serve on")
-	rootCmd.PersistentFlags().BoolVarP(&noWarning, "warning-disable", "w", false, `Disable BH_URL env variable startup warning`)
 
 }
 

cmd/transfer.go

@@ -0,0 +1,328 @@
+/*
+ *
+ * Copyright © 2020 nicksherron <nsherron90@gmail.com>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"strings"
+	"sync"
+
+	"github.com/cheggaaa/pb/v3"
+	"github.com/spf13/cobra"
+)
+
+type cList struct {
+	UUID    string `json:"uuid"`
+	Command string `json:"command"`
+	Created int64  `json:"created"`
+}
+
+type commandsList []cList
+
+var (
+	barTemplate   = `{{string . "message"}}{{counters . }} {{bar . }} {{percent . }} {{speed . "%s inserts/sec" }}`
+	bar           *pb.ProgressBar
+	progress      bool
+	srcUser       string
+	dstUser       string
+	srcURL        string
+	dstURL        string
+	srcPass       string
+	dstPass       string
+	srcToken      string
+	dstToken      string
+	sysRegistered bool
+	workers		  int
+	wg            sync.WaitGroup
+	cmdList       commandsList
+	transferCmd    = &cobra.Command{
+		Use:   "transfer",
+		Short: "transfer bashhub history ",
+		Run: func(cmd *cobra.Command, args []string) {
+			cmd.Flags().Parse(args)
+			sysRegistered = false
+			srcToken = getToken(srcURL, srcUser, srcPass)
+			sysRegistered = false
+			dstToken = getToken(dstURL, dstUser, dstPass)
+			cmdList = getCommandList()
+			counter := 0
+			if !progress {
+				bar = pb.ProgressBarTemplate(barTemplate).Start(len(cmdList)).SetMaxWidth(70)
+				bar.Set("message", "inserting records \t")
+			}
+			for _, v := range cmdList {
+				wg.Add(1)
+				counter++
+				go func(c cList) {
+					defer wg.Done()
+					commandLookup(c.UUID)
+				}(v)
+				if counter > workers {
+					wg.Wait()
+					counter = 0
+				}
+			}
+			wg.Wait()
+		},
+	}
+)
+
+func init() {
+	rootCmd.AddCommand(transferCmd)
+	transferCmd.PersistentFlags().StringVar(&srcURL, "src-url", "https://bashhub.com", "source url")
+	transferCmd.PersistentFlags().StringVar(&srcUser, "src-user", "", "source username")
+	transferCmd.PersistentFlags().StringVar(&srcPass, "src-pass", "", "source password")
+	transferCmd.PersistentFlags().StringVar(&dstURL, "dst-url", "http://localhost:8080", "destination url")
+	transferCmd.PersistentFlags().StringVar(&dstUser, "dst-user", "", "destination username")
+	transferCmd.PersistentFlags().StringVar(&dstPass, "dst-pass", "", "destination password")
+	transferCmd.PersistentFlags().BoolVarP(&progress, "progress", "p", false, "show progress bar")
+	transferCmd.PersistentFlags().IntVarP(&workers, "workers", "w", 10, "max number of concurrent requests")
+}
+
+func sysRegister(mac string, site string, user string, pass string) string {
+
+	var token string
+	func() {
+		var null *string
+		auth := map[string]interface{}{
+			"username": user,
+			"password": pass,
+			"mac":      null,
+		}
+
+		payloadBytes, err := json.Marshal(auth)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		body := bytes.NewReader(payloadBytes)
+
+		u := fmt.Sprintf("%v/api/v1/login", site)
+		req, err := http.NewRequest("POST", u, body)
+		if err != nil {
+			log.Fatal(err)
+		}
+		req.Header.Set("Content-Type", "application/json")
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer resp.Body.Close()
+
+		buf, err := ioutil.ReadAll(resp.Body)
+
+		if err != nil {
+			log.Fatal(err)
+		}
+		j := make(map[string]interface{})
+
+		json.Unmarshal(buf, &j)
+
+		if len(j) == 0 {
+			log.Fatal("login failed for ", site)
+
+		}
+		token = fmt.Sprintf("Bearer %v", j["accessToken"])
+
+	}()
+
+	host, err := os.Hostname()
+	if err != nil {
+		log.Fatal(err)
+	}
+	sys := map[string]interface{}{
+		"clientVersion": "1.2.0",
+		"name":          "migration",
+		"hostname":      host,
+		"mac":           mac,
+	}
+	payloadBytes, err := json.Marshal(sys)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	body := bytes.NewReader(payloadBytes)
+
+	u := fmt.Sprintf("%v/api/v1/system", srcURL)
+	req, err := http.NewRequest("POST", u, body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Add("Authorization", token)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	log.Println(resp.StatusCode)
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.Println(string(b))
+
+	sysRegistered = true
+	return getToken(site, user, pass)
+
+}
+
+func getToken(site string, user string, pass string) string {
+	// function used by bashhub to identify system
+	cmd := exec.Command("python", "-c", "import uuid; print(str(uuid.getnode()))")
+	m, err := cmd.Output()
+	if err != nil {
+		log.Fatal(err)
+	}
+	mac := strings.ReplaceAll(string(m), "\n", ``)
+	auth := map[string]interface{}{
+		"username": user,
+		"password": pass,
+		"mac":      mac,
+	}
+
+	payloadBytes, err := json.Marshal(auth)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	body := bytes.NewReader(payloadBytes)
+
+	u := fmt.Sprintf("%v/api/v1/login", site)
+	req, err := http.NewRequest("POST", u, body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 409 && !sysRegistered {
+		//	register system
+		return sysRegister(mac, site, user, pass)
+	}
+	buf, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	j := make(map[string]interface{})
+
+	json.Unmarshal(buf, &j)
+
+	if len(j) == 0 {
+		log.Fatal("login failed for ", site)
+
+	}
+	return fmt.Sprintf("Bearer %v", j["accessToken"])
+}
+
+func getCommandList() commandsList {
+	u := strings.TrimSpace(srcURL) + "/api/v1/command/search?unique=true&limit=1000000"
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	req.Header.Add("Authorization", srcToken)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+
+	if err != nil {
+		log.Println("Error on response.\n", err)
+	}
+
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		log.Fatalf("failed to get command list from %v, go status code %v", srcURL, resp.StatusCode)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	var result commandsList
+	json.Unmarshal(body, &result)
+
+	return result
+}
+
+func commandLookup(uuid string) {
+	u := strings.TrimSpace(srcURL) + "/api/v1/command/" + strings.TrimSpace(uuid)
+	req, err := http.NewRequest("GET", u, nil)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	req.Header.Add("Authorization", srcToken)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+
+	if err != nil {
+		log.Println("Error on response.\n", err)
+	}
+
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		log.Fatalf("failed command lookup from %v, go status code %v", srcURL, resp.StatusCode)
+	}
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	srcSend(body)
+}
+
+func srcSend(data []byte) {
+	defer func() {
+		if !progress {
+			bar.Add(1)
+		}
+	}()
+	body := bytes.NewReader(data)
+
+	u := dstURL + "/api/v1/import"
+	req, err := http.NewRequest("POST", u, body)
+	if err != nil {
+		log.Fatal(err)
+	}
+	req.Header.Add("Authorization", dstToken)
+	client := &http.Client{}
+	resp, err := client.Do(req)
+
+	if err != nil {
+		log.Println("Error on response.\n", err)
+	}
+
+	defer resp.Body.Close()
+}

go.mod

@@ -2,6 +2,7 @@ module github.com/nicksherron/bashhub-server
 
 require (
 	github.com/appleboy/gin-jwt/v2 v2.6.3
+	github.com/cheggaaa/pb/v3 v3.0.4
 	github.com/fatih/color v1.9.0
 	github.com/gin-gonic/gin v1.5.0
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect

go.sum

@@ -1,7 +1,11 @@
+github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
 github.com/appleboy/gin-jwt/v2 v2.6.3 h1:aK4E3DjihWEBUTjEeRnGkA5nUkmwJPL1CPonMa2usRs=
 github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0=
 github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4=
 github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw=
+github.com/cheggaaa/pb/v3 v3.0.4 h1:QZEPYOj2ix6d5oEg63fbHmpolrnNiwjUsk+h74Yt4bM=
+github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -11,6 +15,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
 github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
@@ -48,14 +53,18 @@ github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdA
 github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
+github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
 github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
@@ -99,8 +108,11 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
 golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU=
+golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

internal/db.go

@@ -97,11 +97,13 @@ func dbInit() {
 	gormdb.AutoMigrate(&Config{})
 
 	//TODO: ensure these are the most efficient indexes
-	gormdb.Model(&User{}).AddIndex("idx_user", "username")
+	gormdb.Model(&User{}).AddUniqueIndex("idx_user", "username")
 	gormdb.Model(&System{}).AddIndex("idx_mac", "mac")
 	gormdb.Model(&Command{}).AddIndex("idx_user_command_created", "user_id, created, command")
 	gormdb.Model(&Command{}).AddIndex("idx_user_uuid", "user_id, uuid")
 	gormdb.Model(&Config{}).AddUniqueIndex("idx_config_id", "id")
+	gormdb.Model(&Command{}).AddUniqueIndex("idx_uuid", "uuid")
+
 
 	// Just need gorm for migration and index creation.
 	gormdb.Close()
@@ -545,5 +547,16 @@ func (status Status) statusGet() (Status, error) {
 		return Status{}, err
 	}
 	return status, err
-
+}
+
+func importCommands(q Query) {
+	_, err := db.Exec(`INSERT INTO commands 
+							("command", "path", "created", "uuid", "exit_status",
+							 "system_name", "session_id", "user_id" )
+							 VALUES ($1,$2,$3,$4,$5,$6,$7,(select "id" from users where "username" = $8)) ON CONFLICT do nothing`,
+		q.Command, q.Path, q.Created, q.Uuid, q.ExitStatus,
+		q.SystemName, q.SessionID, q.Username)
+	if err != nil {
+		log.Println(err)
+	}
 }

internal/server.go

@@ -42,15 +42,14 @@ type User struct {
 }
 
 type Query struct {
-	Uuid       string `json:"uuid"`
 	Command    string `json:"command"`
-	Created    int64  `json:"created"`
 	Path       string `json:"path"`
+	Created    int64  `json:"created"`
+	Uuid       string `json:"uuid"`
 	ExitStatus int    `json:"exitStatus"`
 	Username   string `json:"username"`
 	SystemName string `gorm:"-"  json:"systemName"`
-	//TODO: implement sessions
+	SessionID  string `json:"sessionId"`
-	SessionID string `json:"session_id"`
 }
 
 type Command struct {
@@ -67,6 +66,7 @@ type Command struct {
 	Limit            int    `gorm:"-"`
 	Unique           bool   `gorm:"-"`
 	Query            string `gorm:"-"`
+	SessionID        string `json:"sessionId"`
 }
 
 type System struct {
@@ -371,7 +371,17 @@ func Run() {
 		}
 
 		c.IndentedJSON(http.StatusOK, result)
+	})
 
+	r.POST("/api/v1/import", func(c *gin.Context) {
+		var query Query
+		if err := c.ShouldBindJSON(&query); err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+			return
+		}
+		claims := jwt.ExtractClaims(c)
+		query.Username = claims["username"].(string)
+		importCommands(query)
 	})
 
 	Addr = strings.ReplaceAll(Addr, "http://", "")