merge feature-bhdownload

This commit is contained in:
nicksherron 2020-02-11 19:39:33 -05:00
commit 9f5b50eb87
6 changed files with 372 additions and 12 deletions

View file

@ -34,13 +34,10 @@ var cfgFile string
// rootCmd represents the base command when called without any subcommands
var (
noWarning bool
rootCmd = &cobra.Command{
rootCmd = &cobra.Command{
Run: func(cmd *cobra.Command, args []string) {
cmd.Flags().Parse(args)
if !noWarning {
checkBhEnv()
}
checkBhEnv()
startupMessage()
internal.Run()
},
@ -60,7 +57,6 @@ func init() {
rootCmd.PersistentFlags().StringVar(&internal.LogFile, "log", "", `Set filepath for HTTP log. "" logs to stderr`)
rootCmd.PersistentFlags().StringVar(&internal.DbPath, "db", dbPath(), "DB location (sqlite or postgres)")
rootCmd.PersistentFlags().StringVarP(&internal.Addr, "addr", "a", listenAddr(), "Ip and port to listen and serve on")
rootCmd.PersistentFlags().BoolVarP(&noWarning, "warning-disable", "w", false, `Disable BH_URL env variable startup warning`)
}

328
cmd/transfer.go Normal file
View file

@ -0,0 +1,328 @@
/*
*
* Copyright © 2020 nicksherron <nsherron90@gmail.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package cmd
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"os/exec"
"strings"
"sync"
"github.com/cheggaaa/pb/v3"
"github.com/spf13/cobra"
)
type cList struct {
UUID string `json:"uuid"`
Command string `json:"command"`
Created int64 `json:"created"`
}
type commandsList []cList
var (
barTemplate = `{{string . "message"}}{{counters . }} {{bar . }} {{percent . }} {{speed . "%s inserts/sec" }}`
bar *pb.ProgressBar
progress bool
srcUser string
dstUser string
srcURL string
dstURL string
srcPass string
dstPass string
srcToken string
dstToken string
sysRegistered bool
workers int
wg sync.WaitGroup
cmdList commandsList
transferCmd = &cobra.Command{
Use: "transfer",
Short: "transfer bashhub history ",
Run: func(cmd *cobra.Command, args []string) {
cmd.Flags().Parse(args)
sysRegistered = false
srcToken = getToken(srcURL, srcUser, srcPass)
sysRegistered = false
dstToken = getToken(dstURL, dstUser, dstPass)
cmdList = getCommandList()
counter := 0
if !progress {
bar = pb.ProgressBarTemplate(barTemplate).Start(len(cmdList)).SetMaxWidth(70)
bar.Set("message", "inserting records \t")
}
for _, v := range cmdList {
wg.Add(1)
counter++
go func(c cList) {
defer wg.Done()
commandLookup(c.UUID)
}(v)
if counter > workers {
wg.Wait()
counter = 0
}
}
wg.Wait()
},
}
)
func init() {
rootCmd.AddCommand(transferCmd)
transferCmd.PersistentFlags().StringVar(&srcURL, "src-url", "https://bashhub.com", "source url")
transferCmd.PersistentFlags().StringVar(&srcUser, "src-user", "", "source username")
transferCmd.PersistentFlags().StringVar(&srcPass, "src-pass", "", "source password")
transferCmd.PersistentFlags().StringVar(&dstURL, "dst-url", "http://localhost:8080", "destination url")
transferCmd.PersistentFlags().StringVar(&dstUser, "dst-user", "", "destination username")
transferCmd.PersistentFlags().StringVar(&dstPass, "dst-pass", "", "destination password")
transferCmd.PersistentFlags().BoolVarP(&progress, "progress", "p", false, "show progress bar")
transferCmd.PersistentFlags().IntVarP(&workers, "workers", "w", 10, "max number of concurrent requests")
}
func sysRegister(mac string, site string, user string, pass string) string {
var token string
func() {
var null *string
auth := map[string]interface{}{
"username": user,
"password": pass,
"mac": null,
}
payloadBytes, err := json.Marshal(auth)
if err != nil {
log.Fatal(err)
}
body := bytes.NewReader(payloadBytes)
u := fmt.Sprintf("%v/api/v1/login", site)
req, err := http.NewRequest("POST", u, body)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
j := make(map[string]interface{})
json.Unmarshal(buf, &j)
if len(j) == 0 {
log.Fatal("login failed for ", site)
}
token = fmt.Sprintf("Bearer %v", j["accessToken"])
}()
host, err := os.Hostname()
if err != nil {
log.Fatal(err)
}
sys := map[string]interface{}{
"clientVersion": "1.2.0",
"name": "migration",
"hostname": host,
"mac": mac,
}
payloadBytes, err := json.Marshal(sys)
if err != nil {
log.Fatal(err)
}
body := bytes.NewReader(payloadBytes)
u := fmt.Sprintf("%v/api/v1/system", srcURL)
req, err := http.NewRequest("POST", u, body)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Add("Authorization", token)
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
log.Println(resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
log.Println(string(b))
sysRegistered = true
return getToken(site, user, pass)
}
func getToken(site string, user string, pass string) string {
// function used by bashhub to identify system
cmd := exec.Command("python", "-c", "import uuid; print(str(uuid.getnode()))")
m, err := cmd.Output()
if err != nil {
log.Fatal(err)
}
mac := strings.ReplaceAll(string(m), "\n", ``)
auth := map[string]interface{}{
"username": user,
"password": pass,
"mac": mac,
}
payloadBytes, err := json.Marshal(auth)
if err != nil {
log.Fatal(err)
}
body := bytes.NewReader(payloadBytes)
u := fmt.Sprintf("%v/api/v1/login", site)
req, err := http.NewRequest("POST", u, body)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode == 409 && !sysRegistered {
// register system
return sysRegister(mac, site, user, pass)
}
buf, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
j := make(map[string]interface{})
json.Unmarshal(buf, &j)
if len(j) == 0 {
log.Fatal("login failed for ", site)
}
return fmt.Sprintf("Bearer %v", j["accessToken"])
}
func getCommandList() commandsList {
u := strings.TrimSpace(srcURL) + "/api/v1/command/search?unique=true&limit=1000000"
req, err := http.NewRequest("GET", u, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Add("Authorization", srcToken)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Println("Error on response.\n", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
log.Fatalf("failed to get command list from %v, go status code %v", srcURL, resp.StatusCode)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
var result commandsList
json.Unmarshal(body, &result)
return result
}
func commandLookup(uuid string) {
u := strings.TrimSpace(srcURL) + "/api/v1/command/" + strings.TrimSpace(uuid)
req, err := http.NewRequest("GET", u, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Add("Authorization", srcToken)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Println("Error on response.\n", err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
log.Fatalf("failed command lookup from %v, go status code %v", srcURL, resp.StatusCode)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
srcSend(body)
}
func srcSend(data []byte) {
defer func() {
if !progress {
bar.Add(1)
}
}()
body := bytes.NewReader(data)
u := dstURL + "/api/v1/import"
req, err := http.NewRequest("POST", u, body)
if err != nil {
log.Fatal(err)
}
req.Header.Add("Authorization", dstToken)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Println("Error on response.\n", err)
}
defer resp.Body.Close()
}

1
go.mod
View file

@ -2,6 +2,7 @@ module github.com/nicksherron/bashhub-server
require (
github.com/appleboy/gin-jwt/v2 v2.6.3
github.com/cheggaaa/pb/v3 v3.0.4
github.com/fatih/color v1.9.0
github.com/gin-gonic/gin v1.5.0
github.com/inconshreveable/mousetrap v1.0.0 // indirect

12
go.sum
View file

@ -1,7 +1,11 @@
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/appleboy/gin-jwt/v2 v2.6.3 h1:aK4E3DjihWEBUTjEeRnGkA5nUkmwJPL1CPonMa2usRs=
github.com/appleboy/gin-jwt/v2 v2.6.3/go.mod h1:MfPYA4ogzvOcVkRwAxT7quHOtQmVKDpTwxyUrC2DNw0=
github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4=
github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw=
github.com/cheggaaa/pb/v3 v3.0.4 h1:QZEPYOj2ix6d5oEg63fbHmpolrnNiwjUsk+h74Yt4bM=
github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -11,6 +15,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumC
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
@ -48,14 +53,18 @@ github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdA
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM=
github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v2.0.1+incompatible h1:xQ15muvnzGBHpIpdrNi1DA5x0+TcBZzsIDwmw9uTHzw=
github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
@ -99,8 +108,11 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ=
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 h1:ZBzSG/7F4eNKz2L3GE9o300RX0Az1Bw5HF7PDraD+qU=
golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

View file

@ -97,11 +97,13 @@ func dbInit() {
gormdb.AutoMigrate(&Config{})
//TODO: ensure these are the most efficient indexes
gormdb.Model(&User{}).AddIndex("idx_user", "username")
gormdb.Model(&User{}).AddUniqueIndex("idx_user", "username")
gormdb.Model(&System{}).AddIndex("idx_mac", "mac")
gormdb.Model(&Command{}).AddIndex("idx_user_command_created", "user_id, created, command")
gormdb.Model(&Command{}).AddIndex("idx_user_uuid", "user_id, uuid")
gormdb.Model(&Config{}).AddUniqueIndex("idx_config_id", "id")
gormdb.Model(&Command{}).AddUniqueIndex("idx_uuid", "uuid")
// Just need gorm for migration and index creation.
gormdb.Close()
@ -545,5 +547,16 @@ func (status Status) statusGet() (Status, error) {
return Status{}, err
}
return status, err
}
func importCommands(q Query) {
_, err := db.Exec(`INSERT INTO commands
("command", "path", "created", "uuid", "exit_status",
"system_name", "session_id", "user_id" )
VALUES ($1,$2,$3,$4,$5,$6,$7,(select "id" from users where "username" = $8)) ON CONFLICT do nothing`,
q.Command, q.Path, q.Created, q.Uuid, q.ExitStatus,
q.SystemName, q.SessionID, q.Username)
if err != nil {
log.Println(err)
}
}

View file

@ -42,15 +42,14 @@ type User struct {
}
type Query struct {
Uuid string `json:"uuid"`
Command string `json:"command"`
Created int64 `json:"created"`
Path string `json:"path"`
Created int64 `json:"created"`
Uuid string `json:"uuid"`
ExitStatus int `json:"exitStatus"`
Username string `json:"username"`
SystemName string `gorm:"-" json:"systemName"`
//TODO: implement sessions
SessionID string `json:"session_id"`
SessionID string `json:"sessionId"`
}
type Command struct {
@ -67,6 +66,7 @@ type Command struct {
Limit int `gorm:"-"`
Unique bool `gorm:"-"`
Query string `gorm:"-"`
SessionID string `json:"sessionId"`
}
type System struct {
@ -371,7 +371,17 @@ func Run() {
}
c.IndentedJSON(http.StatusOK, result)
})
r.POST("/api/v1/import", func(c *gin.Context) {
var query Query
if err := c.ShouldBindJSON(&query); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
}
claims := jwt.ExtractClaims(c)
query.Username = claims["username"].(string)
importCommands(query)
})
Addr = strings.ReplaceAll(Addr, "http://", "")