diff --git a/strata/cmd/mongo/lreplica_drivers/lrazureblobdriver/lrazureblobdriver.go b/strata/cmd/mongo/lreplica_drivers/lrazureblobdriver/lrazureblobdriver.go index c55a5d9..13b367c 100644 --- a/strata/cmd/mongo/lreplica_drivers/lrazureblobdriver/lrazureblobdriver.go +++ b/strata/cmd/mongo/lreplica_drivers/lrazureblobdriver/lrazureblobdriver.go @@ -7,26 +7,20 @@ import ( "github.com/facebookgo/rocks-strata/strata" "github.com/facebookgo/rocks-strata/strata/azureblobstorage" + "github.com/facebookgo/rocks-strata/strata/cmd/mongo/lreplica_drivers/lrs3driver" "github.com/facebookgo/rocks-strata/strata/mongo/lreplica" ) +// AzureBlobOptions define basic options of your azure blob storage type AzureBlobOptions struct { Container string `short:"C" long:"container" description:"Azure Blob Storage container name" required:"true"` BlobPrefix string `short:"p" long:"blob-prefix" description:"Prefix used when storing and retrieving files. Optional" optional:"true"` } -// ReplicaOptions are used for commands like backup and restore -type ReplicaOptions struct { - MaxBackgroundCopies int `long:"max-background-copies" default:"16" description:"Backup and restore actions will use up to this many goroutines to copy files"` - Port int `long:"port" default:"27017" description:"Backup should look for a mongod instance that is listening on this port"` - Username string `long:"username" description:"If auth is configured, specify the username with admin privileges here"` - Password string `long:"password" description:"Password for the specified user."` -} - // Options define the common options needed by this strata command type Options struct { - AzureBlobOptions AzureBlobOptions `group:"Azure Blob Options"` - Replica ReplicaOptions `group:"Replica Options"` + AzureBlobOptions AzureBlobOptions `group:"Azure Blob Options"` + Replica lrs3driver.ReplicaOptions `group:"Replica Options"` } // DriverFactory implements strata.DriverFactory @@ -60,9 +54,11 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) { replica, err := lreplica.NewLocalReplica( options.Replica.MaxBackgroundCopies, + options.Replica.LocalHostname, strconv.Itoa(options.Replica.Port), options.Replica.Username, options.Replica.Password, + options.Replica.SslAllowInvalidCertificates, ) if err != nil { return nil, err diff --git a/strata/cmd/mongo/lreplica_drivers/lrldriver/lrldriver.go b/strata/cmd/mongo/lreplica_drivers/lrldriver/lrldriver.go index 5797943..2bd063e 100644 --- a/strata/cmd/mongo/lreplica_drivers/lrldriver/lrldriver.go +++ b/strata/cmd/mongo/lreplica_drivers/lrldriver/lrldriver.go @@ -10,19 +10,19 @@ import ( "github.com/facebookgo/rocks-strata/strata" "github.com/facebookgo/rocks-strata/strata/cmd/mongo/lreplica_drivers/lrs3driver" - "github.com/facebookgo/rocks-strata/strata/mongo/lreplica" "github.com/facebookgo/rocks-strata/strata/lstorage" + "github.com/facebookgo/rocks-strata/strata/mongo/lreplica" ) // FsOptions are common to all commands type FsOptions struct { - Mountpoint string `short:"m" long:"mpoint" description:"Mount point name, such as \"~/sbackup\"" default:"~/sbackup"` + Mountpoint string `short:"m" long:"mpoint" description:"Mount point name, such as \"~/sbackup\"" default:"~/sbackup"` } // Options define the common options needed by this strata command type Options struct { - LocalStorage FsOptions `group:"Storage Options"` - Replica lrs3driver.ReplicaOptions `group:"Replica Options"` + LocalStorage FsOptions `group:"Storage Options"` + Replica lrs3driver.ReplicaOptions `group:"Replica Options"` } // DriverFactory implements strata.DriverFactory @@ -46,9 +46,11 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) { replica, err := lreplica.NewLocalReplica( options.Replica.MaxBackgroundCopies, + options.Replica.LocalHostname, strconv.Itoa(options.Replica.Port), options.Replica.Username, options.Replica.Password, + options.Replica.SslAllowInvalidCertificates, ) if err != nil { return nil, err diff --git a/strata/cmd/mongo/lreplica_drivers/lrminiodriver/lrminiodriver.go b/strata/cmd/mongo/lreplica_drivers/lrminiodriver/lrminiodriver.go index f73c0e8..9aa5620 100644 --- a/strata/cmd/mongo/lreplica_drivers/lrminiodriver/lrminiodriver.go +++ b/strata/cmd/mongo/lreplica_drivers/lrminiodriver/lrminiodriver.go @@ -42,6 +42,7 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) { secure := os.Getenv("MINIO_SECURE") accessKey := os.Getenv("MINIO_ACCESS_KEY_ID") secretKey := os.Getenv("MINIO_SECRET_ACCESS_KEY") + allowInsecureHTTPS := os.Getenv("MINIO_ALLOW_INSECURE_HTTPS") if endPoint == "" || accessKey == "" || secretKey == "" { return nil, errors.New("Environment variables MINIO_ENDPOINT, MINIO_ACCESS_KEY_ID and MINIO_SECRET_ACCESS_KEY must be set") } @@ -55,13 +56,23 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) { return nil, errors.New("Valid values for environment variable MINIO_SECURE are 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False") } + if allowInsecureHTTPS == "" { + allowInsecureHTTPS = "false" + } + + allowInsecureHTTPSBool, err := strconv.ParseBool(allowInsecureHTTPS) + if err != nil { + return nil, errors.New("Valid values for environment variable MINIO_ALLOW_INSECURE_HTTPS are 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False") + } + minio, err := miniostorage.NewMinioStorage( endPoint, accessKey, secretKey, options.Minio.BucketName, options.Minio.BucketPrefix, options.Minio.Region, - secureBool) + secureBool, + allowInsecureHTTPSBool) if err != nil { return nil, err @@ -69,9 +80,11 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) { replica, err := lreplica.NewLocalReplica( options.Replica.MaxBackgroundCopies, + options.Replica.LocalHostname, strconv.Itoa(options.Replica.Port), options.Replica.Username, options.Replica.Password, + options.Replica.SslAllowInvalidCertificates, ) if err != nil { diff --git a/strata/cmd/mongo/lreplica_drivers/lrs3driver/lrs3driver.go b/strata/cmd/mongo/lreplica_drivers/lrs3driver/lrs3driver.go index 9bd393f..f5fa5f1 100644 --- a/strata/cmd/mongo/lreplica_drivers/lrs3driver/lrs3driver.go +++ b/strata/cmd/mongo/lreplica_drivers/lrs3driver/lrs3driver.go @@ -27,10 +27,12 @@ type AWSOptions struct { // ReplicaOptions are used for commands like backup and restore type ReplicaOptions struct { - MaxBackgroundCopies int `long:"max-background-copies" default:"16" description:"Backup and restore actions will use up to this many goroutines to copy files"` - Port int `long:"port" default:"27017" description:"Backup should look for a mongod instance that is listening on this port"` - Username string `long:"username" description:"If auth is configured, specify the username with admin privileges here"` - Password string `long:"password" description:"Password for the specified user."` + LocalHostname string `long:"local-hostname" default:"localhost" description:"'localhost' or a hostname that is accessible on the local machine via e.g. kubernetes network"` + MaxBackgroundCopies int `long:"max-background-copies" default:"16" description:"Backup and restore actions will use up to this many goroutines to copy files"` + Port int `long:"port" default:"27017" description:"Backup should look for a mongod instance that is listening on this port"` + Username string `long:"username" description:"If auth is configured, specify the username with admin privileges here"` + Password string `long:"password" description:"Password for the specified user."` + SslAllowInvalidCertificates bool `long:"sslAllowInvalidCertificates" description:"Allows to connect to a insecure mongo instance"` } // Options define the common options needed by this strata command @@ -70,9 +72,11 @@ func (factory DriverFactory) Driver() (*strata.Driver, error) { } replica, err := lreplica.NewLocalReplica( options.Replica.MaxBackgroundCopies, + options.Replica.LocalHostname, strconv.Itoa(options.Replica.Port), options.Replica.Username, options.Replica.Password, + options.Replica.SslAllowInvalidCertificates, ) if err != nil { return nil, err diff --git a/strata/miniostorage/storage.go b/strata/miniostorage/storage.go index a2af613..2685312 100644 --- a/strata/miniostorage/storage.go +++ b/strata/miniostorage/storage.go @@ -2,8 +2,10 @@ package miniostorage import ( "bytes" + "crypto/tls" "io" "io/ioutil" + "net/http" minio "github.com/minio/minio-go" ) @@ -25,14 +27,19 @@ func (m *MinioStorage) removePrefix(name string) string { } // NewMinioStorage initializes the MinioStorage with Minio arguments -func NewMinioStorage(endPoint, accessKeyID, secretAccessKey, bucket, prefix, region string, secure bool) (*MinioStorage, error) { - +func NewMinioStorage(endPoint, accessKeyID, secretAccessKey, bucket, prefix, region string, secure bool, allowInsecureHTTPS bool) (*MinioStorage, error) { mc, err := minio.New(endPoint, accessKeyID, secretAccessKey, secure) - if err != nil { return nil, err } + if allowInsecureHTTPS { + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + } + mc.SetCustomTransport(tr) + } + if region == "" { region = "us-east-1" } diff --git a/strata/mongo/lreplica/mock_replica.go b/strata/mongo/lreplica/mock_replica.go index b6b78a4..c4dfa54 100644 --- a/strata/mongo/lreplica/mock_replica.go +++ b/strata/mongo/lreplica/mock_replica.go @@ -20,7 +20,7 @@ type mockLocalSessionGetter struct { mongo *mgotest.Server } -func (mlsg *mockLocalSessionGetter) get(string, string, string) (*mgo.Session, error) { +func (mlsg *mockLocalSessionGetter) get(bool, string, string, string, string) (*mgo.Session, error) { return mlsg.mongo.Session(), nil } diff --git a/strata/mongo/lreplica/replica.go b/strata/mongo/lreplica/replica.go index 46a5659..7909aab 100644 --- a/strata/mongo/lreplica/replica.go +++ b/strata/mongo/lreplica/replica.go @@ -6,10 +6,13 @@ package lreplica import ( + "crypto/tls" "errors" "fmt" "io" "io/ioutil" + "log" + "net" "os" "strings" "syscall" @@ -22,17 +25,38 @@ import ( ) type sessionGetter interface { - get(port, username, password string) (*mgo.Session, error) + get(sslAllowInvalidCertificates bool, localHostname, port, username, password string) (*mgo.Session, error) } type localSessionGetter struct{} // port could be the empty string -func (l *localSessionGetter) get(port, username, password string) (*mgo.Session, error) { - addr := "localhost" +func (l *localSessionGetter) get(sslAllowInvalidCertificates bool, localHostname, port, username, password string) (*mgo.Session, error) { + addr := localHostname if port != "" { addr += ":" + port } + + if sslAllowInvalidCertificates { + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + } + + return mgo.DialWithInfo(&mgo.DialInfo{ + Direct: true, + Addrs: []string{addr}, + Timeout: 5 * time.Minute, + Username: username, + Password: password, + DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) { + conn, err := tls.Dial("tcp", addr.String(), tlsConfig) + if err != nil { + log.Println(err) + } + return conn, err + }}) + } + return mgo.DialWithInfo(&mgo.DialInfo{ Direct: true, Addrs: []string{addr}, @@ -44,21 +68,25 @@ func (l *localSessionGetter) get(port, username, password string) (*mgo.Session, // LocalReplica is a replica where all methods that take a ReplicaID must be // run on the host corresponding to ReplicaID type LocalReplica struct { - port string - username string - password string - sessionGetter sessionGetter - maxBackgroundCopies int + localHostname string + port string + username string + password string + sslAllowInvalidCertificates bool + sessionGetter sessionGetter + maxBackgroundCopies int } // NewLocalReplica constructs a LocalReplica -func NewLocalReplica(maxBackgroundCopies int, port, username, password string) (*LocalReplica, error) { +func NewLocalReplica(maxBackgroundCopies int, localHostname, port, username, password string, sslAllowInvalidCertificates bool) (*LocalReplica, error) { return &LocalReplica{ - sessionGetter: &localSessionGetter{}, - maxBackgroundCopies: maxBackgroundCopies, - port: port, - username: username, - password: password, + sessionGetter: &localSessionGetter{}, + maxBackgroundCopies: maxBackgroundCopies, + localHostname: localHostname, + port: port, + username: username, + password: password, + sslAllowInvalidCertificates: sslAllowInvalidCertificates, }, nil } @@ -170,7 +198,7 @@ func nestedBsonMapGet(m bson.M, arg string, moreArgs ...string) (interface{}, er // TODO(agf): Have a way to pass in tags func (r *LocalReplica) CreateSnapshot(replicaID, snapshotID string) (*strata.Snapshot, error) { strata.Log("Getting session for CreateSnapshot()") - session, err := r.sessionGetter.get(r.port, r.username, r.password) + session, err := r.sessionGetter.get(r.sslAllowInvalidCertificates, r.localHostname, r.port, r.username, r.password) if err != nil { return nil, err }