2017-03-06 23:18:59 +03:00
|
|
|
package endpoint
|
|
|
|
|
|
|
|
import (
|
2021-02-15 20:04:37 +03:00
|
|
|
"crypto/tls"
|
|
|
|
"crypto/x509"
|
2017-03-06 23:18:59 +03:00
|
|
|
"errors"
|
|
|
|
"fmt"
|
2021-02-15 20:04:37 +03:00
|
|
|
"io/ioutil"
|
2021-07-08 11:20:45 +03:00
|
|
|
"os"
|
2017-03-06 23:18:59 +03:00
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
2021-02-15 20:04:37 +03:00
|
|
|
lg "log"
|
|
|
|
|
2017-03-06 23:18:59 +03:00
|
|
|
"github.com/Shopify/sarama"
|
2019-10-30 20:17:59 +03:00
|
|
|
"github.com/tidwall/gjson"
|
2021-02-15 20:04:37 +03:00
|
|
|
"github.com/tidwall/tile38/internal/log"
|
2017-03-06 23:18:59 +03:00
|
|
|
)
|
|
|
|
|
2019-10-30 20:17:59 +03:00
|
|
|
const kafkaExpiresAfter = time.Second * 30
|
2017-03-06 23:18:59 +03:00
|
|
|
|
2018-04-19 19:25:39 +03:00
|
|
|
// KafkaConn is an endpoint connection
|
|
|
|
type KafkaConn struct {
|
2017-03-06 23:18:59 +03:00
|
|
|
mu sync.Mutex
|
|
|
|
ep Endpoint
|
|
|
|
conn sarama.SyncProducer
|
|
|
|
ex bool
|
|
|
|
t time.Time
|
|
|
|
}
|
|
|
|
|
2018-04-19 19:25:39 +03:00
|
|
|
// Expired returns true if the connection has expired
|
|
|
|
func (conn *KafkaConn) Expired() bool {
|
2017-03-06 23:18:59 +03:00
|
|
|
conn.mu.Lock()
|
|
|
|
defer conn.mu.Unlock()
|
|
|
|
if !conn.ex {
|
2021-03-31 18:13:44 +03:00
|
|
|
if time.Since(conn.t) > kafkaExpiresAfter {
|
2017-03-06 23:18:59 +03:00
|
|
|
if conn.conn != nil {
|
|
|
|
conn.close()
|
|
|
|
}
|
|
|
|
conn.ex = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return conn.ex
|
|
|
|
}
|
|
|
|
|
2018-04-19 19:25:39 +03:00
|
|
|
func (conn *KafkaConn) close() {
|
2017-03-06 23:18:59 +03:00
|
|
|
if conn.conn != nil {
|
|
|
|
conn.conn.Close()
|
|
|
|
conn.conn = nil
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-04-19 19:25:39 +03:00
|
|
|
// Send sends a message
|
|
|
|
func (conn *KafkaConn) Send(msg string) error {
|
2017-03-06 23:18:59 +03:00
|
|
|
conn.mu.Lock()
|
|
|
|
defer conn.mu.Unlock()
|
|
|
|
|
|
|
|
if conn.ex {
|
|
|
|
return errExpired
|
|
|
|
}
|
|
|
|
conn.t = time.Now()
|
|
|
|
|
2021-02-15 20:04:37 +03:00
|
|
|
if log.Level > 2 {
|
2021-02-19 01:34:01 +03:00
|
|
|
sarama.Logger = lg.New(log.Output(), "[sarama] ", 0)
|
2021-02-15 20:04:37 +03:00
|
|
|
}
|
|
|
|
|
2017-03-06 23:18:59 +03:00
|
|
|
uri := fmt.Sprintf("%s:%d", conn.ep.Kafka.Host, conn.ep.Kafka.Port)
|
|
|
|
if conn.conn == nil {
|
2018-04-17 18:30:49 +03:00
|
|
|
cfg := sarama.NewConfig()
|
2021-02-15 20:04:37 +03:00
|
|
|
|
|
|
|
if conn.ep.Kafka.TLS {
|
|
|
|
log.Debugf("building kafka tls config")
|
|
|
|
tlsConfig, err := newKafkaTLSConfig(conn.ep.Kafka.CertFile, conn.ep.Kafka.KeyFile, conn.ep.Kafka.CACertFile)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
cfg.Net.TLS.Enable = true
|
|
|
|
cfg.Net.TLS.Config = tlsConfig
|
2021-07-08 13:05:16 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
if conn.ep.Kafka.SASL {
|
2021-07-08 11:20:45 +03:00
|
|
|
log.Debugf("building kafka sasl config")
|
|
|
|
cfg.ClientID = "sasl_scram_client"
|
|
|
|
cfg.Net.SASL.Enable = true
|
|
|
|
cfg.Net.SASL.User = os.Getenv("KAFKA_USERNAME")
|
|
|
|
cfg.Net.SASL.Password = os.Getenv("KAFKA_PASSWORD")
|
|
|
|
cfg.Net.SASL.Handshake = true
|
|
|
|
cfg.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: SHA512} }
|
|
|
|
cfg.Net.SASL.Mechanism = sarama.SASLTypeSCRAMSHA512
|
2021-02-15 20:04:37 +03:00
|
|
|
}
|
|
|
|
|
2018-04-17 18:30:49 +03:00
|
|
|
cfg.Net.DialTimeout = time.Second
|
|
|
|
cfg.Net.ReadTimeout = time.Second * 5
|
|
|
|
cfg.Net.WriteTimeout = time.Second * 5
|
2018-07-03 15:27:01 +03:00
|
|
|
// Fix #333 : fix backward incompatibility introduced by sarama library
|
|
|
|
cfg.Producer.Return.Successes = true
|
2020-07-03 21:57:15 +03:00
|
|
|
cfg.Version = sarama.V0_10_0_0
|
2018-07-03 15:27:01 +03:00
|
|
|
|
2018-04-17 18:30:49 +03:00
|
|
|
c, err := sarama.NewSyncProducer([]string{uri}, cfg)
|
2017-03-06 23:18:59 +03:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
conn.conn = c
|
|
|
|
}
|
|
|
|
|
2018-11-15 13:59:08 +03:00
|
|
|
// parse json again to get out info for our kafka key
|
|
|
|
key := gjson.Get(msg, "key")
|
|
|
|
id := gjson.Get(msg, "id")
|
|
|
|
keyValue := fmt.Sprintf("%s-%s", key.String(), id.String())
|
|
|
|
|
2017-03-06 23:18:59 +03:00
|
|
|
message := &sarama.ProducerMessage{
|
2018-11-15 13:59:08 +03:00
|
|
|
Topic: conn.ep.Kafka.TopicName,
|
|
|
|
Key: sarama.StringEncoder(keyValue),
|
2017-03-06 23:18:59 +03:00
|
|
|
Value: sarama.StringEncoder(msg),
|
|
|
|
}
|
|
|
|
|
|
|
|
_, offset, err := conn.conn.SendMessage(message)
|
|
|
|
if err != nil {
|
|
|
|
conn.close()
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if offset < 0 {
|
|
|
|
conn.close()
|
|
|
|
return errors.New("invalid kafka reply")
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2018-04-19 19:25:39 +03:00
|
|
|
func newKafkaConn(ep Endpoint) *KafkaConn {
|
|
|
|
return &KafkaConn{
|
2017-03-06 23:18:59 +03:00
|
|
|
ep: ep,
|
|
|
|
t: time.Now(),
|
|
|
|
}
|
|
|
|
}
|
2021-02-15 20:04:37 +03:00
|
|
|
|
|
|
|
func newKafkaTLSConfig(CertFile, KeyFile, CACertFile string) (*tls.Config, error) {
|
|
|
|
tlsConfig := tls.Config{}
|
|
|
|
|
|
|
|
// Load client cert
|
|
|
|
cert, err := tls.LoadX509KeyPair(CertFile, KeyFile)
|
|
|
|
if err != nil {
|
|
|
|
return &tlsConfig, err
|
|
|
|
}
|
|
|
|
tlsConfig.Certificates = []tls.Certificate{cert}
|
|
|
|
|
|
|
|
// Load CA cert
|
|
|
|
caCert, err := ioutil.ReadFile(CACertFile)
|
|
|
|
if err != nil {
|
|
|
|
return &tlsConfig, err
|
|
|
|
}
|
|
|
|
caCertPool := x509.NewCertPool()
|
|
|
|
caCertPool.AppendCertsFromPEM(caCert)
|
|
|
|
tlsConfig.RootCAs = caCertPool
|
|
|
|
|
|
|
|
return &tlsConfig, err
|
|
|
|
}
|