From cabf8b8e4739e576837111e156763d19a64a3591 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Wed, 21 Feb 2018 22:43:58 -0300 Subject: [PATCH] Use docker to build cassandra jar file --- cassandra/{image => }/Makefile | 19 +- cassandra/cassandra-statefulset.yaml | 4 +- cassandra/go/main.go | 82 ++++++ cassandra/image/files/build.sh | 4 + cassandra/image/files/cassandra-seed.h | 79 ++++++ .../image/files/kubernetes-cassandra.jar | Bin 9843 -> 7884 bytes cassandra/java/pom.xml | 12 +- .../java/io/k8s/cassandra/GoInterface.java | 54 ++++ .../k8s/cassandra/KubernetesSeedProvider.java | 267 ++++-------------- .../cassandra/KubernetesSeedProviderTest.java | 50 ++-- 10 files changed, 315 insertions(+), 256 deletions(-) rename cassandra/{image => }/Makefile (72%) create mode 100644 cassandra/go/main.go create mode 100644 cassandra/image/files/cassandra-seed.h create mode 100644 cassandra/java/src/main/java/io/k8s/cassandra/GoInterface.java diff --git a/cassandra/image/Makefile b/cassandra/Makefile similarity index 72% rename from cassandra/image/Makefile rename to cassandra/Makefile index 9d5774ee..98049622 100644 --- a/cassandra/image/Makefile +++ b/cassandra/Makefile @@ -13,24 +13,29 @@ # limitations under the License. # build the cassandra image. -VERSION=v13 +VERSION=v14 PROJECT_ID?=google_samples PROJECT=gcr.io/${PROJECT_ID} CASSANDRA_VERSION=3.11.2 all: kubernetes-cassandra.jar build -kubernetes-cassandra.jar: ../java/* ../java/src/main/java/io/k8s/cassandra/*.java - cd ../java && mvn clean && mvn package - mv ../java/target/kubernetes-cassandra*.jar files/kubernetes-cassandra.jar - cd ../java && mvn clean +build-go: + go build -a -installsuffix cgo \ + -ldflags "-s -w" \ + -o image/files/cassandra-seed.so -buildmode=c-shared go/main.go + +kubernetes-cassandra.jar: + @echo "Building kubernetes-cassandra.jar" + docker run -v ${PWD}/java:/usr/src/app maven:3-jdk-8-onbuild-alpine mvn clean install + cp java/target/kubernetes-cassandra*.jar image/files/kubernetes-cassandra.jar container: @echo "Building ${PROJECT}/cassandra:${VERSION}" - docker build --pull --build-arg "CASSANDRA_VERSION=${CASSANDRA_VERSION}" -t ${PROJECT}/cassandra:${VERSION} . + docker build --pull --build-arg "CASSANDRA_VERSION=${CASSANDRA_VERSION}" -t ${PROJECT}/cassandra:${VERSION} image container-dev: - docker build --pull --build-arg "CASSANDRA_VERSION=${CASSANDRA_VERSION}" --build-arg "DEV_CONTAINER=true" -t ${PROJECT}/cassandra:${VERSION}-dev . + docker build --pull --build-arg "CASSANDRA_VERSION=${CASSANDRA_VERSION}" --build-arg "DEV_CONTAINER=true" -t ${PROJECT}/cassandra:${VERSION}-dev image build: container container-dev diff --git a/cassandra/cassandra-statefulset.yaml b/cassandra/cassandra-statefulset.yaml index 9cdb4611..02f7f7b9 100644 --- a/cassandra/cassandra-statefulset.yaml +++ b/cassandra/cassandra-statefulset.yaml @@ -18,7 +18,7 @@ spec: terminationGracePeriodSeconds: 1800 containers: - name: cassandra - image: gcr.io/google-samples/cassandra:v13 + image: gcr.io/google-samples/cassandra:v14 imagePullPolicy: Always ports: - containerPort: 7000 @@ -60,6 +60,8 @@ spec: value: "DC1-K8Demo" - name: CASSANDRA_RACK value: "Rack1-K8Demo" + - name: CASSANDRA_SEED_PROVIDER + value: io.k8s.cassandra.KubernetesSeedProvider - name: POD_IP valueFrom: fieldRef: diff --git a/cassandra/go/main.go b/cassandra/go/main.go new file mode 100644 index 00000000..a6b7e1a7 --- /dev/null +++ b/cassandra/go/main.go @@ -0,0 +1,82 @@ +package main + +// #include +// #include +import "C" + +import ( + "context" + "encoding/json" + "log" + "strings" + "unicode" + "unsafe" + + "github.com/ericchiang/k8s" + corev1 "github.com/ericchiang/k8s/apis/core/v1" +) + +type endpoints struct { + IPs []string `json:"ips"` +} + +// GetEndpoints searches the endpoints of a service returning a list of IP addresses. +//export GetEndpoints +func GetEndpoints(namespace, service, defSeeds *C.char) *C.char { + ns := C.GoString(namespace) + svc := C.GoString(service) + seeds := C.GoString(defSeeds) + + s := strings.Map(func(r rune) rune { + if unicode.IsSpace(r) { + return -1 + } + return r + }, seeds) + + nseeds := strings.Split(s, ",") + client, err := k8s.NewInClusterClient() + if err != nil { + log.Printf("unexpected error opening a connection against API server: %v\n", err) + log.Printf("returning default seeds: %v\n", nseeds) + return buildEndpoints(nseeds) + } + + ips := make([]string, 0) + + var endpoints corev1.Endpoints + err = client.Get(context.Background(), ns, svc, &endpoints) + if err != nil { + log.Printf("unexpected error obtaining information about service endpoints: %v\n", err) + log.Printf("returning default seeds: %v\n", nseeds) + return buildEndpoints(nseeds) + } + + for _, endpoint := range endpoints.Subsets { + for _, address := range endpoint.Addresses { + ips = append(ips, *address.Ip) + } + } + + if len(ips) == 0 { + return buildEndpoints(nseeds) + } + + return buildEndpoints(ips) +} + +func buildEndpoints(ips []string) *C.char { + b, err := json.Marshal(&endpoints{ips}) + if err != nil { + log.Printf("unexpected error serializing JSON response: %v\n", err) + rc := C.CString(`{"ips":[]}`) + defer C.free(unsafe.Pointer(rc)) + return rc + } + + rc := C.CString(string(b)) + defer C.free(unsafe.Pointer(rc)) + return rc +} + +func main() {} diff --git a/cassandra/image/files/build.sh b/cassandra/image/files/build.sh index 36333b2e..4ad10775 100755 --- a/cassandra/image/files/build.sh +++ b/cassandra/image/files/build.sh @@ -55,6 +55,10 @@ else rm -rf $CASSANDRA_HOME/pylib; fi +mv /kubernetes-cassandra.jar /usr/local/apache-cassandra-${CASSANDRA_VERSION}/lib +mv /cassandra-seed.so /etc/cassandra/ +mv /cassandra-seed.h /usr/local/lib/include + apt-get -y purge localepurge apt-get -y autoremove apt-get clean diff --git a/cassandra/image/files/cassandra-seed.h b/cassandra/image/files/cassandra-seed.h new file mode 100644 index 00000000..83dbdfe3 --- /dev/null +++ b/cassandra/image/files/cassandra-seed.h @@ -0,0 +1,79 @@ +/* Created by "go tool cgo" - DO NOT EDIT. */ + +/* package command-line-arguments */ + + +#line 1 "cgo-builtin-prolog" + +#include /* for ptrdiff_t below */ + +#ifndef GO_CGO_EXPORT_PROLOGUE_H +#define GO_CGO_EXPORT_PROLOGUE_H + +typedef struct { const char *p; ptrdiff_t n; } _GoString_; + +#endif + +/* Start of preamble from import "C" comments. */ + + +#line 3 "/home/aledbf/go/src/k8s.io/examples/cassandra/go/main.go" + #include + #include + +#line 1 "cgo-generated-wrapper" + + +/* End of preamble from import "C" comments. */ + + +/* Start of boilerplate cgo prologue. */ +#line 1 "cgo-gcc-export-header-prolog" + +#ifndef GO_CGO_PROLOGUE_H +#define GO_CGO_PROLOGUE_H + +typedef signed char GoInt8; +typedef unsigned char GoUint8; +typedef short GoInt16; +typedef unsigned short GoUint16; +typedef int GoInt32; +typedef unsigned int GoUint32; +typedef long long GoInt64; +typedef unsigned long long GoUint64; +typedef GoInt64 GoInt; +typedef GoUint64 GoUint; +typedef __SIZE_TYPE__ GoUintptr; +typedef float GoFloat32; +typedef double GoFloat64; +typedef float _Complex GoComplex64; +typedef double _Complex GoComplex128; + +/* + static assertion to make sure the file is being used on architecture + at least with matching size of GoInt. +*/ +typedef char _check_for_64_bit_pointer_matching_GoInt[sizeof(void*)==64/8 ? 1:-1]; + +typedef _GoString_ GoString; +typedef void *GoMap; +typedef void *GoChan; +typedef struct { void *t; void *v; } GoInterface; +typedef struct { void *data; GoInt len; GoInt cap; } GoSlice; + +#endif + +/* End of boilerplate cgo prologue. */ + +#ifdef __cplusplus +extern "C" { +#endif + + +// GetEndpoints searches the endpoints of a service returning a list of IP addresses. + +extern char* GetEndpoints(char* p0, char* p1, char* p2); + +#ifdef __cplusplus +} +#endif diff --git a/cassandra/image/files/kubernetes-cassandra.jar b/cassandra/image/files/kubernetes-cassandra.jar index ed2621975a4703a31ede77bd5f3f537f3a9fe720..4f7db7869abf6223b498a21e4e3a064ec0487f69 100644 GIT binary patch literal 7884 zcmbtZ1z1#D*QUE0q+>w3MN+yuL=0jW>29P%Iz>QIx*G}U4nY_|I;BGt7*b09;d+&O zk^6o3`Tjl6IdkTjwcfSPUTg2YLQN4K0S)GItf1>E{qgYU4Hf#WEUhKZCa)sHq4p;k z3d}N;tn`rVt`+oeU+6{fS2AUB6?qwHO)Yk1nLXvc4@!z`>_eD}Y|P-kfeH=I5uW)K zdsYSTdsam@2^8ezES(r4mKJ)~+GGVD2L;DB%*6yDyL*bz-=P}#iW-_Nv=NBGJ*4?) zO~IgiH?%JI9BMl8&zg>n8gIpQbk|8cZjq7h+C%JNVScIy12b{$E#RT*nc4i(|91l9 z9|RTx4*!7&_lgK;=-^;zZESD&iyY+-p#-26&sJC%7$T@dmYc|BZRD*TLH17#fgpNW z8%;-hGiwuepd}P8T;riCzBs}6oLT4Tx_b{gSf2R6u}Q6pQz@Cs#US{aEgQ1~VHGUUHJ)c}*K>`3Z)t6O8)hy|`$eb;d&>QJZO+H- z=nanL%SH+5Z;6q|m_!VA4y#^2WyF)G#V5u@Iywj4m&!LbAja@AVG>29L=~B*0}~%>JSR|0tM$SY6LcY2evM@{lL=74&vY86}*|72PV`DFuFp#hFH4 zhx`jPRn(53P@0IhEm-=4Q9w&Bxmr%<)L%F19!7f()9+bSq3J&`)@$Jk1bc(D5^t4l#bmj8Wd&BqruU?6nq!kU6ORF*!zq5f;V4z5cfHC z8q5hpJZ-ok84nP<*e^?!C^u>7v0gLMX>#D2+-rw5q^;vW62R<=@N=iR~2Y!QzPp5pe(oaPf}z@TXsi%Wor(_$3yml z5kupu7bnL-9_(Fm8(6wyGxo{b;~Z%o0 zAl07IqXIZm@-sV+w`~$V5xmKS(rF}-st0;Ue*VMxXjH<9B#vOubhwA4=#3@pxde0d zX8C>As}A>79?g?o&~5C@4E>&^={tDqU3{=~Ka6VN&5US<)F<^l-#;LEYgA?GXu8o{+r&+cWjX13C?8sOt(u*f z&Dg6R4bIZBGQRZV3&of8p{J+EQY3oKj3JK-t4;7JZxP))2oAhCB&sQ+#?W7HQ+qVy zvbB4%S1Njf&|>Av=#QatOCWP;fdB!35rZ*t`-?9xr&*fid=V_8|LtU+d9pl`9gQ7z zJ0*D9REzV3I_Ph;$Uax@E+#M3iaR@qmvr_u4 zW{ptQyiP_#Db!E2=WlKhi0JUB!bwVtj};Dx;2fZP0fC4>ty9E2CuH0 z7?X*l%WiePrRWo(j#;m6hm7`_{^i_`%J*Zm-Tg#T2WkT~0Uc>JikO?b5sb3tv8bfU zIQS)DR~OOEYsC0N4+4Ee+`0!+HZdz0+^L0ya3-IbYaJNVw`as0>dvbQIa4$- z!>OYZb|^SvCh17@X%QP(>$_Dpi#qw=2NPwsKzako?qdxbyj_nG!5xL0TJQJZpbNpz zWTkToRkItb=u?_Qr9-5Pld#*=%bzGG5z`x7pYN(OEWnP$dc)CUzWj~yNUygkAc?G> z@*uEL4k_vY_SHtKuSQN=X7f$b66LNea$GhT&=2 zUE4=9@|KPO-+dIdD9D+JkY((sJlN&Q4hPqi=bGm2zy=BU_X$za{)m{xadun~*#?6}>4q%BwZyifh1c83 z7l7}6ihx7I=79b}pu_!sJ1dY&FxXn!WMbTEao}5Ma2(`5zIQu|;CJgqs+yvuRKSx? z3#EAgVz!}8fAm`6zV6g4H_BPsGYLbS!g|z>P1nwjTQ!!8`s?Em8~lq_?}P)3hU|zt z$_@VS_#D}1i&RLab27Bbmj-tSg z;^{Y5u5T*a14~i+l42s4K4E~cyv>mNyj3v9MwO2m#|8xE*wwGW=1{Zb0%RXZ)+hPw z^L}MqRoCtcJ>T2n8{Hu*6eN{1jATh&Z|GGW7%RJY_~>DjO97vN_Q~kv#9{b*eU#1s zCCxMN=m5ub?vnCJA;}|KU+G3WRM4QyW4sT^+B7^Gj3)N%xUhz$L71d*%S2W zU*a>A(54n0?{8~oll6}jvknv{aK0ksmZsQHwwR~_U?UJN#MDAAcug*sSIp!(}E%+xaiJv(%7O~|4+G65mY~ z9o}6>{809aT(&4Cc6B0dT|sx_K|Du$rQm8@kwuY{g6XDtlB9|L>D>8g9Uj%<6vD}V zq~YuZ?AQ_MW0^Q>tN6|?)$(+XV^Mz3u>P4^@aM$NgU3A=zz?j}X#%0Q#5$DOg5p+f z_hJbH-*p9=lv^z<9}@K$b`#F6#hA~XB^L2G5(^$bWBeF12E=*D=|>?TVIKJ^w!YQ@ zI~y*9Bz=|3Yk{&`8vLXU^>E6rZ*eYul9^4(kYlUGkiNEpLM+`{qnvNRnQYt0hsmJ5@1t^#1ucVv(7z?YAz$Sx>Cy(1-u%_=FE@iCZFRRwdnQjX=VF!`s70N&Um>r1cQwmPv7?M)$G&hB1&eStT?{!Gds({<*hi$w=5RX z1B{Mr+tfbSCptts8phxv|-*z3#Rc>+F9Fg)2{i3Xc_?H#V} zijS$uwAh}dszoRVfb|6hs!0xG9tJU$8(C+Qbc?FnEjG82ID0D^nO;mpz!R*)udl4% zAz`aSqRCY-=2H`OM|zea39INQF0+*>p?k`k780vuV?04mIgvg=J65Iyc9eQ^h&Ppo z9E<(;7~y_XS(P$$(&3=7AjwS=uk>Sjt<;rO+wS3bevm~cL_ma`&dcu@a8N3lH^jS^ zd(o4GK`JYg>U8ynt&jDe&<}M`%|~*BiSU$cvG=(5=K30D&%YmkgR}MZL`Rwe@Us#O z4jB%2ybiWPT2hkHc*8?b-v}$YL3@ZO*&Z8J)6*pxHM3_Z7wdHA&{beQc;vW2S7U%1 zWLnQ%t*@dvy=2x)C$><%i6gYiZM5!~I)j^PE3~X`aHLI}+0x?zp-$qa1KkhO*3l~ z+-Iy1aLDJ~AFczJzCW+E*C+;eKc}}9ws*hbgrKETIz+qIQbI;oG|{(lrBZt6eW-Mi zT2ViOpHmyN&QSavp{&72=lK4z{L&DL?7&W(^Fa5QrX?da3gsf)R|AhGPZ0lZg8p{H zLRP2=V4)`9xyc0AeE*Nc>?Y^O@!YV!V`XJkJjSt4t)-^!0HJ20WYtCMRLnkdyv65~ zbkF(zj)@zd@HhA~^~EoGoiBp$TzeYa_5ykunnapm1lsfvDTxFU$rZ^zV6^7*3*H6p zI6DVocy&h*N;Z~;*D2-Ft&}V?(dJCPOAib9xWSqdx9ytMjZ> zL+{=G`{c(jxpGWFC!Z6V6*K>f$-mrqeyoUajcIdyagy&jlV1Q_RU78vfqK}=!&xB? zOa3o4^88=-8u`cG+cAGD6psI{f?J4Xzj$8(cOTY+3GzvKP))k`?cRCdwix*A+wp;r zPp<38J5;3J|?xnXR%(IkoWeNhi=>(Lz#MXSMYvMKqqB(@0ZgaWOnVjd{(n zS5eMe+9^O2N|hBQ7tod`k$Q{bR0o7HdoTNJwjxY+62aC*WeucVGbbb<>R*rTLE)At0; zl3(KLxQM>+WS+Hftjo58zq4)$MloFiq`!Drqg+Jmnp)X5?%Zr>rUhYHcn^7Ft94Mg zKwLw`kv089ZagKoUekVTqxqr2EmNEmdbbIwH%%<`eCik1mS=-Lv(^gKGlHOALi=Al ze8Mk<~*5*KhPiFU(=sG{ejk$(Ss6NmtbbLaelOxt|4wXX z!wxNWvHxDj`kkBaUvOLeDrT_#4v7Nu6;@YCZIos^2)Y5hf+ots|0|@ejTO7AmF1|; zLz`)S00iKgVI9$-0uXDoF7R3r>U2|-^y6Q+?=?_Zu8VtAf)xI)omYF(};$^z5D8+Jk+0oR+G=Q2A zcJ{y}MXF%hZ^cTrTxrMM5~c(K4`BPFkTUV<<31uXVe$@<9~Cj1u~(K!0AvkDM%eRK zDLdM9*#Q2tJ%fn^%vNxmvRV=*oB^gY&p#n8nR*~>XC#Tq+SmxY-l=J##IVWuz!g@q zV8rV|F)NixQ{olax&wbkh^bEn+kIw0(A4MDLwP%d+eH|_xe9VV&UwnI%|{XK0Lbx8 zXG$R8U3i&ycO0+k9@q(>^g!&zvXPuLfVFD04zDu$>Ef~1Cv=Bm>$??Yyo)mii9GJv zx=!ok5ryJ1YmfK~caGMTZ58S!=&)&C14_cZ{OM}8TDTRn%@#&!2BJP9Jw9w*%xa@> zsc)P=Pb#$Q93%cX@pe|w6MKjFDf4Rn=91S62ZpmHAL~%T`^*{+g4$;wl*}#zFfZnd z1C{X1sgT?+v6RKHda#xGeo`S3cM!fbe2WmUXI^fVCd|~7`1CL;P-C5uNG~5D|)*6^|^9X55G+# zWMmH<=mw6sJd;JSm=lJ=HhMQQna9*pbIj1W8eg=Y%;_l!iZoi$WVIdh5*KSV z(3|LwJwN9q=88;JLQjf{?8O=1XG6G~HK6gutyS;6`=Oal*7x&Irdo*?E^y6n=}70y zc=Sl@LzTi_Sqv z+o$LOQHmKY)rmq(O{ZGzas6Yi$sjMSipd(fvfZGU9IfwA}MMU^7L$eT0PMtT4$NM|zO|_`K4LYMI zoLqQo)C0bw(L_j+OZvKb-lHHW2M)|v@yqXD+wT?I&5e;u;>(fsoDq8e#rBsr$rZxy ztqJIs`RBM4+y#C9qmyx^>GfvEWiR8$xMWy^K0+CObT9V)+V_Y&}K#$=+nf|(+ z|75?p*YOh|>$%WP>^BA2KaTRNx4*e`eg!NIos8e~`geu&>q4&AZLfr|A^vM2H|u{r z&%T15K)(U{+Qj^xbzfKgdOCT9ua0>G{-4R^b(HHF(G|)F)(t4vw7eldx-Q^))PE&F z9p}FbxXJ;p<6e&yuW%jk{xj}%bNwFcT!vLY1`f={MLhmrmg~*pf)otgS4FLaBz zxajwP4IbKI{k3!aP~|tYe+a^Uinuup`za*rxz7J8;?JP$uXeiW mIJlA|04;+2PcL&R>FRJ$Q$&O|USMGGpr2CcqD3aX{PlkzMVZ_H literal 9843 zcmbt)1yq&Y(lFi4L8MdZLkdVrcOxy`-Hm{hbRRfKNOv~~NFyO3-Q6Lb!VjOXdau6s zTkrk;dDipneb$-Ud-k3+Yi8u7pka_89su6CT=IV=e|-?|M;Q@i0R{F<-4VgL@KN-@xW?jNjFWEy2%T(hT_ z{MaEg=IgXWXx6l^QP=w^^x3Ueh1FCcG7BVJN9CKnLD0GaR%9t;GnuMdtr^Wpul z^*;C%h5%8UmOhc^8~_Cyu;!=_3 zj|=prEqSs=7}aq#U3mh%J@7`D&SUX1tN8qEe%ud@V7uukJA2PfnmV(5qEm71q)+AN zmtT0qxNH-UBClYGjx*kEn0CP@Gd1(a>0yZ9;=$ZV~$&y-Ak#=>9IRVd!NK3 zBpmecE@uZXe6!-L^bZPX>X`)za5NC+%9!B`(I@z0mm8uzB zPQOY|LEEIewm`7nhMgYdjH4u&f{!S@$*yYK(?y_fCF2;V*Lk|bQnHv;s9sle^X<2D zwfnxc)$ZPnh~A&AXaB{y5->EhH*#?J%a6#**kC{Skr7LAY%IlTthy5);XX*@e0a(N zNd?NRp*&Kk=O^pC5m*}rgj4fJjhtk5i|{84U*Y9VYUAr{A=JkqT z`x(JFuao_0oLd^+cY{YzfuYa&{NOk%K4r15<8T!wi?E`J3}KPtF2Vcbn~BHY--ki^aSzeShab z5U`&{a>t~ph>xRvlZg-^FYw$a8Q)nFy^~&_V0nuLdiTQ(+Nn%35^1u%?pot-gR~){y`18~SFyx~JdUaPQuR9RGz4MXU{N zZOp74e^^oBrxmdkHQG#Qbl@p9(5f+bNvjKBaKr;}{xYH>)r_5yN8iD+e}^y0M;$@I zey+FB%R{|KVJ0{8{X7GaIt$Ivlda$Qrqu?^qL(5BA8kv#e+(Eu0zYSMh^QH-%zl_iHc&N%-l;T~eRRT)c%gE6j5RYdls z?EZ~1F8R{rsKB5!GkQd|x~_N&Xw$a$b1Z=O2?@iqIaSMEujS{BgR>x~d`O^P)KY3& zEb>ltN=~P8VD8nNp=iLa^|gMZf~%t}6!H52w#sBQP>1sq^K?c=HkOXk9w{oAvXcwC z(FB-!`kHq!{N=RHWe~ckLWfpbGIT@}fo3p%x1)PJ`+ zXf1Xu6axa{kQ@Sn;h$Um`;AReL-bT$tG-M7M0x&s^(iGZB%A^~KwMCFGK*7S+EScU zlNAzGZNE7MGF>fw7foSzQHSt&cQh+4lsC1w?+*D{ZdJP) zwQopnb-FxQZmE~N7>#E}wd-O444YT|OF*!&3CzWA9R z;f~NZyzn%}9SLxZzR;pq_~@sRL@}&|zP*LT_)7A)Bl&W<(#J(;jpZc*ByCewRDq!e zC5$?~f_%=Cj0WW#qYok)z+ogtu1yy7;;cwFQ=C`8l}|v*g=Vk|s|ljY_{!{xWA} zDlv=`dVXdO*E#{KD0TLk*Gmyj_JrM zUaD=Cs_H;oI%Y2wN@>kZgE1Vz<;vG0?RLWov#c0n8G6w=+t6oL3C|e8VX)jC@|6XJ zH7Fum`8&%J<+&gIw=y(Uud8K}F!zErC}9gT-AA?B-}OGqSC>&%wIr|GS)QZci_6I? zsm{8pH5UzPc$TA1`!=s!rT`PiD))=knF!cIM2U@fwwV?$tPJ?FX-3|UKvB9^(NrQm zm7RhQW3Oh4N?B-64(ZWKJ$6o{33H-&WxeH%vX~R|%LCLv8H$2)6}f%P5UrQD(Fg&= ztJ(}!YC3M6hF>Jj@hPPSosJW7tokLzxzsf6KXX8!7BkRMWUI#dE;oM=0826XeEhC81U) zg~9$`BVj?_!jHMU%xUpvb3|*XR@|e4{E9Ve4{wYuGHem8`XLk2j@MbW`{?S}=JnLa z*9n|Os{ARU8zUhHMaZpf6%t3od`*Qd^w4#^WjY!~7{KxrP>qOVyOryvb4#1n;db4| za0!)`Jm2`XIsxYb>kiP5fX!qkv=mh*m!>I(SLJHA<}%vZoj1fjLPU-|C_$EXRcEMn{@^XSTmoIMTpnHnnmUEIlT430$L~zN<+cv(L_vlsl5uh%pk6_+r@?0uZ1fEs9u-ffi+AqaX zx{w9Ngb3>!i!lx|@YP{gUE_9nd%;LU{(B5zwdEhUupFSAwd>zX&)f1C30GAsVB#_M z>@kJHlEBt3njmSnzlNPyfkQ*im3u zNKB}%u(-M@)1otE?G=%e)vITxr+X-Cdw55kvhJ^v3E$F3bf~G!t4bRpz%F#pn5apz zscFtq#*L5pw?AVz-Fx4(e zYS@H#&rxVbg__WiI!0(H%+cW5Gcu|xxt6~nohrnUO;Ek@qVC~VuBe38>k^iq$jM-4 zFBP0INQ8!;nG8HS`;Dpn_mJ5l;dNR!~A^HerCY zBn{gOZk%ZmKg*`1xG7wYjK<>0kK=`7%0jHJr*l<7Ob5voF@kA1JI@hPTv!x0;jY32 zIhNrfj*;esIq0eq&g|Z*Zax~Wi|3P4^L&k`oj&7V3O9HxpFXt5Njgk~G$-LgLrghp$d0Qk0(Vk}C(oLdpT(55 zLJCQD44dIjO0m2wP|cq+Boybna&aSVYCYkcRh5OuwBpMw4?j5!lQ(6{+b^|K!sViZ zj$Kklz9t63UFdQ_VC`NlUb^-nTeV^et0Vi^AF`*oiRrGK(G74Xbdfso7nB>>=njCG zp_oK20Jw*ws%+wIZ(!fxVFL}6#Yx0D*HOykx*JYmjY%Ix+Q>bDA}4@9(_*Z-DiVa> zLiU{GVhz9{-!m8rD=(g|cPvk_d*UveE=0~o&#$z+XL!0Q-kjV`KGg(SPJN)2v4L|% z?iJkXk-DXo&bUH_%xm`XEFSZOZfqkPYs z`XOJF5@_Bulr9WDJ${M-&dQ&?oMSFfKqDHLY&DDY-NohY?qq2>_@`O<+shwYE~*@J>uKsc>qBi{UUo=b zz*#YQ!Sb(s`>=JAKduti!|%?aH~c(>G{w-O38Ul7u(5{Qex`8=4u{O5qlQyb=jGh+gA#FXaw8AUWh?aQk7-&7`@=uXngeU162!r?x%tU(##!+capZz&I~hS~qCA9$RO-z=7wk z&ZQtQ)x$uYm&i-hGgob@P53i|op&@R*zpMqqTVMwn|7<^%dmEO68CCr0#w53&yjB! zqk|oDT|YcN7w_ub!;w=u9>k(ydD|oRRG(ZHv>`8QAQE8$b7*ZUd_A}=r_QCQM!Dl; zo8nXkR)4K=VMx*gclmkJCfAplDm6_>Rsa0c;`DU5mgqp+xX&c|q%YJ^ZCSVw6cIED_I{6yVw*W)q*3_(EtcBfh!jGZ?d3U684p zDmdum0~!%G!q(bfSI#!zVNLiH^cuYH?Io*=ATJ&H=2XXhEE>J=X%2QNGm2J>EHrR~ z`UB(P4bN=X!dB0Q4XcBV-y0@i*CYB4r@A@(%JEL3I_xrYmnG|UFZ2l<;9jZ+Ys`Fi z^>z6IsIMtj6}Ymmp_49$C#n!zI>`~xXRH%k8rSzIZIC0rFKUJ(Jey&s|G2TeJ(0Ng zeBkQ<02=o8joyV&j1A}jY8WKO+<5#Lh~jp0!~{fg^TI`^z@R>iOXZ%Mp#4Jzx`*<~bbJ0dIkihB)|QYrz&X?u*3d0GMr>#mQ60CxcILkz*@ zZL%W1h;8}cmQ|S7YmCp|(8(_5y^PRBT~gpVVB`C|RXnQ0^+pbZxb?L{AsL~VRnNvb zk-!|{Zn z{Yu7)9xc73Vh*3s2pwF4EU7N@=6@Ts*lz3&iM|5yL4NBw`$8sl2gF=tcb#0T0BFgM z$Co@gi|#8WU!*=c?MEe7d?mGcmqV0P5k!a7F79in(zetM*OFbNrRUlK(9ITjQg%M4 z&tuozQ!U&QSCae~?FYiMOB6kWEiMu24L!jYuz7Zv&2MlyyNYtp)fV8@CiJeLwO0|= zJ_$oLZiwzb@8_Lp_XtsGrCNG|$S8N^-FZb^-Yiyp5an-x75F4f+!f~ngYHR~{i)pK zkyzb)&`yjJb-;Q54rDb}vx>7Xb{OYv@P-!m<6(5yOyYfS*T+jaLWKUJ!^7%MC zt}yQ+7HJp8C^j9LN!hP!>H=+LDzPmz3IwfNuB+oO-RO{ek$R8PTCrLhi*(h+H7Gqd zl<Ux}I=;dvJl?1Js9SUJjMIN;st8sEz)W;uA z2V2-OP&hJnUsw=jm74fNl-lk*9^~p(d8{9gk_kl-cBblfVC=wqJDej>K%0dck@fl&*Wg+^S#5J`o*grT0qVkP>M4R+UUcH;zRZQy z%c2+XO4%PWWHeh}pQ-kL>a;&mtLy1oY}>Do_CV8A7j2G_If3NCn&%9V6bFecMA^2r zwIiPKCNBdh<+e?*{rGK~BQwsUTB0oi@VGTg6=hjwKV^$+9D7FVIojlTVvA%BO-Ats zy;r_+4;pqM9L@=#m=py`J3l&6_qfJf`PQB2o3>1TWAzmJQP~SBb@WB;;Og^>tm(E1Va;ZCfu0w)E~oJ4Yi}+}Qov zQZ)%@Nq+NDpU#}WYL+8y;E1Bn3w@^B2VI_;Z3V|>0A(?1ZkObFwjdz?>of!r0sB}CrTH)^??x^j~umIiy*7Z~1DX>j5c+O^_ z#S_*RegY)lt$~aQ?5ra+;O)W@%p|mEaSI)itHQC_caN4h$o`Z9kb6q87DgK{7vG!4 z%{^9c4=vk}8xC|q;BZGaISukRomdYkOE%3+m5t!s4If?qq7Vx1ekj=$YCF9g8#0dXq}q%^o>y zl^I-U#fAZI)@r zQI(-6YrLS)CQ42+nA>O~uE<39*4UZE_ zn^zee{pn-t_>$PyJnp($f}rCh4vob z4whgKC_IOHF~2W52d5>v(IU(EnYYd|Ev1@!W7rN?c5v})l>;C}ESCuXn~(Db(rdAl zOgPf9po+y;#yPve8Oh^l25_&ERd6?W{hEPWlPCeZQUV#uF~=}u!xU^GRWAaKsrRb9 zK|P&Oz3Gm#I4goff?)8*H3MURShMJo2JsLh;r8P%y)ebRH3K#6x@0PCSeeK4A)1Pn z@1Xl&YC%}|#Bh~htnTnwFCZbT^%3!~aL{;ZtA&q!XI}4@tXXb7CJphKJ|hjtP&?3i zu<{w1Vki(UVG89EA)fz~wtnJbE#@T`a-WHo^Aw*2P(c#{srZWJlZ?XE*d>&KlQm3C zzaLXxDhWK7HJ0O(GI#BP$2_qtC?I5(P~`w*zk-srw zkD^QES|O`>?V=mVyc2JA!5zHOjb1$9FP0+4!p|TOzk#QtycJLiLUsTc|ibPB{ znBbw}I8e1NP^~g&P}Z{UJXL)i@D2LkGv(c_lO{IzH_-k575e|ViMIAOwnp}jW=0Mo zpURQ^m@puQcy;U+XBS}wFbTQh(PSTSrLAkcv5jt|zBqTO|lea1^bv2||qInK} zSywi9dw0JcNGJgGe{P((U(`KdK%FS5UHqFg~F{8gL(o%!!2*r!;V%(l_eT1s`u|ox{9VZJ1*%^{c<-Bi{-KcXVcqYne~%J>vBt*# z$E+#;iWz?~|1%bN2s^$52E^SR3E{uok{?0HkF)nDG3gK5`Vo1_VDZf0da@REC2ui diff --git a/cassandra/java/pom.xml b/cassandra/java/pom.xml index f5478203..fcf20941 100644 --- a/cassandra/java/pom.xml +++ b/cassandra/java/pom.xml @@ -17,12 +17,12 @@ 4.0.0 io.k8s.cassandra kubernetes-cassandra - 1.0.2 + 1.0.3 maven-compiler-plugin - 3.5.1 + 3.5.1 1.8 1.8 @@ -33,7 +33,7 @@ 1.1.3 - 3.9 + 3.11.2 @@ -61,7 +61,6 @@ ${logback.version} provided - ch.qos.logback logback-core @@ -72,14 +71,13 @@ org.codehaus.jackson jackson-core-asl - 1.6.3 + 1.9.13 provided - org.codehaus.jackson jackson-mapper-asl - 1.6.3 + 1.9.13 provided diff --git a/cassandra/java/src/main/java/io/k8s/cassandra/GoInterface.java b/cassandra/java/src/main/java/io/k8s/cassandra/GoInterface.java new file mode 100644 index 00000000..17d9b4e0 --- /dev/null +++ b/cassandra/java/src/main/java/io/k8s/cassandra/GoInterface.java @@ -0,0 +1,54 @@ + +/* + * Copyright (C) 2018 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.k8s.cassandra; + +import java.util.Arrays; +import java.util.List; + +import com.sun.jna.Library; +import com.sun.jna.Pointer; +import com.sun.jna.Structure; + +public interface GoInterface extends Library { + public String GetEndpoints(String namespace, String service, String seeds); + + public class GoSlice extends Structure { + public static class ByValue extends GoSlice implements Structure.ByValue { + } + + public Pointer data; + public long len; + public long cap; + + protected List getFieldOrder() { + return Arrays.asList(new String[] { "data", "len", "cap" }); + } + } + + public class GoString extends Structure { + public static class ByValue extends GoString implements Structure.ByValue { + } + + public String p; + public long n; + + protected List getFieldOrder() { + return Arrays.asList(new String[] { "p", "n" }); + } + } +} \ No newline at end of file diff --git a/cassandra/java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java b/cassandra/java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java index a8940d3a..b0a94f0e 100644 --- a/cassandra/java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java +++ b/cassandra/java/src/main/java/io/k8s/cassandra/KubernetesSeedProvider.java @@ -1,3 +1,4 @@ + /* * Copyright (C) 2015 Google Inc. * @@ -16,237 +17,87 @@ package io.k8s.cassandra; -import org.apache.cassandra.config.Config; -import org.apache.cassandra.config.ConfigurationLoader; -import org.apache.cassandra.config.YamlConfigurationLoader; -import org.apache.cassandra.exceptions.ConfigurationException; +import java.io.IOException; +import java.net.InetAddress; +import java.util.Collections; +import java.util.List; +import java.util.Map; + import org.apache.cassandra.locator.SeedProvider; -import org.apache.cassandra.locator.SimpleSeedProvider; -import org.apache.cassandra.utils.FBUtilities; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.net.ssl.*; -import java.io.IOException; -import java.net.InetAddress; -import java.net.URL; -import java.net.UnknownHostException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import com.sun.jna.Native; /** * Self discovery {@link SeedProvider} that creates a list of Cassandra Seeds by * communicating with the Kubernetes API. - *

Various System Variable can be used to configure this provider: + *

+ * Various System Variable can be used to configure this provider: *

    - *
  • KUBERNETES_PORT_443_TCP_ADDR defaults to kubernetes.default.svc.cluster.local
    • - *
  • KUBERNETES_PORT_443_TCP_PORT defaults to 443
    • - *
  • CASSANDRA_SERVICE defaults to cassandra
    • - *
  • POD_NAMESPACE defaults to 'default'
    • - *
  • CASSANDRA_SERVICE_NUM_SEEDS defaults to 8 seeds
    • - *
  • K8S_ACCOUNT_TOKEN defaults to the path for the default token
    • + *
  • CASSANDRA_SERVICE defaults to cassandra
    • + *
  • POD_NAMESPACE defaults to 'default'
    • + *
  • CASSANDRA_SERVICE_NUM_SEEDS defaults to 8 seeds
    • *
*/ public class KubernetesSeedProvider implements SeedProvider { - private static final Logger logger = LoggerFactory.getLogger(KubernetesSeedProvider.class); + private static final Logger logger = LoggerFactory.getLogger(KubernetesSeedProvider.class); - /** - * default seeds to fall back on - */ - private List defaultSeeds; - private TrustManager[] trustAll; + /** + * Create new seed provider + * + * @param params + */ + public KubernetesSeedProvider(Map params) { + } - private HostnameVerifier trustAllHosts; + /** + * Call Kubernetes API to collect a list of seed providers + * + * @return list of seed providers + */ + public List getSeeds() { + GoInterface go = (GoInterface) Native.loadLibrary("cassandra-seed.so", GoInterface.class); - /** - * Create new Seeds - * @param params - */ - public KubernetesSeedProvider(Map params) { + String service = getEnvOrDefault("CASSANDRA_SERVICE", "cassandra"); + String namespace = getEnvOrDefault("POD_NAMESPACE", "default"); - // Create default seeds - defaultSeeds = createDefaultSeeds(); + String initialSeeds = getEnvOrDefault("CASSANDRA_SEEDS", ""); + if (initialSeeds.equals("")) { + initialSeeds = getEnvOrDefault("POD_IP", ""); + } - // TODO: Load the CA cert when it is available on all platforms. - trustAll = new TrustManager[] { - new X509TrustManager() { - public void checkServerTrusted(X509Certificate[] certs, String authType) {} - public void checkClientTrusted(X509Certificate[] certs, String authType) {} - public X509Certificate[] getAcceptedIssuers() { return null; } - } - }; + String seedSizeVar = getEnvOrDefault("CASSANDRA_SERVICE_NUM_SEEDS", "8"); + Integer seedSize = Integer.valueOf(seedSizeVar); - trustAllHosts = new HostnameVerifier() { - public boolean verify(String hostname, SSLSession session) { - return true; - } - }; - } + String data = go.GetEndpoints(namespace, service, initialSeeds); + ObjectMapper mapper = new ObjectMapper(); - /** - * Call kubernetes API to collect a list of seed providers - * @return list of seed providers - */ - public List getSeeds() { + try { + Endpoints endpoints = mapper.readValue(data, Endpoints.class); + logger.info("cassandra seeds: " + endpoints.ips.toString()); + return Collections.unmodifiableList(endpoints.ips); + } catch (IOException e) { + // This should not happen + logger.error("unexpected error building cassandra seeds: " + e.getMessage()); + return Collections.emptyList(); + } + } - String host = getEnvOrDefault("KUBERNETES_PORT_443_TCP_ADDR", "kubernetes.default.svc.cluster.local"); - String port = getEnvOrDefault("KUBERNETES_PORT_443_TCP_PORT", "443"); - String serviceName = getEnvOrDefault("CASSANDRA_SERVICE", "cassandra"); - String podNamespace = getEnvOrDefault("POD_NAMESPACE", "default"); - String path = String.format("/api/v1/namespaces/%s/endpoints/", podNamespace); - String seedSizeVar = getEnvOrDefault("CASSANDRA_SERVICE_NUM_SEEDS", "8"); - Integer seedSize = Integer.valueOf(seedSizeVar); - String accountToken = getEnvOrDefault("K8S_ACCOUNT_TOKEN", "/var/run/secrets/kubernetes.io/serviceaccount/token"); + private static String getEnvOrDefault(String var, String def) { + String val = System.getenv(var); + if (val == null) { + val = def; + } + return val; + } - List seeds = new ArrayList(); - try { - String token = getServiceAccountToken(accountToken); - - SSLContext ctx = SSLContext.getInstance("SSL"); - ctx.init(null, trustAll, new SecureRandom()); - - String PROTO = "https://"; - URL url = new URL(PROTO + host + ":" + port + path + serviceName); - logger.info("Getting endpoints from " + url); - HttpsURLConnection conn = (HttpsURLConnection)url.openConnection(); - - // TODO: Remove this once the CA cert is propagated everywhere, and replace - // with loading the CA cert. - conn.setHostnameVerifier(trustAllHosts); - - conn.setSSLSocketFactory(ctx.getSocketFactory()); - conn.addRequestProperty("Authorization", "Bearer " + token); - ObjectMapper mapper = new ObjectMapper(); - Endpoints endpoints = mapper.readValue(conn.getInputStream(), Endpoints.class); - - if (endpoints != null) { - // Here is a problem point, endpoints.subsets can be null in first node cases. - if (endpoints.subsets != null && !endpoints.subsets.isEmpty()){ - for (Subset subset : endpoints.subsets) { - if (subset.addresses != null && !subset.addresses.isEmpty()) { - for (Address address : subset.addresses) { - seeds.add(InetAddress.getByName(address.ip)); - - if(seeds.size() >= seedSize) { - logger.info("Available num endpoints: " + seeds.size()); - return Collections.unmodifiableList(seeds); - } - } - } - } - } - logger.info("Available num endpoints: " + seeds.size()); - } else { - logger.warn("Endpoints are not available using default seeds in cassandra.yaml"); - return Collections.unmodifiableList(defaultSeeds); - } - } catch (Exception ex) { - logger.warn("Request to kubernetes apiserver failed, using default seeds in cassandra.yaml", ex); - return Collections.unmodifiableList(defaultSeeds); - } - - if (seeds.size() == 0) { - // If we got nothing, we might be the first instance, in that case - // fall back on the seeds that were passed in cassandra.yaml. - logger.warn("Seeds are not available using default seeds in cassandra.yaml"); - return Collections.unmodifiableList(defaultSeeds); - } - - return Collections.unmodifiableList(seeds); - } - - /** - * Code taken from {@link SimpleSeedProvider}. This is used as a fall back - * incase we don't find seeds - * @return - */ - protected List createDefaultSeeds() - { - Config conf; - try { - conf = loadConfig(); - } - catch (Exception e) { - throw new AssertionError(e); - } - String[] hosts = conf.seed_provider.parameters.get("seeds").split(",", -1); - List seeds = new ArrayList(); - for (String host : hosts) { - try { - seeds.add(InetAddress.getByName(host.trim())); - } - catch (UnknownHostException ex) { - // not fatal... DD will bark if there end up being zero seeds. - logger.warn("Seed provider couldn't lookup host {}", host); - } - } - - if(seeds.size() == 0) { - try { - seeds.add(InetAddress.getLocalHost()); - } catch (UnknownHostException e) { - logger.warn("Seed provider couldn't lookup localhost"); - } - } - return Collections.unmodifiableList(seeds); - } - - /** - * Code taken from {@link SimpleSeedProvider} - * @return - */ - protected static Config loadConfig() throws ConfigurationException - { - String loaderClass = System.getProperty("cassandra.config.loader"); - ConfigurationLoader loader = loaderClass == null - ? new YamlConfigurationLoader() - : FBUtilities.construct(loaderClass, "configuration loading"); - return loader.loadConfig(); - } - - private static String getEnvOrDefault(String var, String def) { - String val = System.getenv(var); - if (val == null) { - val = def; - } - return val; - } - - private static String getServiceAccountToken(String file) { - try { - return new String(Files.readAllBytes(Paths.get(file))); - } catch (IOException e) { - logger.warn("unable to load service account token" + file); - throw new RuntimeException("Unable to load services account token " + file); - } - } - - protected List getDefaultSeeds() { - return defaultSeeds; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - static class Address { - public String ip; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - static class Subset { - public List
addresses; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - static class Endpoints { - public List subsets; - } + @JsonIgnoreProperties(ignoreUnknown = true) + static class Endpoints { + public List ips; + } } diff --git a/cassandra/java/src/test/java/io/k8s/cassandra/KubernetesSeedProviderTest.java b/cassandra/java/src/test/java/io/k8s/cassandra/KubernetesSeedProviderTest.java index e4a4ab86..2ef3c313 100644 --- a/cassandra/java/src/test/java/io/k8s/cassandra/KubernetesSeedProviderTest.java +++ b/cassandra/java/src/test/java/io/k8s/cassandra/KubernetesSeedProviderTest.java @@ -16,48 +16,32 @@ package io.k8s.cassandra; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertThat; + +import java.net.InetAddress; +import java.util.HashMap; +import java.util.List; + import org.apache.cassandra.locator.SeedProvider; import org.junit.Ignore; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.hamcrest.Matchers.*; - -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; - -import static org.junit.Assert.*; - public class KubernetesSeedProviderTest { - private static final Logger logger = LoggerFactory.getLogger(KubernetesSeedProviderTest.class); + private static final Logger logger = LoggerFactory.getLogger(KubernetesSeedProviderTest.class); - @Test - @Ignore("has to be run inside of a kube cluster") - public void getSeeds() throws Exception { - SeedProvider provider = new KubernetesSeedProvider(new HashMap()); - List seeds = provider.getSeeds(); - - assertThat(seeds, is(not(empty()))); - - } - - @Test - public void testDefaultSeeds() throws Exception { - - KubernetesSeedProvider provider = new KubernetesSeedProvider(new HashMap()); - List seeds = provider.getDefaultSeeds(); - List seedsTest = new ArrayList<>(); - seedsTest.add(InetAddress.getByName("8.4.4.4")); - seedsTest.add(InetAddress.getByName("8.8.8.8")); - assertThat(seeds, is(not(empty()))); - assertThat(seeds, is(seedsTest)); - logger.debug("seeds loaded {}", seeds); - - } + @Test + @Ignore("has to be run inside of a kube cluster") + public void getSeeds() throws Exception { + SeedProvider provider = new KubernetesSeedProvider(new HashMap()); + List seeds = provider.getSeeds(); + assertThat(seeds, is(not(empty()))); + } } \ No newline at end of file