Drop unnecessary dependencies on vegeta and licenseclassifier (#582)

This commit is contained in:
Markus Thömmes 2021-12-06 14:32:28 +01:00 committed by GitHub
parent a2e8051b64
commit c29dc56d8f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
125 changed files with 360 additions and 14298 deletions

15
go.mod
View File

@ -3,23 +3,8 @@ module knative.dev/caching
go 1.15
require (
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee // indirect
github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.19.5 // indirect
github.com/go-openapi/spec v0.19.6 // indirect
github.com/go-openapi/swag v0.19.15 // indirect
github.com/google/go-cmp v0.5.6
github.com/google/licenseclassifier v0.0.0-20200708223521-3d09a0ea2f39
github.com/imdario/mergo v0.3.9 // indirect
github.com/influxdata/tdigest v0.0.1 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/miekg/dns v1.1.29 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3 // indirect
github.com/tsenart/vegeta v12.7.1-0.20190725001342-b5f4fca92137+incompatible
go.uber.org/zap v1.19.1
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
k8s.io/api v0.21.4
k8s.io/apimachinery v0.21.4
k8s.io/client-go v0.21.4

56
go.sum
View File

@ -95,11 +95,8 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE=
github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee h1:BnPxIde0gjtTnc9Er7cxvBk8DHLWhEux0SxayC8dP6I=
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk=
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@ -139,9 +136,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E=
github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654 h1:XOPLOMn/zT4jIgxfxSsoXPxkrzz0FaCHwp33x5POJ+Q=
github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E=
github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77 h1:iRnqZBF0a1hoOOjOdPKf+IxqlJZOas7A48j77RAc7Yg=
github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77/go.mod h1:Va5MyIzkU0rAM92tn3hb3Anb7oz7KcnixF49+2wOMe4=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
@ -194,21 +188,17 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw=
github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
github.com/go-openapi/spec v0.19.6 h1:rMMMj8cV38KVXK7SFc+I2MWClbEfbK705+j+dyqun5g=
github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
@ -287,8 +277,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/licenseclassifier v0.0.0-20200708223521-3d09a0ea2f39 h1:nkr7S2ETn5pAuBeeoZggV5nXSwOm4nBLz3vscQfA/A8=
github.com/google/licenseclassifier v0.0.0-20200708223521-3d09a0ea2f39/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M=
github.com/google/mako v0.0.0-20190821191249-122f8dcef9e3/go.mod h1:YzLcVlL+NqWnmUEPuhS1LxDDwGO9WNbVlEXaF4IH35g=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@ -363,13 +351,10 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o=
github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY=
github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y=
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
@ -378,8 +363,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aW
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@ -413,10 +396,8 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@ -426,8 +407,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/miekg/dns v1.1.17/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M=
github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@ -524,9 +503,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@ -550,7 +526,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A=
github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -566,10 +541,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tsenart/go-tsz v0.0.0-20180814232043-cdeb9e1e981e/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo=
github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3 h1:pcQGQzTwCg//7FgVywqge1sW9Yf8VMsMdG58MI5kd8s=
github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo=
github.com/tsenart/vegeta v12.7.1-0.20190725001342-b5f4fca92137+incompatible h1:ErZrHhRveAoznVW80gbrxz+qxJNydpA2fcQxTPHkZbU=
github.com/tsenart/vegeta v12.7.1-0.20190725001342-b5f4fca92137+incompatible/go.mod h1:Smz/ZWfhKRcyDDChZkG3CyTHdj87lHzio/HOCkbndXM=
github.com/tsenart/vegeta/v12 v12.8.4/go.mod h1:ZiJtwLn/9M4fTPdMY7bdbIeyNeFVE8/AHbWFqCsUuho=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
@ -622,9 +593,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 h1:kETrAMYZq6WVGPa8IIixL0CaEcIUNi+1WX7grUoi3y8=
golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -634,7 +603,6 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
@ -685,7 +653,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -767,7 +734,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -838,11 +804,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@ -868,7 +832,6 @@ golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@ -913,10 +876,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 h1:4WsZyVtkthqrHTbDCJfiTs8IWNYE4uvsSDgaV6xpp+o=
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@ -1094,7 +1053,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=

View File

@ -33,10 +33,4 @@ import (
_ "k8s.io/code-generator/cmd/lister-gen"
_ "k8s.io/kube-openapi/cmd/openapi-gen"
_ "knative.dev/pkg/codegen/cmd/injection-gen"
// Licenseclassifier
_ "github.com/google/licenseclassifier"
// For load testing
_ "github.com/tsenart/vegeta"
)

View File

@ -1,24 +0,0 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof

View File

@ -1,14 +0,0 @@
sudo: false
language: go
go:
- 1.4
- 1.5
- 1.6
- 1.7
- 1.8
- 1.9
- tip
script:
- go test -v

View File

@ -1,21 +0,0 @@
The MIT License (MIT)
Copyright (c) 2016 Maciej Lisiewski
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -1,77 +0,0 @@
# datasize [![Build Status](https://travis-ci.org/c2h5oh/datasize.svg?branch=master)](https://travis-ci.org/c2h5oh/datasize)
Golang helpers for data sizes
### Constants
Just like `time` package provides `time.Second`, `time.Day` constants `datasize` provides:
* `datasize.B` 1 byte
* `datasize.KB` 1 kilobyte
* `datasize.MB` 1 megabyte
* `datasize.GB` 1 gigabyte
* `datasize.TB` 1 terabyte
* `datasize.PB` 1 petabyte
* `datasize.EB` 1 exabyte
### Helpers
Just like `time` package provides `duration.Nanoseconds() uint64 `, `duration.Hours() float64` helpers `datasize` has.
* `ByteSize.Bytes() uint64`
* `ByteSize.Kilobytes() float4`
* `ByteSize.Megabytes() float64`
* `ByteSize.Gigabytes() float64`
* `ByteSize.Terabytes() float64`
* `ByteSize.Petebytes() float64`
* `ByteSize.Exabytes() float64`
Warning: see limitations at the end of this document about a possible precission loss
### Parsing strings
`datasize.ByteSize` implements `TextUnmarshaler` interface and will automatically parse human readable strings into correct values where it is used:
* `"10 MB"` -> `10* datasize.MB`
* `"10240 g"` -> `10 * datasize.TB`
* `"2000"` -> `2000 * datasize.B`
* `"1tB"` -> `datasize.TB`
* `"5 peta"` -> `5 * datasize.PB`
* `"28 kilobytes"` -> `28 * datasize.KB`
* `"1 gigabyte"` -> `1 * datasize.GB`
You can also do it manually:
```go
var v datasize.ByteSize
err := v.UnmarshalText([]byte("100 mb"))
```
### Printing
`Bytesize.String()` uses largest unit allowing an integer value:
* `(102400 * datasize.MB).String()` -> `"100GB"`
* `(datasize.MB + datasize.KB).String()` -> `"1025KB"`
Use `%d` format string to get value in bytes without a unit.
### JSON and other encoding
Both `TextMarshaler` and `TextUnmarshaler` interfaces are implemented - JSON will just work. Other encoders will work provided they use those interfaces.
### Human readable
`ByteSize.HumanReadable()` or `ByteSize.HR()` returns a string with 1-3 digits, followed by 1 decimal place, a space and unit big enough to get 1-3 digits
* `(102400 * datasize.MB).String()` -> `"100.0 GB"`
* `(datasize.MB + 512 * datasize.KB).String()` -> `"1.5 MB"`
### Limitations
* The underlying data type for `data.ByteSize` is `uint64`, so values outside of 0 to 2^64-1 range will overflow
* size helper functions (like `ByteSize.Kilobytes()`) return `float64`, which can't represent all possible values of `uint64` accurately:
* if the returned value is supposed to have no fraction (ie `(10 * datasize.MB).Kilobytes()`) accuracy loss happens when value is more than 2^53 larger than unit: `.Kilobytes()` over 8 petabytes, `.Megabytes()` over 8 exabytes
* if the returned value is supposed to have a fraction (ie `(datasize.PB + datasize.B).Megabytes()`) in addition to the above note accuracy loss may occur in fractional part too - larger integer part leaves fewer bytes to store fractional part, the smaller the remainder vs unit the move bytes are required to store the fractional part
* Parsing a string with `Mb`, `Tb`, etc units will return a syntax error, because capital followed by lower case is commonly used for bits, not bytes
* Parsing a string with value exceeding 2^64-1 bytes will return 2^64-1 and an out of range error

View File

@ -1,217 +0,0 @@
package datasize
import (
"errors"
"fmt"
"strconv"
"strings"
)
type ByteSize uint64
const (
B ByteSize = 1
KB = B << 10
MB = KB << 10
GB = MB << 10
TB = GB << 10
PB = TB << 10
EB = PB << 10
fnUnmarshalText string = "UnmarshalText"
maxUint64 uint64 = (1 << 64) - 1
cutoff uint64 = maxUint64 / 10
)
var ErrBits = errors.New("unit with capital unit prefix and lower case unit (b) - bits, not bytes ")
func (b ByteSize) Bytes() uint64 {
return uint64(b)
}
func (b ByteSize) KBytes() float64 {
v := b / KB
r := b % KB
return float64(v) + float64(r)/float64(KB)
}
func (b ByteSize) MBytes() float64 {
v := b / MB
r := b % MB
return float64(v) + float64(r)/float64(MB)
}
func (b ByteSize) GBytes() float64 {
v := b / GB
r := b % GB
return float64(v) + float64(r)/float64(GB)
}
func (b ByteSize) TBytes() float64 {
v := b / TB
r := b % TB
return float64(v) + float64(r)/float64(TB)
}
func (b ByteSize) PBytes() float64 {
v := b / PB
r := b % PB
return float64(v) + float64(r)/float64(PB)
}
func (b ByteSize) EBytes() float64 {
v := b / EB
r := b % EB
return float64(v) + float64(r)/float64(EB)
}
func (b ByteSize) String() string {
switch {
case b == 0:
return fmt.Sprint("0B")
case b%EB == 0:
return fmt.Sprintf("%dEB", b/EB)
case b%PB == 0:
return fmt.Sprintf("%dPB", b/PB)
case b%TB == 0:
return fmt.Sprintf("%dTB", b/TB)
case b%GB == 0:
return fmt.Sprintf("%dGB", b/GB)
case b%MB == 0:
return fmt.Sprintf("%dMB", b/MB)
case b%KB == 0:
return fmt.Sprintf("%dKB", b/KB)
default:
return fmt.Sprintf("%dB", b)
}
}
func (b ByteSize) HR() string {
return b.HumanReadable()
}
func (b ByteSize) HumanReadable() string {
switch {
case b > EB:
return fmt.Sprintf("%.1f EB", b.EBytes())
case b > PB:
return fmt.Sprintf("%.1f PB", b.PBytes())
case b > TB:
return fmt.Sprintf("%.1f TB", b.TBytes())
case b > GB:
return fmt.Sprintf("%.1f GB", b.GBytes())
case b > MB:
return fmt.Sprintf("%.1f MB", b.MBytes())
case b > KB:
return fmt.Sprintf("%.1f KB", b.KBytes())
default:
return fmt.Sprintf("%d B", b)
}
}
func (b ByteSize) MarshalText() ([]byte, error) {
return []byte(b.String()), nil
}
func (b *ByteSize) UnmarshalText(t []byte) error {
var val uint64
var unit string
// copy for error message
t0 := t
var c byte
var i int
ParseLoop:
for i < len(t) {
c = t[i]
switch {
case '0' <= c && c <= '9':
if val > cutoff {
goto Overflow
}
c = c - '0'
val *= 10
if val > val+uint64(c) {
// val+v overflows
goto Overflow
}
val += uint64(c)
i++
default:
if i == 0 {
goto SyntaxError
}
break ParseLoop
}
}
unit = strings.TrimSpace(string(t[i:]))
switch unit {
case "Kb", "Mb", "Gb", "Tb", "Pb", "Eb":
goto BitsError
}
unit = strings.ToLower(unit)
switch unit {
case "", "b", "byte":
// do nothing - already in bytes
case "k", "kb", "kilo", "kilobyte", "kilobytes":
if val > maxUint64/uint64(KB) {
goto Overflow
}
val *= uint64(KB)
case "m", "mb", "mega", "megabyte", "megabytes":
if val > maxUint64/uint64(MB) {
goto Overflow
}
val *= uint64(MB)
case "g", "gb", "giga", "gigabyte", "gigabytes":
if val > maxUint64/uint64(GB) {
goto Overflow
}
val *= uint64(GB)
case "t", "tb", "tera", "terabyte", "terabytes":
if val > maxUint64/uint64(TB) {
goto Overflow
}
val *= uint64(TB)
case "p", "pb", "peta", "petabyte", "petabytes":
if val > maxUint64/uint64(PB) {
goto Overflow
}
val *= uint64(PB)
case "E", "EB", "e", "eb", "eB":
if val > maxUint64/uint64(EB) {
goto Overflow
}
val *= uint64(EB)
default:
goto SyntaxError
}
*b = ByteSize(val)
return nil
Overflow:
*b = ByteSize(maxUint64)
return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrRange}
SyntaxError:
*b = 0
return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrSyntax}
BitsError:
*b = 0
return &strconv.NumError{fnUnmarshalText, string(t0), ErrBits}
}

View File

@ -1,8 +1,8 @@
after_success:
- bash <(curl -s https://codecov.io/bash)
go:
- 1.14.x
- 1.15.x
- 1.11.x
- 1.12.x
install:
- GO111MODULE=off go get -u gotest.tools/gotestsum
env:

View File

@ -114,16 +114,16 @@ func getSingleImpl(node interface{}, decodedToken string, nameProvider *swag.Nam
rValue := reflect.Indirect(reflect.ValueOf(node))
kind := rValue.Kind()
if rValue.Type().Implements(jsonPointableType) {
r, err := node.(JSONPointable).JSONLookup(decodedToken)
if err != nil {
return nil, kind, err
}
return r, kind, nil
}
switch kind {
case reflect.Struct:
if rValue.Type().Implements(jsonPointableType) {
r, err := node.(JSONPointable).JSONLookup(decodedToken)
if err != nil {
return nil, kind, err
}
return r, kind, nil
}
nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken)
if !ok {
return nil, kind, fmt.Errorf("object has no field %q", decodedToken)
@ -161,17 +161,17 @@ func getSingleImpl(node interface{}, decodedToken string, nameProvider *swag.Nam
func setSingleImpl(node, data interface{}, decodedToken string, nameProvider *swag.NameProvider) error {
rValue := reflect.Indirect(reflect.ValueOf(node))
if ns, ok := node.(JSONSetable); ok { // pointer impl
return ns.JSONSet(decodedToken, data)
}
if rValue.Type().Implements(jsonSetableType) {
return node.(JSONSetable).JSONSet(decodedToken, data)
}
switch rValue.Kind() {
case reflect.Struct:
if ns, ok := node.(JSONSetable); ok { // pointer impl
return ns.JSONSet(decodedToken, data)
}
if rValue.Type().Implements(jsonSetableType) {
return node.(JSONSetable).JSONSet(decodedToken, data)
}
nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken)
if !ok {
return fmt.Errorf("object has no field %q", decodedToken)
@ -270,22 +270,22 @@ func (p *Pointer) set(node, data interface{}, nameProvider *swag.NameProvider) e
rValue := reflect.Indirect(reflect.ValueOf(node))
kind := rValue.Kind()
if rValue.Type().Implements(jsonPointableType) {
r, err := node.(JSONPointable).JSONLookup(decodedToken)
if err != nil {
return err
}
fld := reflect.ValueOf(r)
if fld.CanAddr() && fld.Kind() != reflect.Interface && fld.Kind() != reflect.Map && fld.Kind() != reflect.Slice && fld.Kind() != reflect.Ptr {
node = fld.Addr().Interface()
switch kind {
case reflect.Struct:
if rValue.Type().Implements(jsonPointableType) {
r, err := node.(JSONPointable).JSONLookup(decodedToken)
if err != nil {
return err
}
fld := reflect.ValueOf(r)
if fld.CanAddr() && fld.Kind() != reflect.Interface && fld.Kind() != reflect.Map && fld.Kind() != reflect.Slice && fld.Kind() != reflect.Ptr {
node = fld.Addr().Interface()
continue
}
node = r
continue
}
node = r
continue
}
switch kind {
case reflect.Struct:
nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken)
if !ok {
return fmt.Errorf("object has no field %q", decodedToken)

View File

@ -1,41 +0,0 @@
linters-settings:
govet:
check-shadowing: true
golint:
min-confidence: 0
gocyclo:
min-complexity: 30
maligned:
suggest-new: true
dupl:
threshold: 100
goconst:
min-len: 2
min-occurrences: 4
linters:
enable-all: true
disable:
- maligned
- lll
- gochecknoglobals
- godox
- gocognit
- whitespace
- wsl
- funlen
- gochecknoglobals
- gochecknoinits
- scopelint
- wrapcheck
- exhaustivestruct
- exhaustive
- nlreturn
- testpackage
- gci
- gofumpt
- goerr113
- gomnd
- tparallel
- nestif
- godot
- errorlint

View File

@ -1,19 +1,10 @@
after_success:
- bash <(curl -s https://codecov.io/bash)
go:
- 1.14.x
- 1.x
- 1.11.x
- 1.12.x
install:
- go get gotest.tools/gotestsum
jobs:
include:
# include linting job, but only for latest go version and amd64 arch
- go: 1.x
arch: amd64
install:
go get github.com/golangci/golangci-lint/cmd/golangci-lint
script:
- golangci-lint run --new-from-rev master
- GO111MODULE=off go get -u gotest.tools/gotestsum
env:
- GO111MODULE=on
language: go

View File

@ -4,7 +4,7 @@
An implementation of JSON Reference - Go language
## Status
Feature complete. Stable API
Work in progress ( 90% done )
## Dependencies
https://github.com/go-openapi/jsonpointer

View File

@ -6,7 +6,7 @@ require (
github.com/go-openapi/jsonpointer v0.19.3
github.com/stretchr/testify v1.3.0
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
golang.org/x/text v0.3.3 // indirect
golang.org/x/text v0.3.2 // indirect
)
go 1.13

View File

@ -5,8 +5,12 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
@ -24,12 +28,14 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=

View File

@ -14,41 +14,11 @@
package spec
import (
"encoding/json"
"github.com/go-openapi/swag"
)
// ContactInfo contact information for the exposed API.
//
// For more information: http://goo.gl/8us55a#contactObject
type ContactInfo struct {
ContactInfoProps
VendorExtensible
}
type ContactInfoProps struct {
Name string `json:"name,omitempty"`
URL string `json:"url,omitempty"`
Email string `json:"email,omitempty"`
}
func (c *ContactInfo) UnmarshalJSON(data []byte) error {
if err := json.Unmarshal(data, &c.ContactInfoProps); err != nil {
return err
}
return json.Unmarshal(data, &c.VendorExtensible)
}
func (c ContactInfo) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(c.ContactInfoProps)
if err != nil {
return nil, err
}
b2, err := json.Marshal(c.VendorExtensible)
if err != nil {
return nil, err
}
return swag.ConcatJSON(b1, b2), nil
}

View File

@ -14,40 +14,10 @@
package spec
import (
"encoding/json"
"github.com/go-openapi/swag"
)
// License information for the exposed API.
//
// For more information: http://goo.gl/8us55a#licenseObject
type License struct {
LicenseProps
VendorExtensible
}
type LicenseProps struct {
Name string `json:"name,omitempty"`
URL string `json:"url,omitempty"`
}
func (l *License) UnmarshalJSON(data []byte) error {
if err := json.Unmarshal(data, &l.LicenseProps); err != nil {
return err
}
return json.Unmarshal(data, &l.VendorExtensible)
}
func (l License) MarshalJSON() ([]byte, error) {
b1, err := json.Marshal(l.LicenseProps)
if err != nil {
return nil, err
}
b2, err := json.Marshal(l.VendorExtensible)
if err != nil {
return nil, err
}
return swag.ConcatJSON(b1, b2), nil
}

View File

@ -20,22 +20,3 @@ linters:
- lll
- gochecknoinits
- gochecknoglobals
- nlreturn
- testpackage
- wrapcheck
- gomnd
- exhaustive
- exhaustivestruct
- goerr113
- wsl
- whitespace
- gofumpt
- godot
- nestif
- godox
- funlen
- gci
- gocognit
- paralleltest
- thelper
- ifshort

View File

@ -1,34 +1,12 @@
after_success:
- bash <(curl -s https://codecov.io/bash)
go:
- 1.14.x
- 1.x
arch:
- amd64
jobs:
include:
# include arch ppc, but only for latest go version - skip testing for race
- go: 1.x
arch: ppc64le
install: ~
script:
- go test -v
#- go: 1.x
# arch: arm
# install: ~
# script:
# - go test -v
# include linting job, but only for latest go version and amd64 arch
- go: 1.x
arch: amd64
install:
go get github.com/golangci/golangci-lint/cmd/golangci-lint
script:
- golangci-lint run --new-from-rev master
- 1.11.x
- 1.12.x
install:
- GO111MODULE=off go get -u gotest.tools/gotestsum
env:
- GO111MODULE=on
language: go
notifications:
slack:

View File

@ -2,6 +2,7 @@
[![license](http://img.shields.io/badge/license-Apache%20v2-orange.svg)](https://raw.githubusercontent.com/go-openapi/swag/master/LICENSE)
[![GoDoc](https://godoc.org/github.com/go-openapi/swag?status.svg)](http://godoc.org/github.com/go-openapi/swag)
[![GolangCI](https://golangci.com/badges/github.com/go-openapi/swag.svg)](https://golangci.com)
[![Go Report Card](https://goreportcard.com/badge/github.com/go-openapi/swag)](https://goreportcard.com/report/github.com/go-openapi/swag)
Contains a bunch of helper functions for go-openapi and go-swagger projects.

View File

@ -88,7 +88,7 @@ func ConvertFloat64(str string) (float64, error) {
return strconv.ParseFloat(str, 64)
}
// ConvertInt8 turn a string into an int8
// ConvertInt8 turn a string into int8 boolean
func ConvertInt8(str string) (int8, error) {
i, err := strconv.ParseInt(str, 10, 8)
if err != nil {
@ -97,7 +97,7 @@ func ConvertInt8(str string) (int8, error) {
return int8(i), nil
}
// ConvertInt16 turn a string into an int16
// ConvertInt16 turn a string into a int16
func ConvertInt16(str string) (int16, error) {
i, err := strconv.ParseInt(str, 10, 16)
if err != nil {
@ -106,7 +106,7 @@ func ConvertInt16(str string) (int16, error) {
return int16(i), nil
}
// ConvertInt32 turn a string into an int32
// ConvertInt32 turn a string into a int32
func ConvertInt32(str string) (int32, error) {
i, err := strconv.ParseInt(str, 10, 32)
if err != nil {
@ -115,12 +115,12 @@ func ConvertInt32(str string) (int32, error) {
return int32(i), nil
}
// ConvertInt64 turn a string into an int64
// ConvertInt64 turn a string into a int64
func ConvertInt64(str string) (int64, error) {
return strconv.ParseInt(str, 10, 64)
}
// ConvertUint8 turn a string into an uint8
// ConvertUint8 turn a string into a uint8
func ConvertUint8(str string) (uint8, error) {
i, err := strconv.ParseUint(str, 10, 8)
if err != nil {
@ -129,7 +129,7 @@ func ConvertUint8(str string) (uint8, error) {
return uint8(i), nil
}
// ConvertUint16 turn a string into an uint16
// ConvertUint16 turn a string into a uint16
func ConvertUint16(str string) (uint16, error) {
i, err := strconv.ParseUint(str, 10, 16)
if err != nil {
@ -138,7 +138,7 @@ func ConvertUint16(str string) (uint16, error) {
return uint16(i), nil
}
// ConvertUint32 turn a string into an uint32
// ConvertUint32 turn a string into a uint32
func ConvertUint32(str string) (uint32, error) {
i, err := strconv.ParseUint(str, 10, 32)
if err != nil {
@ -147,7 +147,7 @@ func ConvertUint32(str string) (uint32, error) {
return uint32(i), nil
}
// ConvertUint64 turn a string into an uint64
// ConvertUint64 turn a string into a uint64
func ConvertUint64(str string) (uint64, error) {
return strconv.ParseUint(str, 10, 64)
}

View File

@ -181,12 +181,12 @@ func IntValueMap(src map[string]*int) map[string]int {
return dst
}
// Int32 returns a pointer to of the int32 value passed in.
// Int32 returns a pointer to of the int64 value passed in.
func Int32(v int32) *int32 {
return &v
}
// Int32Value returns the value of the int32 pointer passed in or
// Int32Value returns the value of the int64 pointer passed in or
// 0 if the pointer is nil.
func Int32Value(v *int32) int32 {
if v != nil {
@ -195,7 +195,7 @@ func Int32Value(v *int32) int32 {
return 0
}
// Int32Slice converts a slice of int32 values into a slice of
// Int32Slice converts a slice of int64 values into a slice of
// int32 pointers
func Int32Slice(src []int32) []*int32 {
dst := make([]*int32, len(src))
@ -299,80 +299,13 @@ func Int64ValueMap(src map[string]*int64) map[string]int64 {
return dst
}
// Uint16 returns a pointer to of the uint16 value passed in.
func Uint16(v uint16) *uint16 {
return &v
}
// Uint16Value returns the value of the uint16 pointer passed in or
// 0 if the pointer is nil.
func Uint16Value(v *uint16) uint16 {
if v != nil {
return *v
}
return 0
}
// Uint16Slice converts a slice of uint16 values into a slice of
// uint16 pointers
func Uint16Slice(src []uint16) []*uint16 {
dst := make([]*uint16, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Uint16ValueSlice converts a slice of uint16 pointers into a slice of
// uint16 values
func Uint16ValueSlice(src []*uint16) []uint16 {
dst := make([]uint16, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Uint16Map converts a string map of uint16 values into a string
// map of uint16 pointers
func Uint16Map(src map[string]uint16) map[string]*uint16 {
dst := make(map[string]*uint16)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Uint16ValueMap converts a string map of uint16 pointers into a string
// map of uint16 values
func Uint16ValueMap(src map[string]*uint16) map[string]uint16 {
dst := make(map[string]uint16)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Uint returns a pointer to of the uint value passed in.
// Uint returns a pouinter to of the uint value passed in.
func Uint(v uint) *uint {
return &v
}
// UintValue returns the value of the uint pointer passed in or
// 0 if the pointer is nil.
// UintValue returns the value of the uint pouinter passed in or
// 0 if the pouinter is nil.
func UintValue(v *uint) uint {
if v != nil {
return *v
@ -380,8 +313,8 @@ func UintValue(v *uint) uint {
return 0
}
// UintSlice converts a slice of uint values into a slice of
// uint pointers
// UintSlice converts a slice of uint values uinto a slice of
// uint pouinters
func UintSlice(src []uint) []*uint {
dst := make([]*uint, len(src))
for i := 0; i < len(src); i++ {
@ -390,7 +323,7 @@ func UintSlice(src []uint) []*uint {
return dst
}
// UintValueSlice converts a slice of uint pointers into a slice of
// UintValueSlice converts a slice of uint pouinters uinto a slice of
// uint values
func UintValueSlice(src []*uint) []uint {
dst := make([]uint, len(src))
@ -402,8 +335,8 @@ func UintValueSlice(src []*uint) []uint {
return dst
}
// UintMap converts a string map of uint values into a string
// map of uint pointers
// UintMap converts a string map of uint values uinto a string
// map of uint pouinters
func UintMap(src map[string]uint) map[string]*uint {
dst := make(map[string]*uint)
for k, val := range src {
@ -413,7 +346,7 @@ func UintMap(src map[string]uint) map[string]*uint {
return dst
}
// UintValueMap converts a string map of uint pointers into a string
// UintValueMap converts a string map of uint pouinters uinto a string
// map of uint values
func UintValueMap(src map[string]*uint) map[string]uint {
dst := make(map[string]uint)
@ -425,13 +358,13 @@ func UintValueMap(src map[string]*uint) map[string]uint {
return dst
}
// Uint32 returns a pointer to of the uint32 value passed in.
// Uint32 returns a pouinter to of the uint64 value passed in.
func Uint32(v uint32) *uint32 {
return &v
}
// Uint32Value returns the value of the uint32 pointer passed in or
// 0 if the pointer is nil.
// Uint32Value returns the value of the uint64 pouinter passed in or
// 0 if the pouinter is nil.
func Uint32Value(v *uint32) uint32 {
if v != nil {
return *v
@ -439,8 +372,8 @@ func Uint32Value(v *uint32) uint32 {
return 0
}
// Uint32Slice converts a slice of uint32 values into a slice of
// uint32 pointers
// Uint32Slice converts a slice of uint64 values uinto a slice of
// uint32 pouinters
func Uint32Slice(src []uint32) []*uint32 {
dst := make([]*uint32, len(src))
for i := 0; i < len(src); i++ {
@ -449,7 +382,7 @@ func Uint32Slice(src []uint32) []*uint32 {
return dst
}
// Uint32ValueSlice converts a slice of uint32 pointers into a slice of
// Uint32ValueSlice converts a slice of uint32 pouinters uinto a slice of
// uint32 values
func Uint32ValueSlice(src []*uint32) []uint32 {
dst := make([]uint32, len(src))
@ -461,8 +394,8 @@ func Uint32ValueSlice(src []*uint32) []uint32 {
return dst
}
// Uint32Map converts a string map of uint32 values into a string
// map of uint32 pointers
// Uint32Map converts a string map of uint32 values uinto a string
// map of uint32 pouinters
func Uint32Map(src map[string]uint32) map[string]*uint32 {
dst := make(map[string]*uint32)
for k, val := range src {
@ -472,7 +405,7 @@ func Uint32Map(src map[string]uint32) map[string]*uint32 {
return dst
}
// Uint32ValueMap converts a string map of uint32 pointers into a string
// Uint32ValueMap converts a string map of uint32 pouinters uinto a string
// map of uint32 values
func Uint32ValueMap(src map[string]*uint32) map[string]uint32 {
dst := make(map[string]uint32)
@ -484,13 +417,13 @@ func Uint32ValueMap(src map[string]*uint32) map[string]uint32 {
return dst
}
// Uint64 returns a pointer to of the uint64 value passed in.
// Uint64 returns a pouinter to of the uint64 value passed in.
func Uint64(v uint64) *uint64 {
return &v
}
// Uint64Value returns the value of the uint64 pointer passed in or
// 0 if the pointer is nil.
// Uint64Value returns the value of the uint64 pouinter passed in or
// 0 if the pouinter is nil.
func Uint64Value(v *uint64) uint64 {
if v != nil {
return *v
@ -498,8 +431,8 @@ func Uint64Value(v *uint64) uint64 {
return 0
}
// Uint64Slice converts a slice of uint64 values into a slice of
// uint64 pointers
// Uint64Slice converts a slice of uint64 values uinto a slice of
// uint64 pouinters
func Uint64Slice(src []uint64) []*uint64 {
dst := make([]*uint64, len(src))
for i := 0; i < len(src); i++ {
@ -508,7 +441,7 @@ func Uint64Slice(src []uint64) []*uint64 {
return dst
}
// Uint64ValueSlice converts a slice of uint64 pointers into a slice of
// Uint64ValueSlice converts a slice of uint64 pouinters uinto a slice of
// uint64 values
func Uint64ValueSlice(src []*uint64) []uint64 {
dst := make([]uint64, len(src))
@ -520,8 +453,8 @@ func Uint64ValueSlice(src []*uint64) []uint64 {
return dst
}
// Uint64Map converts a string map of uint64 values into a string
// map of uint64 pointers
// Uint64Map converts a string map of uint64 values uinto a string
// map of uint64 pouinters
func Uint64Map(src map[string]uint64) map[string]*uint64 {
dst := make(map[string]*uint64)
for k, val := range src {
@ -531,7 +464,7 @@ func Uint64Map(src map[string]uint64) map[string]*uint64 {
return dst
}
// Uint64ValueMap converts a string map of uint64 pointers into a string
// Uint64ValueMap converts a string map of uint64 pouinters uinto a string
// map of uint64 values
func Uint64ValueMap(src map[string]*uint64) map[string]uint64 {
dst := make(map[string]uint64)
@ -543,74 +476,6 @@ func Uint64ValueMap(src map[string]*uint64) map[string]uint64 {
return dst
}
// Float32 returns a pointer to of the float32 value passed in.
func Float32(v float32) *float32 {
return &v
}
// Float32Value returns the value of the float32 pointer passed in or
// 0 if the pointer is nil.
func Float32Value(v *float32) float32 {
if v != nil {
return *v
}
return 0
}
// Float32Slice converts a slice of float32 values into a slice of
// float32 pointers
func Float32Slice(src []float32) []*float32 {
dst := make([]*float32, len(src))
for i := 0; i < len(src); i++ {
dst[i] = &(src[i])
}
return dst
}
// Float32ValueSlice converts a slice of float32 pointers into a slice of
// float32 values
func Float32ValueSlice(src []*float32) []float32 {
dst := make([]float32, len(src))
for i := 0; i < len(src); i++ {
if src[i] != nil {
dst[i] = *(src[i])
}
}
return dst
}
// Float32Map converts a string map of float32 values into a string
// map of float32 pointers
func Float32Map(src map[string]float32) map[string]*float32 {
dst := make(map[string]*float32)
for k, val := range src {
v := val
dst[k] = &v
}
return dst
}
// Float32ValueMap converts a string map of float32 pointers into a string
// map of float32 values
func Float32ValueMap(src map[string]*float32) map[string]float32 {
dst := make(map[string]float32)
for k, val := range src {
if val != nil {
dst[k] = *val
}
}
return dst
}
// Float64 returns a pointer to of the float64 value passed in.
func Float64(v float64) *float64 {
return &v

View File

@ -2,17 +2,13 @@ module github.com/go-openapi/swag
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mailru/easyjson v0.7.6
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
github.com/stretchr/testify v1.6.1
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63
github.com/stretchr/testify v1.3.0
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
gopkg.in/yaml.v2 v2.2.2
)
replace github.com/golang/lint => golang.org/x/lint v0.0.0-20190409202823-959b441ac422
replace sourcegraph.com/sourcegraph/go-diff => github.com/sourcegraph/go-diff v0.5.1
go 1.11

View File

@ -1,29 +1,20 @@
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

View File

@ -51,7 +51,7 @@ type ejUnmarshaler interface {
UnmarshalEasyJSON(w *jlexer.Lexer)
}
// WriteJSON writes json data, prefers finding an appropriate interface to short-circuit the marshaler
// WriteJSON writes json data, prefers finding an appropriate interface to short-circuit the marshaller
// so it takes the fastest option available.
func WriteJSON(data interface{}) ([]byte, error) {
if d, ok := data.(ejMarshaler); ok {
@ -65,8 +65,8 @@ func WriteJSON(data interface{}) ([]byte, error) {
return json.Marshal(data)
}
// ReadJSON reads json data, prefers finding an appropriate interface to short-circuit the unmarshaler
// so it takes the fastest option available
// ReadJSON reads json data, prefers finding an appropriate interface to short-circuit the unmarshaller
// so it takes the fastes option available
func ReadJSON(data []byte, value interface{}) error {
trimmedData := bytes.Trim(data, "\x00")
if d, ok := value.(ejUnmarshaler); ok {
@ -189,7 +189,7 @@ func FromDynamicJSON(data, target interface{}) error {
return json.Unmarshal(b, target)
}
// NameProvider represents an object capable of translating from go property names
// NameProvider represents an object capabale of translating from go property names
// to json property names
// This type is thread-safe.
type NameProvider struct {

View File

@ -19,9 +19,7 @@ import (
"io/ioutil"
"log"
"net/http"
"net/url"
"path/filepath"
"runtime"
"strings"
"time"
)
@ -29,15 +27,6 @@ import (
// LoadHTTPTimeout the default timeout for load requests
var LoadHTTPTimeout = 30 * time.Second
// LoadHTTPBasicAuthUsername the username to use when load requests require basic auth
var LoadHTTPBasicAuthUsername = ""
// LoadHTTPBasicAuthPassword the password to use when load requests require basic auth
var LoadHTTPBasicAuthPassword = ""
// LoadHTTPCustomHeaders an optional collection of custom HTTP headers for load requests
var LoadHTTPCustomHeaders = map[string]string{}
// LoadFromFileOrHTTP loads the bytes from a file or a remote http server based on the path passed in
func LoadFromFileOrHTTP(path string) ([]byte, error) {
return LoadStrategy(path, ioutil.ReadFile, loadHTTPBytes(LoadHTTPTimeout))(path)
@ -59,26 +48,6 @@ func LoadStrategy(path string, local, remote func(string) ([]byte, error)) func(
if err != nil {
return nil, err
}
if strings.HasPrefix(pth, `file://`) {
if runtime.GOOS == "windows" {
// support for canonical file URIs on windows.
// Zero tolerance here for dodgy URIs.
u, _ := url.Parse(upth)
if u.Host != "" {
// assume UNC name (volume share)
// file://host/share/folder\... ==> \\host\share\path\folder
// NOTE: UNC port not yet supported
upth = strings.Join([]string{`\`, u.Host, u.Path}, `\`)
} else {
// file:///c:/folder/... ==> just remove the leading slash
upth = strings.TrimPrefix(upth, `file:///`)
}
} else {
upth = strings.TrimPrefix(upth, `file://`)
}
}
return local(filepath.FromSlash(upth))
}
}
@ -86,19 +55,10 @@ func LoadStrategy(path string, local, remote func(string) ([]byte, error)) func(
func loadHTTPBytes(timeout time.Duration) func(path string) ([]byte, error) {
return func(path string) ([]byte, error) {
client := &http.Client{Timeout: timeout}
req, err := http.NewRequest("GET", path, nil) // nolint: noctx
req, err := http.NewRequest("GET", path, nil)
if err != nil {
return nil, err
}
if LoadHTTPBasicAuthUsername != "" && LoadHTTPBasicAuthPassword != "" {
req.SetBasicAuth(LoadHTTPBasicAuthUsername, LoadHTTPBasicAuthPassword)
}
for key, val := range LoadHTTPCustomHeaders {
req.Header.Set(key, val)
}
resp, err := client.Do(req)
defer func() {
if resp != nil {

View File

@ -31,7 +31,7 @@ var isInitialism func(string) bool
// GoNamePrefixFunc sets an optional rule to prefix go names
// which do not start with a letter.
//
// e.g. to help convert "123" into "{prefix}123"
// e.g. to help converting "123" into "{prefix}123"
//
// The default is to prefix with "X"
var GoNamePrefixFunc func(string) string
@ -91,7 +91,7 @@ func init() {
}
const (
// collectionFormatComma = "csv"
//collectionFormatComma = "csv"
collectionFormatSpace = "ssv"
collectionFormatTab = "tsv"
collectionFormatPipe = "pipes"
@ -370,7 +370,7 @@ func IsZero(data interface{}) bool {
// AddInitialisms add additional initialisms
func AddInitialisms(words ...string) {
for _, word := range words {
// commonInitialisms[upper(word)] = true
//commonInitialisms[upper(word)] = true
commonInitialisms.add(upper(word))
}
// sort again

View File

@ -1,16 +0,0 @@
sudo: false
language: go
go:
- "1.10"
- "1.11"
- "1.12"
matrix:
allow_failures:
- go: master
fast_finish: true
script:
- go get -t -v ./...
- diff -u <(echo -n) <(gofmt -d -s .)
- go generate -x ./... && git diff --exit-code; code=$?; git checkout -- .; (exit $code) # Check that go generate ./... produces a zero diff; clean up any changes afterwards.
- go vet .
- go test -v -race ./...

View File

@ -1,14 +0,0 @@
# Change Log
# All notable changes to this project will be documented in this file.
# This project adheres to [Semantic Versioning](http://semver.org/).
## [0.1.0] UNRELEASED
### Init
- Initial development.
### Changed
- Create a database of forbidden licenses to make matching just forbidden
licenses quicker.
- Remove non-words from the license texts. It makes the matching more precise.
This subsumes what the "remove common prefix" and a few other passes did.
- Remove the "lattice" structure in favor of a simpler and smaller list of
substrings. This gives us a tremendous speed increase of up to 2x.

View File

@ -1,24 +0,0 @@
# How to contribute
We'd love to accept your patches and contributions to this project. There are
just a few small guidelines you need to follow.
## Contributor License Agreement
Contributions to this project must be accompanied by a Contributor License
Agreement. You (or your employer) retain the copyright to your contribution,
this simply gives us permission to use and redistribute your contributions as
part of the project. Head over to <https://cla.developers.google.com/> to see
your current agreements on file or to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted
one (even if it was for a different project), you probably don't need to do it
again.
## Code reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult [GitHub Help] for more
information on using pull requests.
[GitHub Help]: https://help.github.com/articles/about-pull-requests/

View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -1,66 +0,0 @@
# License Classifier
[![Build status](https://travis-ci.org/google/licenseclassifier.svg?branch=master)](https://travis-ci.org/google/licenseclassifier)
## Introduction
The license classifier is a library and set of tools that can analyze text to
determine what type of license it contains. It searches for license texts in a
file and compares them to an archive of known licenses. These files could be,
e.g., `LICENSE` files with a single or multiple licenses in it, or source code
files with the license text in a comment.
A "confidence level" is associated with each result indicating how close the
match was. A confidence level of `1.0` indicates an exact match, while a
confidence level of `0.0` indicates that no license was able to match the text.
## Adding a new license
Adding a new license is straight-forward:
1. Create a file in `licenses/`.
* The filename should be the name of the license or its abbreviation. If
the license is an Open Source license, use the appropriate identifier
specified at https://spdx.org/licenses/.
* If the license is the "header" version of the license, append the suffix
"`.header`" to it. See `licenses/README.md` for more details.
2. Add the license name to the list in `license_type.go`.
3. Regenerate the `licenses.db` file by running the license serializer:
```shell
$ license_serializer -output licenseclassifier/licenses
```
4. Create and run appropriate tests to verify that the license is indeed
present.
## Tools
### Identify license
`identify_license` is a command line tool that can identify the license(s)
within a file.
```shell
$ identify_license LICENSE
LICENSE: GPL-2.0 (confidence: 1, offset: 0, extent: 14794)
LICENSE: LGPL-2.1 (confidence: 1, offset: 18366, extent: 23829)
LICENSE: MIT (confidence: 1, offset: 17255, extent: 1059)
```
### License serializer
The `license_serializer` tool regenerates the `licenses.db` archive. The archive
contains preprocessed license texts for quicker comparisons against unknown
texts.
```shell
$ license_serializer -output licenseclassifier/licenses
```
----
This is not an official Google product (experimental or otherwise), it is just
code that happens to be owned by Google.

View File

@ -1,472 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package licenseclassifier provides methods to identify the open source
// license that most closely matches an unknown license.
package licenseclassifier
import (
"archive/tar"
"bytes"
"compress/gzip"
"fmt"
"html"
"io"
"math"
"regexp"
"sort"
"strings"
"sync"
"unicode"
"github.com/google/licenseclassifier/stringclassifier"
"github.com/google/licenseclassifier/stringclassifier/searchset"
)
// DefaultConfidenceThreshold is the minimum confidence percentage we're willing to accept in order
// to say that a match is good.
const DefaultConfidenceThreshold = 0.80
var (
// Normalizers is a list of functions that get applied to the strings
// before they are registered with the string classifier.
Normalizers = []stringclassifier.NormalizeFunc{
html.UnescapeString,
removeShebangLine,
RemoveNonWords,
NormalizeEquivalentWords,
NormalizePunctuation,
strings.ToLower,
removeIgnorableTexts,
stringclassifier.FlattenWhitespace,
strings.TrimSpace,
}
// commonLicenseWords are words that are common to all known licenses.
// If an unknown text doesn't have at least one of these, then we can
// ignore it.
commonLicenseWords = []*regexp.Regexp{
regexp.MustCompile(`(?i)\bcode\b`),
regexp.MustCompile(`(?i)\blicense\b`),
regexp.MustCompile(`(?i)\boriginal\b`),
regexp.MustCompile(`(?i)\brights\b`),
regexp.MustCompile(`(?i)\bsoftware\b`),
regexp.MustCompile(`(?i)\bterms\b`),
regexp.MustCompile(`(?i)\bversion\b`),
regexp.MustCompile(`(?i)\bwork\b`),
}
)
// License is a classifier pre-loaded with known open source licenses.
type License struct {
c *stringclassifier.Classifier
// Threshold is the lowest confidence percentage acceptable for the
// classifier.
Threshold float64
// archive is a function that must return the contents of the license archive.
// When archive is nil, ReadLicenseFile(LicenseFile) is used to retrieve the
// contents.
archive func() ([]byte, error)
}
// OptionFunc set options on a License struct.
type OptionFunc func(l *License) error
// Archive is an OptionFunc to specify the location of the license archive file.
func Archive(f string) OptionFunc {
return func(l *License) error {
l.archive = func() ([]byte, error) { return ReadLicenseFile(f) }
return nil
}
}
// ArchiveBytes is an OptionFunc that provides the contents of the license archive file.
// The caller must not overwrite the contents of b as it is not copied.
func ArchiveBytes(b []byte) OptionFunc {
return func(l *License) error {
l.archive = func() ([]byte, error) { return b, nil }
return nil
}
}
// ArchiveFunc is an OptionFunc that provides a function that must return the contents
// of the license archive file.
func ArchiveFunc(f func() ([]byte, error)) OptionFunc {
return func(l *License) error {
l.archive = f
return nil
}
}
// New creates a license classifier and pre-loads it with known open source licenses.
func New(threshold float64, options ...OptionFunc) (*License, error) {
classifier := &License{
c: stringclassifier.New(threshold, Normalizers...),
Threshold: threshold,
}
for _, o := range options {
err := o(classifier)
if err != nil {
return nil, fmt.Errorf("error setting option %v: %v", o, err)
}
}
if err := classifier.registerLicenses(); err != nil {
return nil, fmt.Errorf("cannot register licenses from archive: %v", err)
}
return classifier, nil
}
// NewWithForbiddenLicenses creates a license classifier and pre-loads it with
// known open source licenses which are forbidden.
func NewWithForbiddenLicenses(threshold float64, options ...OptionFunc) (*License, error) {
opts := []OptionFunc{Archive(ForbiddenLicenseArchive)}
opts = append(opts, options...)
return New(threshold, opts...)
}
// WithinConfidenceThreshold returns true if the confidence value is above or
// equal to the confidence threshold.
func (c *License) WithinConfidenceThreshold(conf float64) bool {
return conf > c.Threshold || math.Abs(conf-c.Threshold) < math.SmallestNonzeroFloat64
}
// NearestMatch returns the "nearest" match to the given set of known licenses.
// Returned are the name of the license, and a confidence percentage indicating
// how confident the classifier is in the result.
func (c *License) NearestMatch(contents string) *stringclassifier.Match {
if !c.hasCommonLicenseWords(contents) {
return nil
}
m := c.c.NearestMatch(contents)
m.Name = strings.TrimSuffix(m.Name, ".header")
return m
}
// MultipleMatch matches all licenses within an unknown text.
func (c *License) MultipleMatch(contents string, includeHeaders bool) stringclassifier.Matches {
norm := normalizeText(contents)
if !c.hasCommonLicenseWords(norm) {
return nil
}
m := make(map[stringclassifier.Match]bool)
var matches stringclassifier.Matches
for _, v := range c.c.MultipleMatch(norm) {
if !c.WithinConfidenceThreshold(v.Confidence) {
continue
}
if !includeHeaders && strings.HasSuffix(v.Name, ".header") {
continue
}
v.Name = strings.TrimSuffix(v.Name, ".header")
if re, ok := forbiddenRegexps[v.Name]; ok && !re.MatchString(norm) {
continue
}
if _, ok := m[*v]; !ok {
m[*v] = true
matches = append(matches, v)
}
}
sort.Sort(matches)
return matches
}
func normalizeText(s string) string {
for _, n := range Normalizers {
s = n(s)
}
return s
}
// hasCommonLicenseWords returns true if the unknown text has at least one word
// that's common to all licenses.
func (c *License) hasCommonLicenseWords(s string) bool {
for _, re := range commonLicenseWords {
if re.MatchString(s) {
return true
}
}
return false
}
type archivedValue struct {
name string
normalized string
set *searchset.SearchSet
}
// registerLicenses loads all known licenses and adds them to c as known values
// for comparison. The allocated space after ingesting the 'licenses.db'
// archive is ~167M.
func (c *License) registerLicenses() error {
var contents []byte
var err error
if c.archive == nil {
contents, err = ReadLicenseFile(LicenseArchive)
} else {
contents, err = c.archive()
}
if err != nil {
return err
}
reader := bytes.NewReader(contents)
gr, err := gzip.NewReader(reader)
if err != nil {
return err
}
defer gr.Close()
tr := tar.NewReader(gr)
var muVals sync.Mutex
var vals []archivedValue
for i := 0; ; i++ {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
name := strings.TrimSuffix(hdr.Name, ".txt")
// Read normalized value.
var b bytes.Buffer
if _, err := io.Copy(&b, tr); err != nil {
return err
}
normalized := b.String()
b.Reset()
// Read precomputed hashes.
hdr, err = tr.Next()
if err != nil {
return err
}
if _, err := io.Copy(&b, tr); err != nil {
return err
}
var set searchset.SearchSet
searchset.Deserialize(&b, &set)
muVals.Lock()
vals = append(vals, archivedValue{name, normalized, &set})
muVals.Unlock()
}
for _, v := range vals {
if err = c.c.AddPrecomputedValue(v.name, v.normalized, v.set); err != nil {
return err
}
}
return nil
}
// endOfLicenseText is text commonly associated with the end of a license. We
// can remove text that occurs after it.
var endOfLicenseText = []string{
"END OF TERMS AND CONDITIONS",
}
// TrimExtraneousTrailingText removes text after an obvious end of the license
// and does not include substantive text of the license.
func TrimExtraneousTrailingText(s string) string {
for _, e := range endOfLicenseText {
if i := strings.LastIndex(s, e); i != -1 {
return s[:i+len(e)]
}
}
return s
}
var copyrightRE = regexp.MustCompile(`(?m)(?i:Copyright)\s+(?i:©\s+|\(c\)\s+)?(?:\d{2,4})(?:[-,]\s*\d{2,4})*,?\s*(?i:by)?\s*(.*?(?i:\s+Inc\.)?)[.,]?\s*(?i:All rights reserved\.?)?\s*$`)
// CopyrightHolder finds a copyright notification, if it exists, and returns
// the copyright holder.
func CopyrightHolder(contents string) string {
matches := copyrightRE.FindStringSubmatch(contents)
if len(matches) == 2 {
return matches[1]
}
return ""
}
var publicDomainRE = regexp.MustCompile("(?i)(this file )?is( in the)? public domain")
// HasPublicDomainNotice performs a simple regex over the contents to see if a
// public domain notice is in there. As you can imagine, this isn't 100%
// definitive, but can be useful if a license match isn't found.
func (c *License) HasPublicDomainNotice(contents string) bool {
return publicDomainRE.FindString(contents) != ""
}
// ignorableTexts is a list of lines at the start of the string we can remove
// to get a cleaner match.
var ignorableTexts = []*regexp.Regexp{
regexp.MustCompile(`(?i)^(?:the )?mit license(?: \(mit\))?$`),
regexp.MustCompile(`(?i)^(?:new )?bsd license$`),
regexp.MustCompile(`(?i)^copyright and permission notice$`),
regexp.MustCompile(`(?i)^copyright (\(c\) )?(\[yyyy\]|\d{4})[,.]? .*$`),
regexp.MustCompile(`(?i)^(all|some) rights reserved\.?$`),
regexp.MustCompile(`(?i)^@license$`),
regexp.MustCompile(`^\s*$`),
}
// removeIgnorableTexts removes common text, which is not important for
// classification, that shows up before the body of the license.
func removeIgnorableTexts(s string) string {
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
var start int
for ; start < len(lines); start++ {
line := strings.TrimSpace(lines[start])
var matches bool
for _, re := range ignorableTexts {
if re.MatchString(line) {
matches = true
break
}
}
if !matches {
break
}
}
end := len(lines)
if start > end {
return "\n"
}
return strings.Join(lines[start:end], "\n") + "\n"
}
// removeShebangLine removes the '#!...' line if it's the first line in the
// file. Note that if it's the only line in a comment, it won't be removed.
func removeShebangLine(s string) string {
lines := strings.Split(s, "\n")
if len(lines) <= 1 || !strings.HasPrefix(lines[0], "#!") {
return s
}
return strings.Join(lines[1:], "\n")
}
// isDecorative returns true if the line is made up purely of non-letter and
// non-digit characters.
func isDecorative(s string) bool {
for _, c := range s {
if unicode.IsLetter(c) || unicode.IsDigit(c) {
return false
}
}
return true
}
var nonWords = regexp.MustCompile("[[:punct:]]+")
// RemoveNonWords removes non-words from the string.
func RemoveNonWords(s string) string {
return nonWords.ReplaceAllString(s, " ")
}
// interchangeablePunctutation is punctuation that can be normalized.
var interchangeablePunctuation = []struct {
interchangeable *regexp.Regexp
substitute string
}{
// Hyphen, Dash, En Dash, and Em Dash.
{regexp.MustCompile(`[-‒–—]`), "-"},
// Single, Double, Curly Single, and Curly Double.
{regexp.MustCompile("['\"`‘’“”]"), "'"},
// Copyright.
{regexp.MustCompile("©"), "(c)"},
// Hyphen-separated words.
{regexp.MustCompile(`(\S)-\s+(\S)`), "${1}-${2}"},
// Currency and Section. (Different copies of the CDDL use each marker.)
{regexp.MustCompile("[§¤]"), "(s)"},
// Middle Dot
{regexp.MustCompile("·"), "*"},
}
// NormalizePunctuation takes all hyphens and quotes and normalizes them.
func NormalizePunctuation(s string) string {
for _, iw := range interchangeablePunctuation {
s = iw.interchangeable.ReplaceAllString(s, iw.substitute)
}
return s
}
// interchangeableWords are words we can substitute for a normalized form
// without changing the meaning of the license. See
// https://spdx.org/spdx-license-list/matching-guidelines for the list.
var interchangeableWords = []struct {
interchangeable *regexp.Regexp
substitute string
}{
{regexp.MustCompile("(?i)Acknowledgment"), "Acknowledgement"},
{regexp.MustCompile("(?i)Analogue"), "Analog"},
{regexp.MustCompile("(?i)Analyse"), "Analyze"},
{regexp.MustCompile("(?i)Artefact"), "Artifact"},
{regexp.MustCompile("(?i)Authorisation"), "Authorization"},
{regexp.MustCompile("(?i)Authorised"), "Authorized"},
{regexp.MustCompile("(?i)Calibre"), "Caliber"},
{regexp.MustCompile("(?i)Cancelled"), "Canceled"},
{regexp.MustCompile("(?i)Capitalisations"), "Capitalizations"},
{regexp.MustCompile("(?i)Catalogue"), "Catalog"},
{regexp.MustCompile("(?i)Categorise"), "Categorize"},
{regexp.MustCompile("(?i)Centre"), "Center"},
{regexp.MustCompile("(?i)Emphasised"), "Emphasized"},
{regexp.MustCompile("(?i)Favour"), "Favor"},
{regexp.MustCompile("(?i)Favourite"), "Favorite"},
{regexp.MustCompile("(?i)Fulfil"), "Fulfill"},
{regexp.MustCompile("(?i)Fulfilment"), "Fulfillment"},
{regexp.MustCompile("(?i)Initialise"), "Initialize"},
{regexp.MustCompile("(?i)Judgment"), "Judgement"},
{regexp.MustCompile("(?i)Labelling"), "Labeling"},
{regexp.MustCompile("(?i)Labour"), "Labor"},
{regexp.MustCompile("(?i)Licence"), "License"},
{regexp.MustCompile("(?i)Maximise"), "Maximize"},
{regexp.MustCompile("(?i)Modelled"), "Modeled"},
{regexp.MustCompile("(?i)Modelling"), "Modeling"},
{regexp.MustCompile("(?i)Offence"), "Offense"},
{regexp.MustCompile("(?i)Optimise"), "Optimize"},
{regexp.MustCompile("(?i)Organisation"), "Organization"},
{regexp.MustCompile("(?i)Organise"), "Organize"},
{regexp.MustCompile("(?i)Practise"), "Practice"},
{regexp.MustCompile("(?i)Programme"), "Program"},
{regexp.MustCompile("(?i)Realise"), "Realize"},
{regexp.MustCompile("(?i)Recognise"), "Recognize"},
{regexp.MustCompile("(?i)Signalling"), "Signaling"},
{regexp.MustCompile("(?i)Sub[- ]license"), "Sublicense"},
{regexp.MustCompile("(?i)Utilisation"), "Utilization"},
{regexp.MustCompile("(?i)Whilst"), "While"},
{regexp.MustCompile("(?i)Wilful"), "Wilfull"},
{regexp.MustCompile("(?i)Non-commercial"), "Noncommercial"},
{regexp.MustCompile("(?i)Per cent"), "Percent"},
}
// NormalizeEquivalentWords normalizes equivalent words that are interchangeable.
func NormalizeEquivalentWords(s string) string {
for _, iw := range interchangeableWords {
s = iw.interchangeable.ReplaceAllString(s, iw.substitute)
}
return s
}

View File

@ -1,68 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package licenseclassifier
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"strings"
)
const (
// LicenseDirectory is the directory where the prototype licenses are kept.
LicenseDirectory = "src/github.com/google/licenseclassifier/licenses"
// LicenseArchive is the name of the archive containing preprocessed
// license texts.
LicenseArchive = "licenses.db"
// ForbiddenLicenseArchive is the name of the archive containing preprocessed
// forbidden license texts only.
ForbiddenLicenseArchive = "forbidden_licenses.db"
)
// lcRoot computes the location of the licenses data in the licenseclassifier source tree based on the location of this file.
func lcRoot() (string, error) {
_, filename, _, ok := runtime.Caller(0)
if !ok {
return "", fmt.Errorf("unable to compute path of licenseclassifier source")
}
// this file must be in the root of the package, or the relative paths will be wrong.
return filepath.Join(filepath.Dir(filename), "licenses"), nil
}
// ReadLicenseFile locates and reads the license archive file. Absolute paths are used unmodified. Relative paths are expected to be in the licenses directory of the licenseclassifier package.
func ReadLicenseFile(filename string) ([]byte, error) {
if strings.HasPrefix(filename, "/") {
return ioutil.ReadFile(filename)
}
root, err := lcRoot()
if err != nil {
return nil, fmt.Errorf("error locating licenses directory: %v", err)
}
return ioutil.ReadFile(filepath.Join(root, filename))
}
// ReadLicenseDir reads directory containing the license files.
func ReadLicenseDir() ([]os.FileInfo, error) {
root, err := lcRoot()
if err != nil {
return nil, fmt.Errorf("error locating licenses directory: %v", err)
}
return ioutil.ReadDir(root)
}

View File

@ -1,48 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package licenseclassifier
import "regexp"
var (
reCCBYNC = regexp.MustCompile(`(?i).*\bAttribution NonCommercial\b.*`)
reCCBYNCND = regexp.MustCompile(`(?i).*\bAttribution NonCommercial NoDerivs\b.*`)
reCCBYNCSA = regexp.MustCompile(`(?i).*\bAttribution NonCommercial ShareAlike\b.*`)
// forbiddenRegexps are regular expressions we expect to find in
// forbidden licenses. If we think we have a forbidden license but
// don't find the equivalent phrase, then it's probably just a
// misclassification.
forbiddenRegexps = map[string]*regexp.Regexp{
AGPL10: regexp.MustCompile(`(?i).*\bAFFERO GENERAL PUBLIC LICENSE\b.*`),
AGPL30: regexp.MustCompile(`(?i).*\bGNU AFFERO GENERAL PUBLIC LICENSE\b.*`),
CCBYNC10: reCCBYNC,
CCBYNC20: reCCBYNC,
CCBYNC25: reCCBYNC,
CCBYNC30: reCCBYNC,
CCBYNC40: reCCBYNC,
CCBYNCND10: regexp.MustCompile(`(?i).*\bAttribution NoDerivs NonCommercial\b.*`),
CCBYNCND20: reCCBYNCND,
CCBYNCND25: reCCBYNCND,
CCBYNCND30: reCCBYNCND,
CCBYNCND40: regexp.MustCompile(`(?i).*\bAttribution NonCommercial NoDerivatives\b.*`),
CCBYNCSA10: reCCBYNCSA,
CCBYNCSA20: reCCBYNCSA,
CCBYNCSA25: reCCBYNCSA,
CCBYNCSA30: reCCBYNCSA,
CCBYNCSA40: reCCBYNCSA,
WTFPL: regexp.MustCompile(`(?i).*\bDO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE\b.*`),
}
)

View File

@ -1,9 +0,0 @@
module github.com/google/licenseclassifier
go 1.11
require (
github.com/google/go-cmp v0.2.0
github.com/sergi/go-diff v1.0.0
github.com/stretchr/testify v1.3.0 // indirect
)

View File

@ -1,11 +0,0 @@
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=

View File

@ -1,20 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package sets provides sets for storing collections of unique elements.
package sets
// present is an empty struct used as the "value" in the map[int], since
// empty structs consume zero bytes (unlike 1 unnecessary byte per bool).
type present struct{}

View File

@ -1,228 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sets
import (
"fmt"
"sort"
"strings"
)
// StringSet stores a set of unique string elements.
type StringSet struct {
set map[string]present
}
// NewStringSet creates a StringSet containing the supplied initial string elements.
func NewStringSet(elements ...string) *StringSet {
s := &StringSet{}
s.set = make(map[string]present)
s.Insert(elements...)
return s
}
// Copy returns a newly allocated copy of the supplied StringSet.
func (s *StringSet) Copy() *StringSet {
c := NewStringSet()
if s != nil {
for e := range s.set {
c.set[e] = present{}
}
}
return c
}
// Insert zero or more string elements into the StringSet.
// As expected for a Set, elements already present in the StringSet are
// simply ignored.
func (s *StringSet) Insert(elements ...string) {
for _, e := range elements {
s.set[e] = present{}
}
}
// Delete zero or more string elements from the StringSet.
// Any elements not present in the StringSet are simply ignored.
func (s *StringSet) Delete(elements ...string) {
for _, e := range elements {
delete(s.set, e)
}
}
// Intersect returns a new StringSet containing the intersection of the
// receiver and argument StringSets. Returns an empty set if the argument is nil.
func (s *StringSet) Intersect(other *StringSet) *StringSet {
if other == nil {
return NewStringSet()
}
// Point a and b to the maps, setting a to the smaller of the two.
a, b := s.set, other.set
if len(b) < len(a) {
a, b = b, a
}
// Perform the intersection.
intersect := NewStringSet()
for e := range a {
if _, ok := b[e]; ok {
intersect.set[e] = present{}
}
}
return intersect
}
// Disjoint returns true if the intersection of the receiver and the argument
// StringSets is the empty set. Returns true if the argument is nil or either
// StringSet is the empty set.
func (s *StringSet) Disjoint(other *StringSet) bool {
if other == nil || len(other.set) == 0 || len(s.set) == 0 {
return true
}
// Point a and b to the maps, setting a to the smaller of the two.
a, b := s.set, other.set
if len(b) < len(a) {
a, b = b, a
}
// Check for non-empty intersection.
for e := range a {
if _, ok := b[e]; ok {
return false // Early-exit because intersecting.
}
}
return true
}
// Difference returns a new StringSet containing the elements in the receiver
// that are not present in the argument StringSet. Returns a copy of the
// receiver if the argument is nil.
func (s *StringSet) Difference(other *StringSet) *StringSet {
if other == nil {
return s.Copy()
}
// Insert only the elements in the receiver that are not present in the
// argument StringSet.
diff := NewStringSet()
for e := range s.set {
if _, ok := other.set[e]; !ok {
diff.set[e] = present{}
}
}
return diff
}
// Unique returns a new StringSet containing the elements in the receiver
// that are not present in the argument StringSet *and* the elements in the
// argument StringSet that are not in the receiver (which is the union of two
// disjoint sets). Returns a copy of the
// receiver if the argument is nil.
func (s *StringSet) Unique(other *StringSet) *StringSet {
if other == nil {
return s.Copy()
}
sNotInOther := s.Difference(other)
otherNotInS := other.Difference(s)
// Duplicate Union implementation here to avoid extra Copy, since both
// sNotInOther and otherNotInS are already copies.
unique := sNotInOther
for e := range otherNotInS.set {
unique.set[e] = present{}
}
return unique
}
// Equal returns true if the receiver and the argument StringSet contain
// exactly the same elements.
func (s *StringSet) Equal(other *StringSet) bool {
if s == nil || other == nil {
return s == nil && other == nil
}
// Two sets of different length cannot have the exact same unique elements.
if len(s.set) != len(other.set) {
return false
}
// Only one loop is needed. If the two sets are known to be of equal
// length, then the two sets are equal only if exactly all of the elements
// in the first set are found in the second.
for e := range s.set {
if _, ok := other.set[e]; !ok {
return false
}
}
return true
}
// Union returns a new StringSet containing the union of the receiver and
// argument StringSets. Returns a copy of the receiver if the argument is nil.
func (s *StringSet) Union(other *StringSet) *StringSet {
union := s.Copy()
if other != nil {
for e := range other.set {
union.set[e] = present{}
}
}
return union
}
// Contains returns true if element is in the StringSet.
func (s *StringSet) Contains(element string) bool {
_, in := s.set[element]
return in
}
// Len returns the number of unique elements in the StringSet.
func (s *StringSet) Len() int {
return len(s.set)
}
// Empty returns true if the receiver is the empty set.
func (s *StringSet) Empty() bool {
return len(s.set) == 0
}
// Elements returns a []string of the elements in the StringSet, in no
// particular (or consistent) order.
func (s *StringSet) Elements() []string {
elements := []string{} // Return at least an empty slice rather than nil.
for e := range s.set {
elements = append(elements, e)
}
return elements
}
// Sorted returns a sorted []string of the elements in the StringSet.
func (s *StringSet) Sorted() []string {
elements := s.Elements()
sort.Strings(elements)
return elements
}
// String formats the StringSet elements as sorted strings, representing them
// in "array initializer" syntax.
func (s *StringSet) String() string {
elements := s.Sorted()
var quoted []string
for _, e := range elements {
quoted = append(quoted, fmt.Sprintf("%q", e))
}
return fmt.Sprintf("{%s}", strings.Join(quoted, ", "))
}

View File

@ -1,394 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package licenseclassifier
// *** NOTE: Update this file when adding a new license. You need to:
//
// 1. Add the canonical name to the list, and
// 2. Categorize the license.
import "github.com/google/licenseclassifier/internal/sets"
// Canonical names of the licenses.
const (
// The names come from the https://spdx.org/licenses website, and are
// also the filenames of the licenses in licenseclassifier/licenses.
AFL11 = "AFL-1.1"
AFL12 = "AFL-1.2"
AFL20 = "AFL-2.0"
AFL21 = "AFL-2.1"
AFL30 = "AFL-3.0"
AGPL10 = "AGPL-1.0"
AGPL30 = "AGPL-3.0"
Apache10 = "Apache-1.0"
Apache11 = "Apache-1.1"
Apache20 = "Apache-2.0"
APSL10 = "APSL-1.0"
APSL11 = "APSL-1.1"
APSL12 = "APSL-1.2"
APSL20 = "APSL-2.0"
Artistic10cl8 = "Artistic-1.0-cl8"
Artistic10Perl = "Artistic-1.0-Perl"
Artistic10 = "Artistic-1.0"
Artistic20 = "Artistic-2.0"
BCL = "BCL"
Beerware = "Beerware"
BSD2ClauseFreeBSD = "BSD-2-Clause-FreeBSD"
BSD2ClauseNetBSD = "BSD-2-Clause-NetBSD"
BSD2Clause = "BSD-2-Clause"
BSD3ClauseAttribution = "BSD-3-Clause-Attribution"
BSD3ClauseClear = "BSD-3-Clause-Clear"
BSD3ClauseLBNL = "BSD-3-Clause-LBNL"
BSD3Clause = "BSD-3-Clause"
BSD4Clause = "BSD-4-Clause"
BSD4ClauseUC = "BSD-4-Clause-UC"
BSDProtection = "BSD-Protection"
BSL10 = "BSL-1.0"
CC010 = "CC0-1.0"
CCBY10 = "CC-BY-1.0"
CCBY20 = "CC-BY-2.0"
CCBY25 = "CC-BY-2.5"
CCBY30 = "CC-BY-3.0"
CCBY40 = "CC-BY-4.0"
CCBYNC10 = "CC-BY-NC-1.0"
CCBYNC20 = "CC-BY-NC-2.0"
CCBYNC25 = "CC-BY-NC-2.5"
CCBYNC30 = "CC-BY-NC-3.0"
CCBYNC40 = "CC-BY-NC-4.0"
CCBYNCND10 = "CC-BY-NC-ND-1.0"
CCBYNCND20 = "CC-BY-NC-ND-2.0"
CCBYNCND25 = "CC-BY-NC-ND-2.5"
CCBYNCND30 = "CC-BY-NC-ND-3.0"
CCBYNCND40 = "CC-BY-NC-ND-4.0"
CCBYNCSA10 = "CC-BY-NC-SA-1.0"
CCBYNCSA20 = "CC-BY-NC-SA-2.0"
CCBYNCSA25 = "CC-BY-NC-SA-2.5"
CCBYNCSA30 = "CC-BY-NC-SA-3.0"
CCBYNCSA40 = "CC-BY-NC-SA-4.0"
CCBYND10 = "CC-BY-ND-1.0"
CCBYND20 = "CC-BY-ND-2.0"
CCBYND25 = "CC-BY-ND-2.5"
CCBYND30 = "CC-BY-ND-3.0"
CCBYND40 = "CC-BY-ND-4.0"
CCBYSA10 = "CC-BY-SA-1.0"
CCBYSA20 = "CC-BY-SA-2.0"
CCBYSA25 = "CC-BY-SA-2.5"
CCBYSA30 = "CC-BY-SA-3.0"
CCBYSA40 = "CC-BY-SA-4.0"
CDDL10 = "CDDL-1.0"
CDDL11 = "CDDL-1.1"
CommonsClause = "Commons-Clause"
CPAL10 = "CPAL-1.0"
CPL10 = "CPL-1.0"
eGenix = "eGenix"
EPL10 = "EPL-1.0"
EPL20 = "EPL-2.0"
EUPL10 = "EUPL-1.0"
EUPL11 = "EUPL-1.1"
Facebook2Clause = "Facebook-2-Clause"
Facebook3Clause = "Facebook-3-Clause"
FacebookExamples = "Facebook-Examples"
FreeImage = "FreeImage"
FTL = "FTL"
GPL10 = "GPL-1.0"
GPL20 = "GPL-2.0"
GPL20withautoconfexception = "GPL-2.0-with-autoconf-exception"
GPL20withbisonexception = "GPL-2.0-with-bison-exception"
GPL20withclasspathexception = "GPL-2.0-with-classpath-exception"
GPL20withfontexception = "GPL-2.0-with-font-exception"
GPL20withGCCexception = "GPL-2.0-with-GCC-exception"
GPL30 = "GPL-3.0"
GPL30withautoconfexception = "GPL-3.0-with-autoconf-exception"
GPL30withGCCexception = "GPL-3.0-with-GCC-exception"
GUSTFont = "GUST-Font-License"
ImageMagick = "ImageMagick"
IPL10 = "IPL-1.0"
ISC = "ISC"
LGPL20 = "LGPL-2.0"
LGPL21 = "LGPL-2.1"
LGPL30 = "LGPL-3.0"
LGPLLR = "LGPLLR"
Libpng = "Libpng"
Lil10 = "Lil-1.0"
LinuxOpenIB = "Linux-OpenIB"
LPL102 = "LPL-1.02"
LPL10 = "LPL-1.0"
LPPL13c = "LPPL-1.3c"
MIT = "MIT"
MPL10 = "MPL-1.0"
MPL11 = "MPL-1.1"
MPL20 = "MPL-2.0"
MSPL = "MS-PL"
NCSA = "NCSA"
NPL10 = "NPL-1.0"
NPL11 = "NPL-1.1"
OFL11 = "OFL-1.1"
OpenSSL = "OpenSSL"
OpenVision = "OpenVision"
OSL10 = "OSL-1.0"
OSL11 = "OSL-1.1"
OSL20 = "OSL-2.0"
OSL21 = "OSL-2.1"
OSL30 = "OSL-3.0"
PHP301 = "PHP-3.01"
PHP30 = "PHP-3.0"
PIL = "PIL"
PostgreSQL = "PostgreSQL"
Python20complete = "Python-2.0-complete"
Python20 = "Python-2.0"
QPL10 = "QPL-1.0"
Ruby = "Ruby"
SGIB10 = "SGI-B-1.0"
SGIB11 = "SGI-B-1.1"
SGIB20 = "SGI-B-2.0"
SISSL12 = "SISSL-1.2"
SISSL = "SISSL"
Sleepycat = "Sleepycat"
UnicodeTOU = "Unicode-TOU"
UnicodeDFS2015 = "Unicode-DFS-2015"
UnicodeDFS2016 = "Unicode-DFS-2016"
Unlicense = "Unlicense"
UPL10 = "UPL-1.0"
W3C19980720 = "W3C-19980720"
W3C20150513 = "W3C-20150513"
W3C = "W3C"
WTFPL = "WTFPL"
X11 = "X11"
Xnet = "Xnet"
Zend20 = "Zend-2.0"
ZeroBSD = "0BSD"
ZlibAcknowledgement = "zlib-acknowledgement"
Zlib = "Zlib"
ZPL11 = "ZPL-1.1"
ZPL20 = "ZPL-2.0"
ZPL21 = "ZPL-2.1"
)
var (
// Licenses Categorized by Type
// restricted - Licenses in this category require mandatory source
// distribution if we ships a product that includes third-party code
// protected by such a license.
restrictedType = sets.NewStringSet(
BCL,
CCBYND10,
CCBYND20,
CCBYND25,
CCBYND30,
CCBYND40,
CCBYSA10,
CCBYSA20,
CCBYSA25,
CCBYSA30,
CCBYSA40,
GPL10,
GPL20,
GPL20withautoconfexception,
GPL20withbisonexception,
GPL20withclasspathexception,
GPL20withfontexception,
GPL20withGCCexception,
GPL30,
GPL30withautoconfexception,
GPL30withGCCexception,
LGPL20,
LGPL21,
LGPL30,
NPL10,
NPL11,
OSL10,
OSL11,
OSL20,
OSL21,
OSL30,
QPL10,
Sleepycat,
)
// reciprocal - These licenses allow usage of software made available
// under such licenses freely in *unmodified* form. If the third-party
// source code is modified in any way these modifications to the
// original third-party source code must be made available.
reciprocalType = sets.NewStringSet(
APSL10,
APSL11,
APSL12,
APSL20,
CDDL10,
CDDL11,
CPL10,
EPL10,
EPL20,
FreeImage,
IPL10,
MPL10,
MPL11,
MPL20,
Ruby,
)
// notice - These licenses contain few restrictions, allowing original
// or modified third-party software to be shipped in any product
// without endangering or encumbering our source code. All of the
// licenses in this category do, however, have an "original Copyright
// notice" or "advertising clause", wherein any external distributions
// must include the notice or clause specified in the license.
noticeType = sets.NewStringSet(
AFL11,
AFL12,
AFL20,
AFL21,
AFL30,
Apache10,
Apache11,
Apache20,
Artistic10cl8,
Artistic10Perl,
Artistic10,
Artistic20,
BSL10,
BSD2ClauseFreeBSD,
BSD2ClauseNetBSD,
BSD2Clause,
BSD3ClauseAttribution,
BSD3ClauseClear,
BSD3ClauseLBNL,
BSD3Clause,
BSD4Clause,
BSD4ClauseUC,
BSDProtection,
CCBY10,
CCBY20,
CCBY25,
CCBY30,
CCBY40,
FTL,
ISC,
ImageMagick,
Libpng,
Lil10,
LinuxOpenIB,
LPL102,
LPL10,
MSPL,
MIT,
NCSA,
OpenSSL,
PHP301,
PHP30,
PIL,
Python20,
Python20complete,
PostgreSQL,
SGIB10,
SGIB11,
SGIB20,
UnicodeDFS2015,
UnicodeDFS2016,
UnicodeTOU,
UPL10,
W3C19980720,
W3C20150513,
W3C,
X11,
Xnet,
Zend20,
ZlibAcknowledgement,
Zlib,
ZPL11,
ZPL20,
ZPL21,
)
// permissive - These licenses can be used in (relatively rare) cases
// where third-party software is under a license (not "Public Domain"
// or "free for any use" like 'unencumbered') that is even more lenient
// than a 'notice' license. Use the 'permissive' license type when even
// a copyright notice is not required for license compliance.
permissiveType = sets.NewStringSet()
// unencumbered - Licenses that basically declare that the code is "free for any use".
unencumberedType = sets.NewStringSet(
CC010,
Unlicense,
ZeroBSD,
)
// byexceptiononly - Licenses that are incompatible with all (or most)
// uses in combination with our source code. Commercial third-party
// packages that are purchased and licensed only for a specific use
// fall into this category.
byExceptionOnlyType = sets.NewStringSet(
Beerware,
OFL11,
OpenVision,
)
// forbidden - Licenses that are forbidden to be used.
forbiddenType = sets.NewStringSet(
AGPL10,
AGPL30,
CCBYNC10,
CCBYNC20,
CCBYNC25,
CCBYNC30,
CCBYNC40,
CCBYNCND10,
CCBYNCND20,
CCBYNCND25,
CCBYNCND30,
CCBYNCND40,
CCBYNCSA10,
CCBYNCSA20,
CCBYNCSA25,
CCBYNCSA30,
CCBYNCSA40,
CommonsClause,
Facebook2Clause,
Facebook3Clause,
FacebookExamples,
WTFPL,
)
// LicenseTypes is a set of the types of licenses Google recognizes.
LicenseTypes = sets.NewStringSet(
"restricted",
"reciprocal",
"notice",
"permissive",
"unencumbered",
"by_exception_only",
)
)
// LicenseType returns the type the license has.
func LicenseType(name string) string {
switch {
case restrictedType.Contains(name):
return "restricted"
case reciprocalType.Contains(name):
return "reciprocal"
case noticeType.Contains(name):
return "notice"
case permissiveType.Contains(name):
return "permissive"
case unencumberedType.Contains(name):
return "unencumbered"
case forbiddenType.Contains(name):
return "FORBIDDEN"
}
return ""
}

View File

@ -1,24 +0,0 @@
# How to contribute
We'd love to accept your patches and contributions to this project. There are
just a few small guidelines you need to follow.
## Contributor License Agreement
Contributions to this project must be accompanied by a Contributor License
Agreement. You (or your employer) retain the copyright to your contribution,
this simply gives us permission to use and redistribute your contributions as
part of the project. Head over to <https://cla.developers.google.com/> to see
your current agreements on file or to sign a new one.
You generally only need to submit a CLA once, so if you've already submitted
one (even if it was for a different project), you probably don't need to do it
again.
## Code reviews
All submissions, including submissions by project members, require review. We
use GitHub pull requests for this purpose. Consult [GitHub Help] for more
information on using pull requests.
[GitHub Help]: https://help.github.com/articles/about-pull-requests/

View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -1,65 +0,0 @@
# StringClassifier
StringClassifier is a library to classify an unknown text against a set of known
texts. The classifier uses the [Levenshtein Distance] algorithm to determine
which of the known texts most closely matches the unknown text. The Levenshtein
Distance is normalized into a "confidence percentage" between 1 and 0, where 1.0
indicates an exact match and 0.0 indicates a complete mismatch.
[Levenshtein Distance]: https://en.wikipedia.org/wiki/Levenshtein_distance
## Types of matching
There are two kinds of matching algorithms the string classifier can perform:
1. [Nearest matching](#nearest), and
2. [Multiple matching](#multiple).
### Normalization
To get the best match, normalizing functions can be applied to the texts. For
example, flattening whitespaces removes a lot of inconsequential formatting
differences that would otherwise lower the matching confidence percentage.
```go
sc := stringclassifier.New(stringclassifier.FlattenWhitespace, strings.ToLower)
```
The normalizating functions are run on all the known texts that are added to the
classifier. They're also run on the unknown text before classification.
### Nearest matching {#nearest}
A nearest match returns the name of the known text that most closely matches the
full unknown text. This is most useful when the unknown text doesn't have
extraneous text around it.
Example:
```go
func IdentifyText(sc *stringclassifier.Classifier, name, unknown string) {
m := sc.NearestMatch(unknown)
log.Printf("The nearest match to %q is %q (confidence: %v)", name, m.Name, m.Confidence)
}
```
## Multiple matching {#multiple}
Multiple matching identifies all of the known texts which may exist in the
unknown text. It can also detect a known text in an unknown text even if there's
extraneous text around the unknown text. As with nearest matching, a confidence
percentage for each match is given.
Example:
```go
log.Printf("The text %q contains:", name)
for _, m := range sc.MultipleMatch(unknown, false) {
log.Printf(" %q (conf: %v, offset: %v)", m.Name, m.Confidence, m.Offset)
}
```
## Disclaimer
This is not an official Google product (experimental or otherwise), it is just
code that happens to be owned by Google.

View File

@ -1,560 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package stringclassifier finds the nearest match between a string and a set of known values. It
// uses the Levenshtein Distance (LD) algorithm to determine this. A match with a large LD is less
// likely to be correct than one with a small LD. A confidence percentage is returned, which
// indicates how confident the algorithm is that the match is correct. The higher the percentage,
// the greater the confidence that the match is correct.
//
// Example Usage:
//
// type Text struct {
// Name string
// Text string
// }
//
// func NewClassifier(knownTexts []Text) (*stringclassifier.Classifier, error) {
// sc := stringclassifier.New(stringclassifier.FlattenWhitespace)
// for _, known := range knownTexts {
// if err := sc.AddValue(known.Name, known.Text); err != nil {
// return nil, err
// }
// }
// return sc, nil
// }
//
// func IdentifyTexts(sc *stringclassifier.Classifier, unknownTexts []*Text) {
// for _, unknown := range unknownTexts {
// m := sc.NearestMatch(unknown.Text)
// log.Printf("The nearest match to %q is %q (confidence: %v)",
// unknown.Name, m.Name, m.Confidence)
// }
// }
package stringclassifier
import (
"fmt"
"log"
"math"
"regexp"
"sort"
"sync"
"github.com/google/licenseclassifier/stringclassifier/internal/pq"
"github.com/google/licenseclassifier/stringclassifier/searchset"
"github.com/sergi/go-diff/diffmatchpatch"
)
// The diff/match/patch algorithm.
var dmp = diffmatchpatch.New()
const (
// DefaultConfidenceThreshold is the minimum ratio threshold between
// the matching range and the full source range that we're willing to
// accept in order to say that the matching range will produce a
// sufficiently good edit distance. I.e., if the matching range is
// below this threshold we won't run the Levenshtein Distance algorithm
// on it.
DefaultConfidenceThreshold float64 = 0.80
defaultMinDiffRatio float64 = 0.75
)
// A Classifier matches a string to a set of known values.
type Classifier struct {
muValues sync.RWMutex
values map[string]*knownValue
normalizers []NormalizeFunc
threshold float64
// MinDiffRatio defines the minimum ratio of the length difference
// allowed to consider a known value a possible match. This is used as
// a performance optimization to eliminate values that are unlikely to
// be a match.
//
// For example, a value of 0.75 means that the shorter string must be
// at least 75% the length of the longer string to consider it a
// possible match.
//
// Setting this to 1.0 will require that strings are identical length.
// Setting this to 0 will consider all known values as possible
// matches.
MinDiffRatio float64
}
// NormalizeFunc is a function that is used to normalize a string prior to comparison.
type NormalizeFunc func(string) string
// New creates a new Classifier with the provided NormalizeFuncs. Each
// NormalizeFunc is applied in order to a string before comparison.
func New(threshold float64, funcs ...NormalizeFunc) *Classifier {
return &Classifier{
values: make(map[string]*knownValue),
normalizers: append([]NormalizeFunc(nil), funcs...),
threshold: threshold,
MinDiffRatio: defaultMinDiffRatio,
}
}
// knownValue identifies a value in the corpus to match against.
type knownValue struct {
key string
normalizedValue string
reValue *regexp.Regexp
set *searchset.SearchSet
}
// AddValue adds a known value to be matched against. If a value already exists
// for key, an error is returned.
func (c *Classifier) AddValue(key, value string) error {
c.muValues.Lock()
defer c.muValues.Unlock()
if _, ok := c.values[key]; ok {
return fmt.Errorf("value already registered with key %q", key)
}
norm := c.normalize(value)
c.values[key] = &knownValue{
key: key,
normalizedValue: norm,
reValue: regexp.MustCompile(norm),
}
return nil
}
// AddPrecomputedValue adds a known value to be matched against. The value has
// already been normalized and the SearchSet object deserialized, so no
// processing is necessary.
func (c *Classifier) AddPrecomputedValue(key, value string, set *searchset.SearchSet) error {
c.muValues.Lock()
defer c.muValues.Unlock()
if _, ok := c.values[key]; ok {
return fmt.Errorf("value already registered with key %q", key)
}
set.GenerateNodeList()
c.values[key] = &knownValue{
key: key,
normalizedValue: value,
reValue: regexp.MustCompile(value),
set: set,
}
return nil
}
// normalize a string by applying each of the registered NormalizeFuncs.
func (c *Classifier) normalize(s string) string {
for _, fn := range c.normalizers {
s = fn(s)
}
return s
}
// Match identifies the result of matching a string against a knownValue.
type Match struct {
Name string // Name of knownValue that was matched
Confidence float64 // Confidence percentage
Offset int // The offset into the unknown string the match was made
Extent int // The length from the offset into the unknown string
}
// Matches is a list of Match-es. This is here mainly so that the list can be
// sorted.
type Matches []*Match
func (m Matches) Len() int { return len(m) }
func (m Matches) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
func (m Matches) Less(i, j int) bool {
if math.Abs(m[j].Confidence-m[i].Confidence) < math.SmallestNonzeroFloat64 {
if m[i].Name == m[j].Name {
if m[i].Offset > m[j].Offset {
return false
}
if m[i].Offset == m[j].Offset {
return m[i].Extent > m[j].Extent
}
return true
}
return m[i].Name < m[j].Name
}
return m[i].Confidence > m[j].Confidence
}
// Names returns an unsorted slice of the names of the matched licenses.
func (m Matches) Names() []string {
var names []string
for _, n := range m {
names = append(names, n.Name)
}
return names
}
// uniquify goes through the matches and removes any that are contained within
// one with a higher confidence. This assumes that Matches is sorted.
func (m Matches) uniquify() Matches {
type matchedRange struct {
offset, extent int
}
var matched []matchedRange
var matches Matches
OUTER:
for _, match := range m {
for _, mr := range matched {
if match.Offset >= mr.offset && match.Offset <= mr.offset+mr.extent {
continue OUTER
}
}
matched = append(matched, matchedRange{match.Offset, match.Extent})
matches = append(matches, match)
}
return matches
}
// NearestMatch returns the name of the known value that most closely matches
// the unknown string and a confidence percentage is returned indicating how
// confident the classifier is in the result. A percentage of "1.0" indicates
// an exact match, while a percentage of "0.0" indicates a complete mismatch.
//
// If the string is equidistant from multiple known values, it is undefined
// which will be returned.
func (c *Classifier) NearestMatch(s string) *Match {
pq := c.nearestMatch(s)
if pq.Len() == 0 {
return &Match{}
}
return pq.Pop().(*Match)
}
// MultipleMatch tries to determine which known strings are found within an
// unknown string. This differs from "NearestMatch" in that it looks only at
// those areas within the unknown string that are likely to match. A list of
// potential matches are returned. It's up to the caller to determine which
// ones are acceptable.
func (c *Classifier) MultipleMatch(s string) (matches Matches) {
pq := c.multipleMatch(s)
if pq == nil {
return matches
}
// A map to remove duplicate entries.
m := make(map[Match]bool)
for pq.Len() != 0 {
v := pq.Pop().(*Match)
if _, ok := m[*v]; !ok {
m[*v] = true
matches = append(matches, v)
}
}
sort.Sort(matches)
return matches.uniquify()
}
// possibleMatch identifies a known value and it's diffRatio to a given string.
type possibleMatch struct {
value *knownValue
diffRatio float64
}
// likelyMatches is a slice of possibleMatches that can be sorted by their
// diffRatio to a given string, such that the most likely matches (based on
// length) are at the beginning.
type likelyMatches []possibleMatch
func (m likelyMatches) Len() int { return len(m) }
func (m likelyMatches) Less(i, j int) bool { return m[i].diffRatio > m[j].diffRatio }
func (m likelyMatches) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
// nearestMatch returns a Queue of values that the unknown string may be. The
// values are compared via their Levenshtein Distance and ranked with the
// nearest match at the beginning.
func (c *Classifier) nearestMatch(unknown string) *pq.Queue {
var mu sync.Mutex // Protect the priority queue.
pq := pq.NewQueue(func(x, y interface{}) bool {
return x.(*Match).Confidence > y.(*Match).Confidence
}, nil)
unknown = c.normalize(unknown)
if len(unknown) == 0 {
return pq
}
c.muValues.RLock()
var likely likelyMatches
for _, v := range c.values {
dr := diffRatio(unknown, v.normalizedValue)
if dr < c.MinDiffRatio {
continue
}
if unknown == v.normalizedValue {
// We found an exact match.
pq.Push(&Match{Name: v.key, Confidence: 1.0, Offset: 0, Extent: len(unknown)})
c.muValues.RUnlock()
return pq
}
likely = append(likely, possibleMatch{value: v, diffRatio: dr})
}
c.muValues.RUnlock()
sort.Sort(likely)
var wg sync.WaitGroup
classifyString := func(name, unknown, known string) {
defer wg.Done()
diffs := dmp.DiffMain(unknown, known, true)
distance := dmp.DiffLevenshtein(diffs)
confidence := confidencePercentage(len(unknown), len(known), distance)
if confidence > 0.0 {
mu.Lock()
pq.Push(&Match{Name: name, Confidence: confidence, Offset: 0, Extent: len(unknown)})
mu.Unlock()
}
}
wg.Add(len(likely))
for _, known := range likely {
go classifyString(known.value.key, unknown, known.value.normalizedValue)
}
wg.Wait()
return pq
}
// matcher finds all potential matches of "known" in "unknown". The results are
// placed in "queue".
type matcher struct {
unknown *searchset.SearchSet
normUnknown string
threshold float64
mu sync.Mutex
queue *pq.Queue
}
// newMatcher creates a "matcher" object.
func newMatcher(unknown string, threshold float64) *matcher {
return &matcher{
unknown: searchset.New(unknown, searchset.DefaultGranularity),
normUnknown: unknown,
threshold: threshold,
queue: pq.NewQueue(func(x, y interface{}) bool {
return x.(*Match).Confidence > y.(*Match).Confidence
}, nil),
}
}
// findMatches takes a known text and finds all potential instances of it in
// the unknown text. The resulting matches can then filtered to determine which
// are the best matches.
func (m *matcher) findMatches(known *knownValue) {
var mrs []searchset.MatchRanges
if all := known.reValue.FindAllStringIndex(m.normUnknown, -1); all != nil {
// We found exact matches. Just use those!
for _, a := range all {
var start, end int
for i, tok := range m.unknown.Tokens {
if tok.Offset == a[0] {
start = i
} else if tok.Offset >= a[len(a)-1]-len(tok.Text) {
end = i
break
}
}
mrs = append(mrs, searchset.MatchRanges{{
SrcStart: 0,
SrcEnd: len(known.set.Tokens),
TargetStart: start,
TargetEnd: end + 1,
}})
}
} else {
// No exact match. Perform a more thorough match.
mrs = searchset.FindPotentialMatches(known.set, m.unknown)
}
var wg sync.WaitGroup
for _, mr := range mrs {
if !m.withinConfidenceThreshold(known.set, mr) {
continue
}
wg.Add(1)
go func(mr searchset.MatchRanges) {
start, end := mr.TargetRange(m.unknown)
conf := levDist(m.normUnknown[start:end], known.normalizedValue)
if conf > 0.0 {
m.mu.Lock()
m.queue.Push(&Match{Name: known.key, Confidence: conf, Offset: start, Extent: end - start})
m.mu.Unlock()
}
wg.Done()
}(mr)
}
wg.Wait()
}
// withinConfidenceThreshold returns the Confidence we have in the potential
// match. It does this by calculating the ratio of what's matching to the
// original known text.
func (m *matcher) withinConfidenceThreshold(known *searchset.SearchSet, mr searchset.MatchRanges) bool {
return float64(mr.Size())/float64(len(known.Tokens)) >= m.threshold
}
// multipleMatch returns a Queue of values that might be within the unknown
// string. The values are compared via their Levenshtein Distance and ranked
// with the nearest match at the beginning.
func (c *Classifier) multipleMatch(unknown string) *pq.Queue {
normUnknown := c.normalize(unknown)
if normUnknown == "" {
return nil
}
m := newMatcher(normUnknown, c.threshold)
c.muValues.RLock()
var kvals []*knownValue
for _, known := range c.values {
kvals = append(kvals, known)
}
c.muValues.RUnlock()
var wg sync.WaitGroup
wg.Add(len(kvals))
for _, known := range kvals {
go func(known *knownValue) {
if known.set == nil {
k := searchset.New(known.normalizedValue, searchset.DefaultGranularity)
c.muValues.Lock()
c.values[known.key].set = k
c.muValues.Unlock()
}
m.findMatches(known)
wg.Done()
}(known)
}
wg.Wait()
return m.queue
}
// levDist runs the Levenshtein Distance algorithm on the known and unknown
// texts to measure how well they match.
func levDist(unknown, known string) float64 {
if len(known) == 0 || len(unknown) == 0 {
log.Printf("Zero-sized texts in Levenshtein Distance algorithm: known==%d, unknown==%d", len(known), len(unknown))
return 0.0
}
// Calculate the differences between the potentially matching known
// text and the unknown text.
diffs := dmp.DiffMain(unknown, known, false)
end := diffRangeEnd(known, diffs)
// Now execute the Levenshtein Distance algorithm to see how much it
// does match.
distance := dmp.DiffLevenshtein(diffs[:end])
return confidencePercentage(unknownTextLength(unknown, diffs), len(known), distance)
}
// unknownTextLength returns the length of the unknown text based on the diff range.
func unknownTextLength(unknown string, diffs []diffmatchpatch.Diff) int {
last := len(diffs) - 1
for ; last >= 0; last-- {
if diffs[last].Type == diffmatchpatch.DiffEqual {
break
}
}
ulen := 0
for i := 0; i < last+1; i++ {
switch diffs[i].Type {
case diffmatchpatch.DiffEqual, diffmatchpatch.DiffDelete:
ulen += len(diffs[i].Text)
}
}
return ulen
}
// diffRangeEnd returns the end index for the "Diff" objects that constructs
// (or nearly constructs) the "known" value.
func diffRangeEnd(known string, diffs []diffmatchpatch.Diff) (end int) {
var seen string
for end = 0; end < len(diffs); end++ {
if seen == known {
// Once we've constructed the "known" value, then we've
// reached the point in the diff list where more
// "Diff"s would just make the Levenshtein Distance
// less valid. There shouldn't be further "DiffEqual"
// nodes, because there's nothing further to match in
// the "known" text.
break
}
switch diffs[end].Type {
case diffmatchpatch.DiffEqual, diffmatchpatch.DiffInsert:
seen += diffs[end].Text
}
}
return end
}
// confidencePercentage calculates how confident we are in the result of the
// match. A percentage of "1.0" means an identical match. A confidence of "0.0"
// means a complete mismatch.
func confidencePercentage(ulen, klen, distance int) float64 {
if ulen == 0 && klen == 0 {
return 1.0
}
if ulen == 0 || klen == 0 || (distance > ulen && distance > klen) {
return 0.0
}
return 1.0 - float64(distance)/float64(max(ulen, klen))
}
// diffRatio calculates the ratio of the length of s1 and s2, returned as a
// percentage of the length of the longer string. E.g., diffLength("abcd", "e")
// would return 0.25 because "e" is 25% of the size of "abcd". Comparing
// strings of equal length will return 1.
func diffRatio(s1, s2 string) float64 {
x, y := len(s1), len(s2)
if x == 0 && y == 0 {
// Both strings are zero length
return 1.0
}
if x < y {
return float64(x) / float64(y)
}
return float64(y) / float64(x)
}
func max(a, b int) int {
if a > b {
return a
}
return b
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
// wsRegexp is a regexp used to identify blocks of whitespace.
var wsRegexp = regexp.MustCompile(`\s+`)
// FlattenWhitespace will flatten contiguous blocks of whitespace down to a single space.
var FlattenWhitespace NormalizeFunc = func(s string) string {
return wsRegexp.ReplaceAllString(s, " ")
}

View File

@ -1,111 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package pq provides a priority queue.
package pq
import "container/heap"
// NewQueue returns an unbounded priority queue that compares elements using
// less; the minimal element is at the top of the queue.
//
// If setIndex is not nil, the queue calls setIndex to inform each element of
// its position in the queue. If an element's priority changes, its position in
// the queue may be incorrect. Call Fix on the element's index to update the
// queue. Call Remove on the element's index to remove it from the queue.
func NewQueue(less func(x, y interface{}) bool, setIndex func(x interface{}, idx int)) *Queue {
return &Queue{
heap: pqHeap{
less: less,
setIndex: setIndex,
},
}
}
// Queue is a priority queue that supports updating the priority of an element.
// A Queue must be created with NewQueue.
type Queue struct {
heap pqHeap
}
// Len returns the number of elements in the queue.
func (pq *Queue) Len() int {
return pq.heap.Len()
}
// Push adds x to the queue.
func (pq *Queue) Push(x interface{}) {
heap.Push(&pq.heap, x)
}
// Min returns the minimal element.
// Min panics if the queue is empty.
func (pq *Queue) Min() interface{} {
return pq.heap.a[0]
}
// Pop removes and returns the minimal element.
// Pop panics if the queue is empty.
func (pq *Queue) Pop() interface{} {
return heap.Pop(&pq.heap)
}
// Fix adjusts the heap to reflect that the element at index has changed priority.
func (pq *Queue) Fix(index int) {
heap.Fix(&pq.heap, index)
}
// Remove removes the element at index i from the heap.
func (pq *Queue) Remove(index int) {
heap.Remove(&pq.heap, index)
}
// pqHeap implements heap.Interface.
type pqHeap struct {
a []interface{}
less func(x, y interface{}) bool
setIndex func(x interface{}, idx int)
}
func (h pqHeap) Len() int {
return len(h.a)
}
func (h pqHeap) Less(i, j int) bool {
return h.less(h.a[i], h.a[j])
}
func (h pqHeap) Swap(i, j int) {
h.a[i], h.a[j] = h.a[j], h.a[i]
if h.setIndex != nil {
h.setIndex(h.a[i], i)
h.setIndex(h.a[j], j)
}
}
func (h *pqHeap) Push(x interface{}) {
n := len(h.a)
if h.setIndex != nil {
h.setIndex(x, n)
}
h.a = append(h.a, x)
}
func (h *pqHeap) Pop() interface{} {
old := h.a
n := len(old)
x := old[n-1]
h.a = old[:n-1]
return x
}

View File

@ -1,491 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package searchset generates hashes for all substrings of a text. Potential
// matches between two SearchSet objects can then be determined quickly.
// Generating the hashes can be expensive, so it's best to perform it once. If
// the text is part of a known corpus, then the SearchSet can be serialized and
// kept in an archive.
//
// Matching occurs by "mapping" ranges from the source text into the target
// text but still retaining the source order:
//
// SOURCE: |-----------------------------|
//
// TARGET: |*****************************************|
//
// MAP SOURCE SECTIONS ONTO TARGET IN SOURCE ORDER:
//
// S: |-[--]-----[---]------[----]------|
// / | \
// |---| |---------| |-------------|
// T: |*****************************************|
//
// Note that a single source range may match many different ranges in the
// target. The matching algorithm untangles these so that all matched ranges
// are in order with respect to the source ranges. This is especially important
// since the source text may occur more than once in the target text. The
// algorithm finds each potential occurrence of S in T and returns all as
// potential matched ranges.
package searchset
import (
"encoding/gob"
"fmt"
"io"
"sort"
"github.com/google/licenseclassifier/stringclassifier/searchset/tokenizer"
)
// DefaultGranularity is the minimum size (in words) of the hash chunks.
const DefaultGranularity = 3
// SearchSet is a set of substrings that have hashes associated with them,
// making it fast to search for potential matches.
type SearchSet struct {
// Tokens is a tokenized list of the original input string.
Tokens tokenizer.Tokens
// Hashes is a map of checksums to a range of tokens.
Hashes tokenizer.Hash
// Checksums is a list of checksums ordered from longest range to
// shortest.
Checksums []uint32
// ChecksumRanges are the token ranges for the above checksums.
ChecksumRanges tokenizer.TokenRanges
nodes []*node
}
// node consists of a range of tokens along with the checksum for those tokens.
type node struct {
checksum uint32
tokens *tokenizer.TokenRange
}
func (n *node) String() string {
return fmt.Sprintf("[%d:%d]", n.tokens.Start, n.tokens.End)
}
// New creates a new SearchSet object. It generates a hash for each substring of "s".
func New(s string, granularity int) *SearchSet {
toks := tokenizer.Tokenize(s)
// Start generating hash values for all substrings within the text.
h := make(tokenizer.Hash)
checksums, tokenRanges := toks.GenerateHashes(h, func(a, b int) int {
if a < b {
return a
}
return b
}(len(toks), granularity))
sset := &SearchSet{
Tokens: toks,
Hashes: h,
Checksums: checksums,
ChecksumRanges: tokenRanges,
}
sset.GenerateNodeList()
return sset
}
// GenerateNodeList creates a node list out of the search set.
func (s *SearchSet) GenerateNodeList() {
if len(s.Tokens) == 0 {
return
}
for i := 0; i < len(s.Checksums); i++ {
s.nodes = append(s.nodes, &node{
checksum: s.Checksums[i],
tokens: s.ChecksumRanges[i],
})
}
}
// Serialize emits the SearchSet out so that it can be recreated at a later
// time.
func (s *SearchSet) Serialize(w io.Writer) error {
return gob.NewEncoder(w).Encode(s)
}
// Deserialize reads a file with a serialized SearchSet in it and reconstructs it.
func Deserialize(r io.Reader, s *SearchSet) error {
if err := gob.NewDecoder(r).Decode(&s); err != nil {
return err
}
s.GenerateNodeList()
return nil
}
// MatchRange is the range within the source text that is a match to the range
// in the target text.
type MatchRange struct {
// Offsets into the source tokens.
SrcStart, SrcEnd int
// Offsets into the target tokens.
TargetStart, TargetEnd int
}
// in returns true if the start and end are enclosed in the match range.
func (m *MatchRange) in(start, end int) bool {
return start >= m.TargetStart && end <= m.TargetEnd
}
func (m *MatchRange) String() string {
return fmt.Sprintf("[%v, %v)->[%v, %v)", m.SrcStart, m.SrcEnd, m.TargetStart, m.TargetEnd)
}
// MatchRanges is a list of "MatchRange"s. The ranges are monotonically
// increasing in value and indicate a single potential occurrence of the source
// text in the target text.
type MatchRanges []*MatchRange
func (m MatchRanges) Len() int { return len(m) }
func (m MatchRanges) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
func (m MatchRanges) Less(i, j int) bool {
if m[i].TargetStart < m[j].TargetStart {
return true
}
return m[i].TargetStart == m[j].TargetStart && m[i].SrcStart < m[j].SrcStart
}
// TargetRange is the start and stop token offsets into the target text.
func (m MatchRanges) TargetRange(target *SearchSet) (start, end int) {
start = target.Tokens[m[0].TargetStart].Offset
end = target.Tokens[m[len(m)-1].TargetEnd-1].Offset + len(target.Tokens[m[len(m)-1].TargetEnd-1].Text)
return start, end
}
// Size is the number of source tokens that were matched.
func (m MatchRanges) Size() int {
sum := 0
for _, mr := range m {
sum += mr.SrcEnd - mr.SrcStart
}
return sum
}
// FindPotentialMatches returns the ranges in the target (unknown) text that
// are best potential matches to the source (known) text.
func FindPotentialMatches(src, target *SearchSet) []MatchRanges {
matchedRanges := getMatchedRanges(src, target)
if len(matchedRanges) == 0 {
return nil
}
// Cleanup the matching ranges so that we get the longest contiguous ranges.
for i := 0; i < len(matchedRanges); i++ {
matchedRanges[i] = coalesceMatchRanges(matchedRanges[i])
}
return matchedRanges
}
// getMatchedRanges finds the ranges in the target text that match the source
// text. There can be multiple occurrences of the source text within the target
// text. Each separate occurrence is an entry in the returned slice.
func getMatchedRanges(src, target *SearchSet) []MatchRanges {
matched := targetMatchedRanges(src, target)
if len(matched) == 0 {
return nil
}
sort.Sort(matched)
matched = untangleSourceRanges(matched)
matchedRanges := splitRanges(matched)
return mergeConsecutiveRanges(matchedRanges)
}
func extendsAny(tr tokenizer.TokenRanges, mr []MatchRanges) bool {
if len(mr) == 0 {
return false
}
for _, tv := range tr {
for _, mv := range mr {
if tv.Start >= mv[0].TargetStart && tv.Start <= mv[len(mv)-1].TargetEnd {
return true
}
}
}
return false
}
// targetMatchedRanges finds matching sequences in target and src ordered by target position
func targetMatchedRanges(src, target *SearchSet) MatchRanges {
if src.nodes == nil {
return nil
}
var matched MatchRanges
var previous *node
var possible []MatchRanges
for _, tgtNode := range target.nodes {
sr, ok := src.Hashes[tgtNode.checksum]
if !ok || (previous != nil && tgtNode.tokens.Start > previous.tokens.End) || !extendsAny(sr, possible) {
for _, r := range possible {
matched = append(matched, r...)
}
possible = possible[:0]
previous = nil
}
if !ok {
// There isn't a match in the source.
continue
}
// Maps index within `possible` to the slice of ranges extended by a new range
extended := make(map[int]*MatchRanges)
// Go over the set of source ranges growing lists of `possible` match ranges.
tv := tgtNode.tokens
for _, sv := range sr {
r := &MatchRange{
SrcStart: sv.Start,
SrcEnd: sv.End,
TargetStart: tv.Start,
TargetEnd: tv.End,
}
found := false
// Grow or extend each abutting `possible` match range.
for i, p := range possible {
last := p[len(p)-1]
if sv.Start >= last.SrcStart && sv.Start <= last.SrcEnd && tv.Start >= last.TargetStart && tv.Start <= last.TargetEnd {
found = true
possible[i] = append(possible[i], r)
extended[i] = &possible[i]
}
}
if !found {
// Did not abut any existing ranges, start a new `possible` match range.
mrs := make(MatchRanges, 0, 2)
mrs = append(mrs, r)
possible = append(possible, mrs)
extended[len(possible)-1] = &possible[len(possible)-1]
}
}
if len(extended) < len(possible) {
// Ranges not extended--add to `matched` if not included in other range.
for i := 0; i < len(possible); {
_, updated := extended[i]
if updated {
i++ // Keep in `possible` and advance to next index.
continue
}
p1 := possible[i]
found := false // whether found as subrange of another `possible` match.
for _, p2 := range extended {
if p1[0].SrcStart >= (*p2)[0].SrcStart && p1[0].TargetStart >= (*p2)[0].TargetStart {
found = true
break
}
}
if !found {
matched = append(matched, p1...)
} // else included in other match.
// Finished -- delete from `possible` and continue from same index.
possible = append(possible[:i], possible[i+1:]...)
}
}
previous = tgtNode
}
// At end of file, terminate all `possible` match ranges.
for i := 0; i < len(possible); i++ {
p1 := possible[i]
found := false // whether found as subrange of another `possible` match.
for j := i + 1; j < len(possible); {
p2 := possible[j]
if p1[0].SrcStart <= p2[0].SrcStart && p1[0].TargetStart <= p2[0].TargetStart {
// Delete later sub-ranges included in this range.
possible = append(possible[:j], possible[j+1:]...)
continue
}
// Skip if subrange of a later range
if p1[0].SrcStart >= p2[0].SrcStart && p1[0].TargetStart >= p2[0].TargetStart {
found = true
}
j++
}
if !found {
matched = append(matched, p1...)
}
}
return matched
}
// untangleSourceRanges goes through the ranges and removes any whose source
// ranges are "out of order". A source range is "out of order" if the source
// range is out of sequence with the source ranges before and after it. This
// happens when more than one source range maps to the same target range.
// E.g.:
//
// SrcStart: 20, SrcEnd: 30, TargetStart: 127, TargetEnd: 137
// 1: SrcStart: 12, SrcEnd: 17, TargetStart: 138, TargetEnd: 143
// 2: SrcStart: 32, SrcEnd: 37, TargetStart: 138, TargetEnd: 143
// SrcStart: 38, SrcEnd: 40, TargetStart: 144, TargetEnd: 146
//
// Here (1) is out of order, because the source range [12, 17) is out of
// sequence with the surrounding source sequences, but [32, 37) is.
func untangleSourceRanges(matched MatchRanges) MatchRanges {
mr := MatchRanges{matched[0]}
NEXT:
for i := 1; i < len(matched); i++ {
if mr[len(mr)-1].TargetStart == matched[i].TargetStart && mr[len(mr)-1].TargetEnd == matched[i].TargetEnd {
// The matched range has already been added.
continue
}
if i+1 < len(matched) && equalTargetRange(matched[i], matched[i+1]) {
// A sequence of ranges match the same target range.
// Find the first one that has a source range greater
// than the currently matched range. Omit all others.
if matched[i].SrcStart > mr[len(mr)-1].SrcStart {
mr = append(mr, matched[i])
continue
}
for j := i + 1; j < len(matched) && equalTargetRange(matched[i], matched[j]); j++ {
// Check subsequent ranges to see if we can
// find one that matches in the correct order.
if matched[j].SrcStart > mr[len(mr)-1].SrcStart {
mr = append(mr, matched[j])
i = j
continue NEXT
}
}
}
mr = append(mr, matched[i])
}
return mr
}
// equalTargetRange returns true if the two MatchRange's cover the same target range.
func equalTargetRange(this, that *MatchRange) bool {
return this.TargetStart == that.TargetStart && this.TargetEnd == that.TargetEnd
}
// splitRanges splits the matched ranges so that a single match range has a
// monotonically increasing source range (indicating a single, potential
// instance of the source in the target).
func splitRanges(matched MatchRanges) []MatchRanges {
var matchedRanges []MatchRanges
mr := MatchRanges{matched[0]}
for i := 1; i < len(matched); i++ {
if mr[len(mr)-1].SrcStart > matched[i].SrcStart {
matchedRanges = append(matchedRanges, mr)
mr = MatchRanges{matched[i]}
} else {
mr = append(mr, matched[i])
}
}
matchedRanges = append(matchedRanges, mr)
return matchedRanges
}
// mergeConsecutiveRanges goes through the matched ranges and merges
// consecutive ranges. Two ranges are consecutive if the end of the previous
// matched range and beginning of the next matched range overlap. "matched"
// should have 1 or more MatchRanges, each with one or more MatchRange objects.
func mergeConsecutiveRanges(matched []MatchRanges) []MatchRanges {
mr := []MatchRanges{matched[0]}
// Convenience functions.
prevMatchedRange := func() MatchRanges {
return mr[len(mr)-1]
}
prevMatchedRangeLastElem := func() *MatchRange {
return prevMatchedRange()[len(prevMatchedRange())-1]
}
// This algorithm compares the start of each MatchRanges object to the
// end of the previous MatchRanges object. If they overlap, then it
// tries to combine them. Note that a 0 offset into a MatchRanges
// object (e.g., matched[i][0]) is its first MatchRange, which
// indicates the start of the whole matched range.
NEXT:
for i := 1; i < len(matched); i++ {
if prevMatchedRangeLastElem().TargetEnd > matched[i][0].TargetStart {
// Consecutive matched ranges overlap. Merge them.
if prevMatchedRangeLastElem().TargetStart < matched[i][0].TargetStart {
// The last element of the previous matched
// range overlaps with the first element of the
// current matched range. Concatenate them.
if prevMatchedRangeLastElem().TargetEnd < matched[i][0].TargetEnd {
prevMatchedRangeLastElem().SrcEnd += matched[i][0].TargetEnd - prevMatchedRangeLastElem().TargetEnd
prevMatchedRangeLastElem().TargetEnd = matched[i][0].TargetEnd
}
mr[len(mr)-1] = append(prevMatchedRange(), matched[i][1:]...)
continue
}
for j := 1; j < len(matched[i]); j++ {
// Find the positions in the ranges where the
// tail end of the previous matched range
// overlaps with the start of the next matched
// range.
for k := len(prevMatchedRange()) - 1; k > 0; k-- {
if prevMatchedRange()[k].SrcStart < matched[i][j].SrcStart &&
prevMatchedRange()[k].TargetStart < matched[i][j].TargetStart {
// Append the next range to the previous range.
if prevMatchedRange()[k].TargetEnd < matched[i][j].TargetStart {
// Coalesce the ranges.
prevMatchedRange()[k].SrcEnd += matched[i][j-1].TargetEnd - prevMatchedRange()[k].TargetEnd
prevMatchedRange()[k].TargetEnd = matched[i][j-1].TargetEnd
}
mr[len(mr)-1] = append(prevMatchedRange()[:k+1], matched[i][j:]...)
continue NEXT
}
}
}
}
mr = append(mr, matched[i])
}
return mr
}
// coalesceMatchRanges coalesces overlapping match ranges into a single
// contiguous match range.
func coalesceMatchRanges(matchedRanges MatchRanges) MatchRanges {
coalesced := MatchRanges{matchedRanges[0]}
for i := 1; i < len(matchedRanges); i++ {
c := coalesced[len(coalesced)-1]
mr := matchedRanges[i]
if mr.SrcStart <= c.SrcEnd && mr.SrcStart >= c.SrcStart {
var se, ts, te int
if mr.SrcEnd > c.SrcEnd {
se = mr.SrcEnd
} else {
se = c.SrcEnd
}
if mr.TargetStart < c.TargetStart {
ts = mr.TargetStart
} else {
ts = c.TargetStart
}
if mr.TargetEnd > c.TargetEnd {
te = mr.TargetEnd
} else {
te = c.TargetEnd
}
coalesced[len(coalesced)-1] = &MatchRange{
SrcStart: c.SrcStart,
SrcEnd: se,
TargetStart: ts,
TargetEnd: te,
}
} else {
coalesced = append(coalesced, mr)
}
}
return coalesced
}

View File

@ -1,175 +0,0 @@
// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package tokenizer converts a text into a stream of tokens.
package tokenizer
import (
"bytes"
"fmt"
"hash/crc32"
"sort"
"unicode"
"unicode/utf8"
)
// Token is a non-whitespace sequence (i.e., word or punctuation) in the
// original string. This is not meant for use outside of this package.
type token struct {
Text string
Offset int
}
// Tokens is a list of Token objects.
type Tokens []*token
// newToken creates a new token object with an invalid (negative) offset, which
// will be set before the token's used.
func newToken() *token {
return &token{Offset: -1}
}
// Tokenize converts a string into a stream of tokens.
func Tokenize(s string) (toks Tokens) {
tok := newToken()
for i := 0; i < len(s); {
r, size := utf8.DecodeRuneInString(s[i:])
switch {
case unicode.IsSpace(r):
if tok.Offset >= 0 {
toks = append(toks, tok)
tok = newToken()
}
case unicode.IsPunct(r):
if tok.Offset >= 0 {
toks = append(toks, tok)
tok = newToken()
}
toks = append(toks, &token{
Text: string(r),
Offset: i,
})
default:
if tok.Offset == -1 {
tok.Offset = i
}
tok.Text += string(r)
}
i += size
}
if tok.Offset != -1 {
// Add any remaining token that wasn't yet included in the list.
toks = append(toks, tok)
}
return toks
}
// GenerateHashes generates hashes for "size" length substrings. The
// "stringifyTokens" call takes a long time to run, so not all substrings have
// hashes, i.e. we skip some of the smaller substrings.
func (t Tokens) GenerateHashes(h Hash, size int) ([]uint32, TokenRanges) {
if size == 0 {
return nil, nil
}
var css []uint32
var tr TokenRanges
for offset := 0; offset+size <= len(t); offset += size / 2 {
var b bytes.Buffer
t.stringifyTokens(&b, offset, size)
cs := crc32.ChecksumIEEE(b.Bytes())
css = append(css, cs)
tr = append(tr, &TokenRange{offset, offset + size})
h.add(cs, offset, offset+size)
if size <= 1 {
break
}
}
return css, tr
}
// stringifyTokens serializes a sublist of tokens into a bytes buffer.
func (t Tokens) stringifyTokens(b *bytes.Buffer, offset, size int) {
for j := offset; j < offset+size; j++ {
if j != offset {
b.WriteRune(' ')
}
b.WriteString(t[j].Text)
}
}
// TokenRange indicates the range of tokens that map to a particular checksum.
type TokenRange struct {
Start int
End int
}
func (t *TokenRange) String() string {
return fmt.Sprintf("[%v, %v)", t.Start, t.End)
}
// TokenRanges is a list of TokenRange objects. The chance that two different
// strings map to the same checksum is very small, but unfortunately isn't
// zero, so we use this instead of making the assumption that they will all be
// unique.
type TokenRanges []*TokenRange
func (t TokenRanges) Len() int { return len(t) }
func (t TokenRanges) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t TokenRanges) Less(i, j int) bool { return t[i].Start < t[j].Start }
// CombineUnique returns the combination of both token ranges with no duplicates.
func (t TokenRanges) CombineUnique(other TokenRanges) TokenRanges {
if len(other) == 0 {
return t
}
if len(t) == 0 {
return other
}
cu := append(t, other...)
sort.Sort(cu)
if len(cu) == 0 {
return nil
}
res := TokenRanges{cu[0]}
for prev, i := cu[0], 1; i < len(cu); i++ {
if prev.Start != cu[i].Start || prev.End != cu[i].End {
res = append(res, cu[i])
prev = cu[i]
}
}
return res
}
// Hash is a map of the hashes of a section of text to the token range covering that text.
type Hash map[uint32]TokenRanges
// add associates a token range, [start, end], to a checksum.
func (h Hash) add(checksum uint32, start, end int) {
ntr := &TokenRange{Start: start, End: end}
if r, ok := h[checksum]; ok {
for _, tr := range r {
if tr.Start == ntr.Start && tr.End == ntr.End {
// The token range already exists at this
// checksum. No need to re-add it.
return
}
}
}
h[checksum] = append(h[checksum], ntr)
}

View File

@ -1,12 +0,0 @@
version = 1
test_patterns = [
"*_test.go"
]
[[analyzers]]
name = "go"
enabled = true
[analyzers.meta]
import_path = "github.com/imdario/mergo"

View File

@ -4,6 +4,4 @@ install:
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
script:
- go test -race -v ./...
after_script:
- $HOME/gopath/bin/goveralls -service=travis-ci -repotoken $COVERALLS_TOKEN

View File

@ -13,7 +13,6 @@ It is ready for production use. [It is used in several projects by Docker, Googl
[![Build Status][1]][2]
[![Coverage Status][7]][8]
[![Sourcegraph][9]][10]
[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield)
[1]: https://travis-ci.org/imdario/mergo.png
[2]: https://travis-ci.org/imdario/mergo
@ -28,7 +27,7 @@ It is ready for production use. [It is used in several projects by Docker, Googl
### Latest release
[Release v0.3.7](https://github.com/imdario/mergo/releases/tag/v0.3.7).
[Release v0.3.4](https://github.com/imdario/mergo/releases/tag/v0.3.4).
### Important note
@ -218,21 +217,6 @@ If I can help you, you have an idea or you are using Mergo in your projects, don
Written by [Dario Castañé](http://dario.im).
## Top Contributors
[![0](https://sourcerer.io/fame/imdario/imdario/mergo/images/0)](https://sourcerer.io/fame/imdario/imdario/mergo/links/0)
[![1](https://sourcerer.io/fame/imdario/imdario/mergo/images/1)](https://sourcerer.io/fame/imdario/imdario/mergo/links/1)
[![2](https://sourcerer.io/fame/imdario/imdario/mergo/images/2)](https://sourcerer.io/fame/imdario/imdario/mergo/links/2)
[![3](https://sourcerer.io/fame/imdario/imdario/mergo/images/3)](https://sourcerer.io/fame/imdario/imdario/mergo/links/3)
[![4](https://sourcerer.io/fame/imdario/imdario/mergo/images/4)](https://sourcerer.io/fame/imdario/imdario/mergo/links/4)
[![5](https://sourcerer.io/fame/imdario/imdario/mergo/images/5)](https://sourcerer.io/fame/imdario/imdario/mergo/links/5)
[![6](https://sourcerer.io/fame/imdario/imdario/mergo/images/6)](https://sourcerer.io/fame/imdario/imdario/mergo/links/6)
[![7](https://sourcerer.io/fame/imdario/imdario/mergo/images/7)](https://sourcerer.io/fame/imdario/imdario/mergo/links/7)
## License
[BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE).
[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_large)

View File

@ -72,7 +72,6 @@ func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, conf
case reflect.Struct:
srcMap := src.Interface().(map[string]interface{})
for key := range srcMap {
config.overwriteWithEmptyValue = true
srcValue := srcMap[key]
fieldName := changeInitialCase(key, unicode.ToUpper)
dstElement := dst.FieldByName(fieldName)
@ -99,11 +98,11 @@ func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, conf
continue
}
if srcKind == dstKind {
if _, err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
return
}
} else if dstKind == reflect.Interface && dstElement.Kind() == reflect.Interface {
if _, err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
return
}
} else if srcKind == reflect.Map {
@ -157,8 +156,7 @@ func _map(dst, src interface{}, opts ...func(*Config)) error {
// To be friction-less, we redirect equal-type arguments
// to deepMerge. Only because arguments can be anything.
if vSrc.Kind() == vDst.Kind() {
_, err := deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
return err
return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
}
switch vSrc.Kind() {
case reflect.Struct:

View File

@ -9,41 +9,25 @@
package mergo
import (
"fmt"
"reflect"
"unsafe"
)
func hasExportedField(dst reflect.Value) (exported bool) {
for i, n := 0, dst.NumField(); i < n; i++ {
field := dst.Type().Field(i)
if isExportedComponent(&field) {
return true
if field.Anonymous && dst.Field(i).Kind() == reflect.Struct {
exported = exported || hasExportedField(dst.Field(i))
} else {
exported = exported || len(field.PkgPath) == 0
}
}
return
}
func isExportedComponent(field *reflect.StructField) bool {
name := field.Name
pkgPath := field.PkgPath
if len(pkgPath) > 0 {
return false
}
c := name[0]
if 'a' <= c && c <= 'z' || c == '_' {
return false
}
return true
}
type Config struct {
Overwrite bool
AppendSlice bool
TypeCheck bool
Transformers Transformers
overwriteWithEmptyValue bool
overwriteSliceWithEmptyValue bool
Overwrite bool
AppendSlice bool
Transformers Transformers
}
type Transformers interface {
@ -53,17 +37,12 @@ type Transformers interface {
// Traverses recursively both values, assigning src's fields values to dst.
// The map argument tracks comparisons that have already been seen, which allows
// short circuiting on recursive types.
func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (dst reflect.Value, err error) {
dst = dstIn
func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) {
overwrite := config.Overwrite
typeCheck := config.TypeCheck
overwriteWithEmptySrc := config.overwriteWithEmptyValue
overwriteSliceWithEmptySrc := config.overwriteSliceWithEmptyValue
if !src.IsValid() {
return
}
if dst.CanAddr() {
addr := dst.UnsafeAddr()
h := 17 * addr
@ -71,7 +50,7 @@ func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int,
typ := dst.Type()
for p := seen; p != nil; p = p.next {
if p.ptr == addr && p.typ == typ {
return dst, nil
return nil
}
}
// Remember, remember...
@ -85,170 +64,130 @@ func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int,
}
}
if dst.IsValid() && src.IsValid() && src.Type() != dst.Type() {
err = fmt.Errorf("cannot append two different types (%s, %s)", src.Kind(), dst.Kind())
return
}
switch dst.Kind() {
case reflect.Struct:
if hasExportedField(dst) {
dstCp := reflect.New(dst.Type()).Elem()
for i, n := 0, dst.NumField(); i < n; i++ {
dstField := dst.Field(i)
structField := dst.Type().Field(i)
// copy un-exported struct fields
if !isExportedComponent(&structField) {
rf := dstCp.Field(i)
rf = reflect.NewAt(rf.Type(), unsafe.Pointer(rf.UnsafeAddr())).Elem() //nolint:gosec
dstRF := dst.Field(i)
if !dst.Field(i).CanAddr() {
continue
}
dstRF = reflect.NewAt(dstRF.Type(), unsafe.Pointer(dstRF.UnsafeAddr())).Elem() //nolint:gosec
rf.Set(dstRF)
continue
}
dstField, err = deepMerge(dstField, src.Field(i), visited, depth+1, config)
if err != nil {
if err = deepMerge(dst.Field(i), src.Field(i), visited, depth+1, config); err != nil {
return
}
dstCp.Field(i).Set(dstField)
}
if dst.CanSet() {
dst.Set(dstCp)
} else {
dst = dstCp
}
return
} else {
if (isReflectNil(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc) {
dst = src
if dst.CanSet() && !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) {
dst.Set(src)
}
}
case reflect.Map:
if dst.IsNil() && !src.IsNil() {
if dst.CanSet() {
dst.Set(reflect.MakeMap(dst.Type()))
} else {
dst = src
return
}
dst.Set(reflect.MakeMap(dst.Type()))
}
for _, key := range src.MapKeys() {
srcElement := src.MapIndex(key)
dstElement := dst.MapIndex(key)
if !srcElement.IsValid() {
continue
}
if dst.MapIndex(key).IsValid() {
k := dstElement.Interface()
dstElement = reflect.ValueOf(k)
}
if isReflectNil(srcElement) {
if overwrite || isReflectNil(dstElement) {
dst.SetMapIndex(key, srcElement)
dstElement := dst.MapIndex(key)
switch srcElement.Kind() {
case reflect.Chan, reflect.Func, reflect.Map, reflect.Interface, reflect.Slice:
if srcElement.IsNil() {
continue
}
fallthrough
default:
if !srcElement.CanInterface() {
continue
}
switch reflect.TypeOf(srcElement.Interface()).Kind() {
case reflect.Struct:
fallthrough
case reflect.Ptr:
fallthrough
case reflect.Map:
srcMapElm := srcElement
dstMapElm := dstElement
if srcMapElm.CanInterface() {
srcMapElm = reflect.ValueOf(srcMapElm.Interface())
if dstMapElm.IsValid() {
dstMapElm = reflect.ValueOf(dstMapElm.Interface())
}
}
if err = deepMerge(dstMapElm, srcMapElm, visited, depth+1, config); err != nil {
return
}
case reflect.Slice:
srcSlice := reflect.ValueOf(srcElement.Interface())
var dstSlice reflect.Value
if !dstElement.IsValid() || dstElement.IsNil() {
dstSlice = reflect.MakeSlice(srcSlice.Type(), 0, srcSlice.Len())
} else {
dstSlice = reflect.ValueOf(dstElement.Interface())
}
if !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
dstSlice = srcSlice
} else if config.AppendSlice {
dstSlice = reflect.AppendSlice(dstSlice, srcSlice)
}
dst.SetMapIndex(key, dstSlice)
}
continue
}
if !srcElement.CanInterface() {
if dstElement.IsValid() && reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Map {
continue
}
if srcElement.CanInterface() {
srcElement = reflect.ValueOf(srcElement.Interface())
if dstElement.IsValid() {
dstElement = reflect.ValueOf(dstElement.Interface())
if srcElement.IsValid() && (overwrite || (!dstElement.IsValid() || isEmptyValue(dstElement))) {
if dst.IsNil() {
dst.Set(reflect.MakeMap(dst.Type()))
}
dst.SetMapIndex(key, srcElement)
}
dstElement, err = deepMerge(dstElement, srcElement, visited, depth+1, config)
if err != nil {
return
}
dst.SetMapIndex(key, dstElement)
}
case reflect.Slice:
newSlice := dst
if (!isEmptyValue(src) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
if typeCheck && src.Type() != dst.Type() {
return dst, fmt.Errorf("cannot override two slices with different type (%s, %s)", src.Type(), dst.Type())
}
newSlice = src
if !dst.CanSet() {
break
}
if !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
dst.Set(src)
} else if config.AppendSlice {
if typeCheck && src.Type() != dst.Type() {
err = fmt.Errorf("cannot append two slice with different type (%s, %s)", src.Type(), dst.Type())
return
}
newSlice = reflect.AppendSlice(dst, src)
dst.Set(reflect.AppendSlice(dst, src))
}
if dst.CanSet() {
dst.Set(newSlice)
} else {
dst = newSlice
}
case reflect.Ptr, reflect.Interface:
if isReflectNil(src) {
case reflect.Ptr:
fallthrough
case reflect.Interface:
if src.IsNil() {
break
}
if dst.Kind() != reflect.Ptr && src.Type().AssignableTo(dst.Type()) {
if dst.IsNil() || overwrite {
if overwrite || isEmptyValue(dst) {
if dst.CanSet() {
dst.Set(src)
} else {
dst = src
}
}
}
break
}
if src.Kind() != reflect.Interface {
if dst.IsNil() || (src.Kind() != reflect.Ptr && overwrite) {
if dst.IsNil() || overwrite {
if dst.CanSet() && (overwrite || isEmptyValue(dst)) {
dst.Set(src)
}
} else if src.Kind() == reflect.Ptr {
if dst, err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
return
}
dst = dst.Addr()
} else if dst.Elem().Type() == src.Type() {
if dst, err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil {
if err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil {
return
}
} else {
return dst, ErrDifferentArgumentsTypes
return ErrDifferentArgumentsTypes
}
break
}
if dst.IsNil() || overwrite {
if (overwrite || isEmptyValue(dst)) && (overwriteWithEmptySrc || !isEmptyValue(src)) {
if dst.CanSet() {
dst.Set(src)
} else {
dst = src
}
if dst.CanSet() && (overwrite || isEmptyValue(dst)) {
dst.Set(src)
}
} else if _, err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
} else if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
return
}
default:
overwriteFull := (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst))
if overwriteFull {
if dst.CanSet() {
dst.Set(src)
} else {
dst = src
}
if dst.CanSet() && !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) {
dst.Set(src)
}
}
return
}
@ -260,7 +199,7 @@ func Merge(dst, src interface{}, opts ...func(*Config)) error {
return merge(dst, src, opts...)
}
// MergeWithOverwrite will do the same as Merge except that non-empty dst attributes will be overridden by
// MergeWithOverwrite will do the same as Merge except that non-empty dst attributes will be overriden by
// non-empty src attribute values.
// Deprecated: use Merge(…) with WithOverride
func MergeWithOverwrite(dst, src interface{}, opts ...func(*Config)) error {
@ -279,26 +218,11 @@ func WithOverride(config *Config) {
config.Overwrite = true
}
// WithOverwriteWithEmptyValue will make merge override non empty dst attributes with empty src attributes values.
func WithOverwriteWithEmptyValue(config *Config) {
config.overwriteWithEmptyValue = true
}
// WithOverrideEmptySlice will make merge override empty dst slice with empty src slice.
func WithOverrideEmptySlice(config *Config) {
config.overwriteSliceWithEmptyValue = true
}
// WithAppendSlice will make merge append slices instead of overwriting it.
// WithAppendSlice will make merge append slices instead of overwriting it
func WithAppendSlice(config *Config) {
config.AppendSlice = true
}
// WithTypeCheck will make merge check types while overwriting it (must be used with WithOverride).
func WithTypeCheck(config *Config) {
config.TypeCheck = true
}
func merge(dst, src interface{}, opts ...func(*Config)) error {
var (
vDst, vSrc reflect.Value
@ -314,25 +238,8 @@ func merge(dst, src interface{}, opts ...func(*Config)) error {
if vDst, vSrc, err = resolveValues(dst, src); err != nil {
return err
}
if !vDst.CanSet() {
return fmt.Errorf("cannot set dst, needs reference")
}
if vDst.Type() != vSrc.Type() {
return ErrDifferentArgumentsTypes
}
_, err = deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
return err
}
// IsReflectNil is the reflect value provided nil
func isReflectNil(v reflect.Value) bool {
k := v.Kind()
switch k {
case reflect.Interface, reflect.Slice, reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr:
// Both interface and slice are nil if first word is 0.
// Both are always bigger than a word; assume flagIndir.
return v.IsNil()
default:
return false
}
return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
}

View File

@ -1 +0,0 @@
/test/*.dat*

View File

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2018 InfluxData Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -1,42 +0,0 @@
# tdigest
This is an implementation of Ted Dunning's [t-digest](https://github.com/tdunning/t-digest/) in Go.
The implementation is based off [Derrick Burns' C++ implementation](https://github.com/derrickburns/tdigest).
## Example
```go
package main
import (
"log"
"github.com/influxdata/tdigest"
)
func main() {
td := tdigest.NewWithCompression(1000)
for _, x := range []float64{1, 2, 3, 4, 5, 5, 4, 3, 2, 1} {
td.Add(x, 1)
}
// Compute Quantiles
log.Println("50th", td.Quantile(0.5))
log.Println("75th", td.Quantile(0.75))
log.Println("90th", td.Quantile(0.9))
log.Println("99th", td.Quantile(0.99))
// Compute CDFs
log.Println("CDF(1) = ", td.CDF(1))
log.Println("CDF(2) = ", td.CDF(2))
log.Println("CDF(3) = ", td.CDF(3))
log.Println("CDF(4) = ", td.CDF(4))
log.Println("CDF(5) = ", td.CDF(5))
}
```
## TODO
Only the methods for a single TDigest have been implemented.
The methods to merge two or more existing t-digests into a single t-digest have yet to be implemented.

View File

@ -1,60 +0,0 @@
package tdigest
import (
"fmt"
"sort"
)
// ErrWeightLessThanZero is used when the weight is not able to be processed.
const ErrWeightLessThanZero = Error("centroid weight cannot be less than zero")
// Error is a domain error encountered while processing tdigests
type Error string
func (e Error) Error() string {
return string(e)
}
// Centroid average position of all points in a shape
type Centroid struct {
Mean float64
Weight float64
}
func (c *Centroid) String() string {
return fmt.Sprintf("{mean: %f weight: %f}", c.Mean, c.Weight)
}
// Add averages the two centroids together and update this centroid
func (c *Centroid) Add(r Centroid) error {
if r.Weight < 0 {
return ErrWeightLessThanZero
}
if c.Weight != 0 {
c.Weight += r.Weight
c.Mean += r.Weight * (r.Mean - c.Mean) / c.Weight
} else {
c.Weight = r.Weight
c.Mean = r.Mean
}
return nil
}
// CentroidList is sorted by the Mean of the centroid, ascending.
type CentroidList []Centroid
// Clear clears the list.
func (l *CentroidList) Clear() {
*l = (*l)[:0]
}
func (l CentroidList) Len() int { return len(l) }
func (l CentroidList) Less(i, j int) bool { return l[i].Mean < l[j].Mean }
func (l CentroidList) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
// NewCentroidList creates a priority queue for the centroids
func NewCentroidList(centroids []Centroid) CentroidList {
l := CentroidList(centroids)
sort.Sort(l)
return l
}

View File

@ -1,10 +0,0 @@
module github.com/influxdata/tdigest
require (
github.com/google/go-cmp v0.2.0
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 // indirect
)
go 1.13

View File

@ -1,9 +0,0 @@
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de h1:xSjD6HQTqT0H/k60N5yYBtnN1OEkVy7WIo/DYyxKRO0=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 h1:4WsZyVtkthqrHTbDCJfiTs8IWNYE4uvsSDgaV6xpp+o=
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=

View File

@ -1,276 +0,0 @@
package tdigest
import (
"math"
"sort"
)
// TDigest is a data structure for accurate on-line accumulation of
// rank-based statistics such as quantiles and trimmed means.
type TDigest struct {
Compression float64
maxProcessed int
maxUnprocessed int
processed CentroidList
unprocessed CentroidList
cumulative []float64
processedWeight float64
unprocessedWeight float64
min float64
max float64
}
// New initializes a new distribution with a default compression.
func New() *TDigest {
return NewWithCompression(1000)
}
// NewWithCompression initializes a new distribution with custom compression.
func NewWithCompression(c float64) *TDigest {
t := &TDigest{
Compression: c,
}
t.maxProcessed = processedSize(0, t.Compression)
t.maxUnprocessed = unprocessedSize(0, t.Compression)
t.processed = make(CentroidList, 0, t.maxProcessed)
t.unprocessed = make(CentroidList, 0, t.maxUnprocessed+1)
t.Reset()
return t
}
// Reset resets the distribution to its initial state.
func (t *TDigest) Reset() {
t.processed = t.processed[:0]
t.unprocessed = t.unprocessed[:0]
t.cumulative = t.cumulative[:0]
t.processedWeight = 0
t.unprocessedWeight = 0
t.min = math.MaxFloat64
t.max = -math.MaxFloat64
}
// Add adds a value x with a weight w to the distribution.
func (t *TDigest) Add(x, w float64) {
if math.IsNaN(x) {
return
}
t.AddCentroid(Centroid{Mean: x, Weight: w})
}
// AddCentroidList can quickly add multiple centroids.
func (t *TDigest) AddCentroidList(c CentroidList) {
l := c.Len()
for i := 0; i < l; i++ {
diff := l - i
room := t.maxUnprocessed - t.unprocessed.Len()
mid := i + diff
if room < diff {
mid = i + room
}
for i < mid {
t.AddCentroid(c[i])
i++
}
}
}
// AddCentroid adds a single centroid.
func (t *TDigest) AddCentroid(c Centroid) {
t.unprocessed = append(t.unprocessed, c)
t.unprocessedWeight += c.Weight
if t.processed.Len() > t.maxProcessed ||
t.unprocessed.Len() > t.maxUnprocessed {
t.process()
}
}
func (t *TDigest) process() {
if t.unprocessed.Len() > 0 ||
t.processed.Len() > t.maxProcessed {
// Append all processed centroids to the unprocessed list and sort
t.unprocessed = append(t.unprocessed, t.processed...)
sort.Sort(&t.unprocessed)
// Reset processed list with first centroid
t.processed.Clear()
t.processed = append(t.processed, t.unprocessed[0])
t.processedWeight += t.unprocessedWeight
t.unprocessedWeight = 0
soFar := t.unprocessed[0].Weight
limit := t.processedWeight * t.integratedQ(1.0)
for _, centroid := range t.unprocessed[1:] {
projected := soFar + centroid.Weight
if projected <= limit {
soFar = projected
(&t.processed[t.processed.Len()-1]).Add(centroid)
} else {
k1 := t.integratedLocation(soFar / t.processedWeight)
limit = t.processedWeight * t.integratedQ(k1+1.0)
soFar += centroid.Weight
t.processed = append(t.processed, centroid)
}
}
t.min = math.Min(t.min, t.processed[0].Mean)
t.max = math.Max(t.max, t.processed[t.processed.Len()-1].Mean)
t.updateCumulative()
t.unprocessed.Clear()
}
}
// Centroids returns a copy of processed centroids.
// Useful when aggregating multiple t-digests.
//
// Pass in the CentroidList as the buffer to write into.
func (t *TDigest) Centroids() CentroidList {
t.process()
cl := make([]Centroid, len(t.processed))
copy(cl, t.processed)
return cl
}
func (t *TDigest) Count() float64 {
t.process()
count := 0.0
for _, centroid := range t.processed {
count += centroid.Weight
}
return count
}
func (t *TDigest) updateCumulative() {
if n := t.processed.Len() + 1; n <= cap(t.cumulative) {
t.cumulative = t.cumulative[:n]
} else {
t.cumulative = make([]float64, n)
}
prev := 0.0
for i, centroid := range t.processed {
cur := centroid.Weight
t.cumulative[i] = prev + cur/2.0
prev = prev + cur
}
t.cumulative[t.processed.Len()] = prev
}
// Quantile returns the (approximate) quantile of
// the distribution. Accepted values for q are between 0.0 and 1.0.
// Returns NaN if Count is zero or bad inputs.
func (t *TDigest) Quantile(q float64) float64 {
t.process()
if q < 0 || q > 1 || t.processed.Len() == 0 {
return math.NaN()
}
if t.processed.Len() == 1 {
return t.processed[0].Mean
}
index := q * t.processedWeight
if index <= t.processed[0].Weight/2.0 {
return t.min + 2.0*index/t.processed[0].Weight*(t.processed[0].Mean-t.min)
}
lower := sort.Search(len(t.cumulative), func(i int) bool {
return t.cumulative[i] >= index
})
if lower+1 != len(t.cumulative) {
z1 := index - t.cumulative[lower-1]
z2 := t.cumulative[lower] - index
return weightedAverage(t.processed[lower-1].Mean, z2, t.processed[lower].Mean, z1)
}
z1 := index - t.processedWeight - t.processed[lower-1].Weight/2.0
z2 := (t.processed[lower-1].Weight / 2.0) - z1
return weightedAverage(t.processed[t.processed.Len()-1].Mean, z1, t.max, z2)
}
// CDF returns the cumulative distribution function for a given value x.
func (t *TDigest) CDF(x float64) float64 {
t.process()
switch t.processed.Len() {
case 0:
return 0.0
case 1:
width := t.max - t.min
if x <= t.min {
return 0.0
}
if x >= t.max {
return 1.0
}
if (x - t.min) <= width {
// min and max are too close together to do any viable interpolation
return 0.5
}
return (x - t.min) / width
}
if x <= t.min {
return 0.0
}
if x >= t.max {
return 1.0
}
m0 := t.processed[0].Mean
// Left Tail
if x <= m0 {
if m0-t.min > 0 {
return (x - t.min) / (m0 - t.min) * t.processed[0].Weight / t.processedWeight / 2.0
}
return 0.0
}
// Right Tail
mn := t.processed[t.processed.Len()-1].Mean
if x >= mn {
if t.max-mn > 0.0 {
return 1.0 - (t.max-x)/(t.max-mn)*t.processed[t.processed.Len()-1].Weight/t.processedWeight/2.0
}
return 1.0
}
upper := sort.Search(t.processed.Len(), func(i int) bool {
return t.processed[i].Mean > x
})
z1 := x - t.processed[upper-1].Mean
z2 := t.processed[upper].Mean - x
return weightedAverage(t.cumulative[upper-1], z2, t.cumulative[upper], z1) / t.processedWeight
}
func (t *TDigest) integratedQ(k float64) float64 {
return (math.Sin(math.Min(k, t.Compression)*math.Pi/t.Compression-math.Pi/2.0) + 1.0) / 2.0
}
func (t *TDigest) integratedLocation(q float64) float64 {
return t.Compression * (math.Asin(2.0*q-1.0) + math.Pi/2.0) / math.Pi
}
func weightedAverage(x1, w1, x2, w2 float64) float64 {
if x1 <= x2 {
return weightedAverageSorted(x1, w1, x2, w2)
}
return weightedAverageSorted(x2, w2, x1, w1)
}
func weightedAverageSorted(x1, w1, x2, w2 float64) float64 {
x := (x1*w1 + x2*w2) / (w1 + w2)
return math.Max(x1, math.Min(x, x2))
}
func processedSize(size int, compression float64) int {
if size == 0 {
return int(2 * math.Ceil(compression))
}
return size
}
func unprocessedSize(size int, compression float64) int {
if size == 0 {
return int(8 * math.Ceil(compression))
}
return size
}

View File

@ -1,5 +0,0 @@
Docs: https://godoc.org/github.com/josharian/intern
See also [Go issue 5160](https://golang.org/issue/5160).
License: MIT

View File

@ -1,3 +0,0 @@
module github.com/josharian/intern
go 1.5

View File

@ -1,44 +0,0 @@
// Package intern interns strings.
// Interning is best effort only.
// Interned strings may be removed automatically
// at any time without notification.
// All functions may be called concurrently
// with themselves and each other.
package intern
import "sync"
var (
pool sync.Pool = sync.Pool{
New: func() interface{} {
return make(map[string]string)
},
}
)
// String returns s, interned.
func String(s string) string {
m := pool.Get().(map[string]string)
c, ok := m[s]
if ok {
pool.Put(m)
return c
}
m[s] = s
pool.Put(m)
return s
}
// Bytes returns b converted to a string, interned.
func Bytes(b []byte) string {
m := pool.Get().(map[string]string)
c, ok := m[string(b)]
if ok {
pool.Put(m)
return c
}
s := string(b)
m[s] = s
pool.Put(m)
return s
}

View File

@ -1,21 +0,0 @@
MIT License
Copyright (c) 2019 Josh Bleecher Snyder
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@ -4,7 +4,6 @@ package buffer
import (
"io"
"net"
"sync"
)
@ -53,12 +52,14 @@ func putBuf(buf []byte) {
// getBuf gets a chunk from reuse pool or creates a new one if reuse failed.
func getBuf(size int) []byte {
if size >= config.PooledSize {
if c := buffers[size]; c != nil {
v := c.Get()
if v != nil {
return v.([]byte)
}
if size < config.PooledSize {
return make([]byte, 0, size)
}
if c := buffers[size]; c != nil {
v := c.Get()
if v != nil {
return v.([]byte)
}
}
return make([]byte, 0, size)
@ -77,12 +78,9 @@ type Buffer struct {
// EnsureSpace makes sure that the current chunk contains at least s free bytes,
// possibly creating a new chunk.
func (b *Buffer) EnsureSpace(s int) {
if cap(b.Buf)-len(b.Buf) < s {
b.ensureSpaceSlow(s)
if cap(b.Buf)-len(b.Buf) >= s {
return
}
}
func (b *Buffer) ensureSpaceSlow(s int) {
l := len(b.Buf)
if l > 0 {
if cap(b.toPool) != cap(b.Buf) {
@ -107,22 +105,18 @@ func (b *Buffer) ensureSpaceSlow(s int) {
// AppendByte appends a single byte to buffer.
func (b *Buffer) AppendByte(data byte) {
b.EnsureSpace(1)
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
b.EnsureSpace(1)
}
b.Buf = append(b.Buf, data)
}
// AppendBytes appends a byte slice to buffer.
func (b *Buffer) AppendBytes(data []byte) {
if len(data) <= cap(b.Buf)-len(b.Buf) {
b.Buf = append(b.Buf, data...) // fast path
} else {
b.appendBytesSlow(data)
}
}
func (b *Buffer) appendBytesSlow(data []byte) {
for len(data) > 0 {
b.EnsureSpace(1)
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
b.EnsureSpace(1)
}
sz := cap(b.Buf) - len(b.Buf)
if sz > len(data) {
@ -134,18 +128,12 @@ func (b *Buffer) appendBytesSlow(data []byte) {
}
}
// AppendString appends a string to buffer.
// AppendBytes appends a string to buffer.
func (b *Buffer) AppendString(data string) {
if len(data) <= cap(b.Buf)-len(b.Buf) {
b.Buf = append(b.Buf, data...) // fast path
} else {
b.appendStringSlow(data)
}
}
func (b *Buffer) appendStringSlow(data string) {
for len(data) > 0 {
b.EnsureSpace(1)
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
b.EnsureSpace(1)
}
sz := cap(b.Buf) - len(b.Buf)
if sz > len(data) {
@ -168,22 +156,26 @@ func (b *Buffer) Size() int {
// DumpTo outputs the contents of a buffer to a writer and resets the buffer.
func (b *Buffer) DumpTo(w io.Writer) (written int, err error) {
bufs := net.Buffers(b.bufs)
if len(b.Buf) > 0 {
bufs = append(bufs, b.Buf)
}
n, err := bufs.WriteTo(w)
var n int
for _, buf := range b.bufs {
if err == nil {
n, err = w.Write(buf)
written += n
}
putBuf(buf)
}
if err == nil {
n, err = w.Write(b.Buf)
written += n
}
putBuf(b.toPool)
b.bufs = nil
b.Buf = nil
b.toPool = nil
return int(n), err
return
}
// BuildBytes creates a single byte slice with all the contents of the buffer. Data is
@ -200,7 +192,7 @@ func (b *Buffer) BuildBytes(reuse ...[]byte) []byte {
var ret []byte
size := b.Size()
// If we got a buffer as argument and it is big enough, reuse it.
// If we got a buffer as argument and it is big enought, reuse it.
if len(reuse) == 1 && cap(reuse[0]) >= size {
ret = reuse[0][:0]
} else {

View File

@ -5,7 +5,6 @@
package jlexer
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
@ -15,8 +14,6 @@ import (
"unicode"
"unicode/utf16"
"unicode/utf8"
"github.com/josharian/intern"
)
// tokenKind determines type of a token.
@ -35,10 +32,9 @@ const (
type token struct {
kind tokenKind // Type of a token.
boolValue bool // Value if a boolean literal token.
byteValueCloned bool // true if byteValue was allocated and does not refer to original json body
byteValue []byte // Raw value of a token.
delimValue byte
boolValue bool // Value if a boolean literal token.
byteValue []byte // Raw value of a token.
delimValue byte
}
// Lexer is a JSON lexer: it iterates over JSON tokens in a byte slice.
@ -244,65 +240,23 @@ func (r *Lexer) fetchNumber() {
// findStringLen tries to scan into the string literal for ending quote char to determine required size.
// The size will be exact if no escapes are present and may be inexact if there are escaped chars.
func findStringLen(data []byte) (isValid bool, length int) {
for {
idx := bytes.IndexByte(data, '"')
if idx == -1 {
return false, len(data)
func findStringLen(data []byte) (isValid, hasEscapes bool, length int) {
delta := 0
for i := 0; i < len(data); i++ {
switch data[i] {
case '\\':
i++
delta++
if i < len(data) && data[i] == 'u' {
delta++
}
case '"':
return true, (delta > 0), (i - delta)
}
if idx == 0 || (idx > 0 && data[idx-1] != '\\') {
return true, length + idx
}
// count \\\\\\\ sequences. even number of slashes means quote is not really escaped
cnt := 1
for idx-cnt-1 >= 0 && data[idx-cnt-1] == '\\' {
cnt++
}
if cnt%2 == 0 {
return true, length + idx
}
length += idx + 1
data = data[idx+1:]
}
}
// unescapeStringToken performs unescaping of string token.
// if no escaping is needed, original string is returned, otherwise - a new one allocated
func (r *Lexer) unescapeStringToken() (err error) {
data := r.token.byteValue
var unescapedData []byte
for {
i := bytes.IndexByte(data, '\\')
if i == -1 {
break
}
escapedRune, escapedBytes, err := decodeEscape(data[i:])
if err != nil {
r.errParse(err.Error())
return err
}
if unescapedData == nil {
unescapedData = make([]byte, 0, len(r.token.byteValue))
}
var d [4]byte
s := utf8.EncodeRune(d[:], escapedRune)
unescapedData = append(unescapedData, data[:i]...)
unescapedData = append(unescapedData, d[:s]...)
data = data[i+escapedBytes:]
}
if unescapedData != nil {
r.token.byteValue = append(unescapedData, data...)
r.token.byteValueCloned = true
}
return
return false, false, len(data)
}
// getu4 decodes \uXXXX from the beginning of s, returning the hex value,
@ -332,30 +286,36 @@ func getu4(s []byte) rune {
return val
}
// decodeEscape processes a single escape sequence and returns number of bytes processed.
func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
// processEscape processes a single escape sequence and returns number of bytes processed.
func (r *Lexer) processEscape(data []byte) (int, error) {
if len(data) < 2 {
return 0, 0, errors.New("incorrect escape symbol \\ at the end of token")
return 0, fmt.Errorf("syntax error at %v", string(data))
}
c := data[1]
switch c {
case '"', '/', '\\':
return rune(c), 2, nil
r.token.byteValue = append(r.token.byteValue, c)
return 2, nil
case 'b':
return '\b', 2, nil
r.token.byteValue = append(r.token.byteValue, '\b')
return 2, nil
case 'f':
return '\f', 2, nil
r.token.byteValue = append(r.token.byteValue, '\f')
return 2, nil
case 'n':
return '\n', 2, nil
r.token.byteValue = append(r.token.byteValue, '\n')
return 2, nil
case 'r':
return '\r', 2, nil
r.token.byteValue = append(r.token.byteValue, '\r')
return 2, nil
case 't':
return '\t', 2, nil
r.token.byteValue = append(r.token.byteValue, '\t')
return 2, nil
case 'u':
rr := getu4(data)
if rr < 0 {
return 0, 0, errors.New("incorrectly escaped \\uXXXX sequence")
return 0, errors.New("syntax error")
}
read := 6
@ -368,10 +328,13 @@ func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
rr = unicode.ReplacementChar
}
}
return rr, read, nil
var d [4]byte
s := utf8.EncodeRune(d[:], rr)
r.token.byteValue = append(r.token.byteValue, d[:s]...)
return read, nil
}
return 0, 0, errors.New("incorrectly escaped bytes")
return 0, errors.New("syntax error")
}
// fetchString scans a string literal token.
@ -379,14 +342,43 @@ func (r *Lexer) fetchString() {
r.pos++
data := r.Data[r.pos:]
isValid, length := findStringLen(data)
isValid, hasEscapes, length := findStringLen(data)
if !isValid {
r.pos += length
r.errParse("unterminated string literal")
return
}
r.token.byteValue = data[:length]
r.pos += length + 1 // skip closing '"' as well
if !hasEscapes {
r.token.byteValue = data[:length]
r.pos += length + 1
return
}
r.token.byteValue = make([]byte, 0, length)
p := 0
for i := 0; i < len(data); {
switch data[i] {
case '"':
r.pos += i + 1
r.token.byteValue = append(r.token.byteValue, data[p:i]...)
i++
return
case '\\':
r.token.byteValue = append(r.token.byteValue, data[p:i]...)
off, err := r.processEscape(data[i:])
if err != nil {
r.errParse(err.Error())
return
}
i += off
p = i
default:
i++
}
}
r.errParse("unterminated string literal")
}
// scanToken scans the next token if no token is currently available in the lexer.
@ -401,7 +393,6 @@ func (r *Lexer) scanToken() {
// consume resets the current token to allow scanning the next one.
func (r *Lexer) consume() {
r.token.kind = tokenUndef
r.token.byteValueCloned = false
r.token.delimValue = 0
}
@ -529,7 +520,6 @@ func (r *Lexer) Skip() {
func (r *Lexer) SkipRecursive() {
r.scanToken()
var start, end byte
startPos := r.start
switch r.token.delimValue {
case '{':
@ -555,14 +545,6 @@ func (r *Lexer) SkipRecursive() {
level--
if level == 0 {
r.pos += i + 1
if !json.Valid(r.Data[startPos:r.pos]) {
r.pos = len(r.Data)
r.fatalError = &LexerError{
Reason: "skipped array/object json value is invalid",
Offset: r.pos,
Data: string(r.Data[r.pos:]),
}
}
return
}
case c == '\\' && inQuotes:
@ -620,7 +602,7 @@ func (r *Lexer) Consumed() {
}
}
func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) {
func (r *Lexer) unsafeString() (string, []byte) {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
@ -628,13 +610,6 @@ func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) {
r.errInvalidToken("string")
return "", nil
}
if !skipUnescape {
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return "", nil
}
}
bytes := r.token.byteValue
ret := bytesToStr(r.token.byteValue)
r.consume()
@ -646,19 +621,13 @@ func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) {
// Warning: returned string may point to the input buffer, so the string should not outlive
// the input buffer. Intended pattern of usage is as an argument to a switch statement.
func (r *Lexer) UnsafeString() string {
ret, _ := r.unsafeString(false)
ret, _ := r.unsafeString()
return ret
}
// UnsafeBytes returns the byte slice if the token is a string literal.
func (r *Lexer) UnsafeBytes() []byte {
_, ret := r.unsafeString(false)
return ret
}
// UnsafeFieldName returns current member name string token
func (r *Lexer) UnsafeFieldName(skipUnescape bool) string {
ret, _ := r.unsafeString(skipUnescape)
_, ret := r.unsafeString()
return ret
}
@ -671,34 +640,7 @@ func (r *Lexer) String() string {
r.errInvalidToken("string")
return ""
}
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return ""
}
var ret string
if r.token.byteValueCloned {
ret = bytesToStr(r.token.byteValue)
} else {
ret = string(r.token.byteValue)
}
r.consume()
return ret
}
// StringIntern reads a string literal, and performs string interning on it.
func (r *Lexer) StringIntern() string {
if r.token.kind == tokenUndef && r.Ok() {
r.FetchToken()
}
if !r.Ok() || r.token.kind != tokenString {
r.errInvalidToken("string")
return ""
}
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return ""
}
ret := intern.Bytes(r.token.byteValue)
ret := string(r.token.byteValue)
r.consume()
return ret
}
@ -712,10 +654,6 @@ func (r *Lexer) Bytes() []byte {
r.errInvalidToken("string")
return nil
}
if err := r.unescapeStringToken(); err != nil {
r.errInvalidToken("string")
return nil
}
ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue)))
n, err := base64.StdEncoding.Decode(ret, r.token.byteValue)
if err != nil {
@ -901,7 +839,7 @@ func (r *Lexer) Int() int {
}
func (r *Lexer) Uint8Str() uint8 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -918,7 +856,7 @@ func (r *Lexer) Uint8Str() uint8 {
}
func (r *Lexer) Uint16Str() uint16 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -935,7 +873,7 @@ func (r *Lexer) Uint16Str() uint16 {
}
func (r *Lexer) Uint32Str() uint32 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -952,7 +890,7 @@ func (r *Lexer) Uint32Str() uint32 {
}
func (r *Lexer) Uint64Str() uint64 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -977,7 +915,7 @@ func (r *Lexer) UintptrStr() uintptr {
}
func (r *Lexer) Int8Str() int8 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -994,7 +932,7 @@ func (r *Lexer) Int8Str() int8 {
}
func (r *Lexer) Int16Str() int16 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -1011,7 +949,7 @@ func (r *Lexer) Int16Str() int16 {
}
func (r *Lexer) Int32Str() int32 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -1028,7 +966,7 @@ func (r *Lexer) Int32Str() int32 {
}
func (r *Lexer) Int64Str() int64 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -1066,7 +1004,7 @@ func (r *Lexer) Float32() float32 {
}
func (r *Lexer) Float32Str() float32 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}
@ -1099,7 +1037,7 @@ func (r *Lexer) Float64() float64 {
}
func (r *Lexer) Float64Str() float64 {
s, b := r.unsafeString(false)
s, b := r.unsafeString()
if !r.Ok() {
return 0
}

View File

@ -270,25 +270,16 @@ func (w *Writer) Bool(v bool) {
const chars = "0123456789abcdef"
func getTable(falseValues ...int) [128]bool {
table := [128]bool{}
for i := 0; i < 128; i++ {
table[i] = true
func isNotEscapedSingleChar(c byte, escapeHTML bool) bool {
// Note: might make sense to use a table if there are more chars to escape. With 4 chars
// it benchmarks the same.
if escapeHTML {
return c != '<' && c != '>' && c != '&' && c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf
} else {
return c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf
}
for _, v := range falseValues {
table[v] = false
}
return table
}
var (
htmlEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '&', '<', '>', '\\')
htmlNoEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '\\')
)
func (w *Writer) String(s string) {
w.Buffer.AppendByte('"')
@ -297,21 +288,15 @@ func (w *Writer) String(s string) {
p := 0 // last non-escape symbol
escapeTable := &htmlEscapeTable
if w.NoEscapeHTML {
escapeTable = &htmlNoEscapeTable
}
for i := 0; i < len(s); {
c := s[i]
if c < utf8.RuneSelf {
if escapeTable[c] {
// single-width character, no escaping is required
i++
continue
}
if isNotEscapedSingleChar(c, !w.NoEscapeHTML) {
// single-width character, no escaping is required
i++
continue
} else if c < utf8.RuneSelf {
// single-with character, need to escape
w.Buffer.AppendString(s[p:i])
switch c {
case '\t':

View File

@ -1,25 +0,0 @@
# This is the official list of go-diff authors for copyright purposes.
# This file is distinct from the CONTRIBUTORS files.
# See the latter for an explanation.
# Names should be added to this file as
# Name or Organization <email address>
# The email address is not required for organizations.
# Please keep the list sorted.
Danny Yoo <dannyyoo@google.com>
James Kolb <jkolb@google.com>
Jonathan Amsterdam <jba@google.com>
Markus Zimmermann <markus.zimmermann@nethead.at> <markus.zimmermann@symflower.com> <zimmski@gmail.com>
Matt Kovars <akaskik@gmail.com>
Örjan Persson <orjan@spotify.com>
Osman Masood <oamasood@gmail.com>
Robert Carlsen <rwcarlsen@gmail.com>
Rory Flynn <roryflynn@users.noreply.github.com>
Sergi Mansilla <sergi.mansilla@gmail.com>
Shatrugna Sadhu <ssadhu@apcera.com>
Shawn Smith <shawnpsmith@gmail.com>
Stas Maksimov <maksimov@gmail.com>
Tor Arvid Lund <torarvid@gmail.com>
Zac Bergquist <zbergquist99@gmail.com>

View File

@ -1,32 +0,0 @@
# This is the official list of people who can contribute
# (and typically have contributed) code to the go-diff
# repository.
#
# The AUTHORS file lists the copyright holders; this file
# lists people. For example, ACME Inc. employees would be listed here
# but not in AUTHORS, because ACME Inc. would hold the copyright.
#
# When adding J Random Contributor's name to this file,
# either J's name or J's organization's name should be
# added to the AUTHORS file.
#
# Names should be added to this file like so:
# Name <email address>
#
# Please keep the list sorted.
Danny Yoo <dannyyoo@google.com>
James Kolb <jkolb@google.com>
Jonathan Amsterdam <jba@google.com>
Markus Zimmermann <markus.zimmermann@nethead.at> <markus.zimmermann@symflower.com> <zimmski@gmail.com>
Matt Kovars <akaskik@gmail.com>
Örjan Persson <orjan@spotify.com>
Osman Masood <oamasood@gmail.com>
Robert Carlsen <rwcarlsen@gmail.com>
Rory Flynn <roryflynn@users.noreply.github.com>
Sergi Mansilla <sergi.mansilla@gmail.com>
Shatrugna Sadhu <ssadhu@apcera.com>
Shawn Smith <shawnpsmith@gmail.com>
Stas Maksimov <maksimov@gmail.com>
Tor Arvid Lund <torarvid@gmail.com>
Zac Bergquist <zbergquist99@gmail.com>

View File

@ -1,20 +0,0 @@
Copyright (c) 2012-2016 The go-diff Authors. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation
the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.

File diff suppressed because it is too large Load Diff

View File

@ -1,46 +0,0 @@
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
// https://github.com/sergi/go-diff
// See the included LICENSE file for license details.
//
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
// Original library is Copyright (c) 2006 Google Inc.
// http://code.google.com/p/google-diff-match-patch/
// Package diffmatchpatch offers robust algorithms to perform the operations required for synchronizing plain text.
package diffmatchpatch
import (
"time"
)
// DiffMatchPatch holds the configuration for diff-match-patch operations.
type DiffMatchPatch struct {
// Number of seconds to map a diff before giving up (0 for infinity).
DiffTimeout time.Duration
// Cost of an empty edit operation in terms of edit characters.
DiffEditCost int
// How far to search for a match (0 = exact location, 1000+ = broad match). A match this many characters away from the expected location will add 1.0 to the score (0.0 is a perfect match).
MatchDistance int
// When deleting a large block of text (over ~64 characters), how close do the contents have to be to match the expected contents. (0.0 = perfection, 1.0 = very loose). Note that MatchThreshold controls how closely the end points of a delete need to match.
PatchDeleteThreshold float64
// Chunk size for context length.
PatchMargin int
// The number of bits in an int.
MatchMaxBits int
// At what point is no match declared (0.0 = perfection, 1.0 = very loose).
MatchThreshold float64
}
// New creates a new DiffMatchPatch object with default parameters.
func New() *DiffMatchPatch {
// Defaults.
return &DiffMatchPatch{
DiffTimeout: time.Second,
DiffEditCost: 4,
MatchThreshold: 0.5,
MatchDistance: 1000,
PatchDeleteThreshold: 0.5,
PatchMargin: 4,
MatchMaxBits: 32,
}
}

View File

@ -1,160 +0,0 @@
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
// https://github.com/sergi/go-diff
// See the included LICENSE file for license details.
//
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
// Original library is Copyright (c) 2006 Google Inc.
// http://code.google.com/p/google-diff-match-patch/
package diffmatchpatch
import (
"math"
)
// MatchMain locates the best instance of 'pattern' in 'text' near 'loc'.
// Returns -1 if no match found.
func (dmp *DiffMatchPatch) MatchMain(text, pattern string, loc int) int {
// Check for null inputs not needed since null can't be passed in C#.
loc = int(math.Max(0, math.Min(float64(loc), float64(len(text)))))
if text == pattern {
// Shortcut (potentially not guaranteed by the algorithm)
return 0
} else if len(text) == 0 {
// Nothing to match.
return -1
} else if loc+len(pattern) <= len(text) && text[loc:loc+len(pattern)] == pattern {
// Perfect match at the perfect spot! (Includes case of null pattern)
return loc
}
// Do a fuzzy compare.
return dmp.MatchBitap(text, pattern, loc)
}
// MatchBitap locates the best instance of 'pattern' in 'text' near 'loc' using the Bitap algorithm.
// Returns -1 if no match was found.
func (dmp *DiffMatchPatch) MatchBitap(text, pattern string, loc int) int {
// Initialise the alphabet.
s := dmp.MatchAlphabet(pattern)
// Highest score beyond which we give up.
scoreThreshold := dmp.MatchThreshold
// Is there a nearby exact match? (speedup)
bestLoc := indexOf(text, pattern, loc)
if bestLoc != -1 {
scoreThreshold = math.Min(dmp.matchBitapScore(0, bestLoc, loc,
pattern), scoreThreshold)
// What about in the other direction? (speedup)
bestLoc = lastIndexOf(text, pattern, loc+len(pattern))
if bestLoc != -1 {
scoreThreshold = math.Min(dmp.matchBitapScore(0, bestLoc, loc,
pattern), scoreThreshold)
}
}
// Initialise the bit arrays.
matchmask := 1 << uint((len(pattern) - 1))
bestLoc = -1
var binMin, binMid int
binMax := len(pattern) + len(text)
lastRd := []int{}
for d := 0; d < len(pattern); d++ {
// Scan for the best match; each iteration allows for one more error. Run a binary search to determine how far from 'loc' we can stray at this error level.
binMin = 0
binMid = binMax
for binMin < binMid {
if dmp.matchBitapScore(d, loc+binMid, loc, pattern) <= scoreThreshold {
binMin = binMid
} else {
binMax = binMid
}
binMid = (binMax-binMin)/2 + binMin
}
// Use the result from this iteration as the maximum for the next.
binMax = binMid
start := int(math.Max(1, float64(loc-binMid+1)))
finish := int(math.Min(float64(loc+binMid), float64(len(text))) + float64(len(pattern)))
rd := make([]int, finish+2)
rd[finish+1] = (1 << uint(d)) - 1
for j := finish; j >= start; j-- {
var charMatch int
if len(text) <= j-1 {
// Out of range.
charMatch = 0
} else if _, ok := s[text[j-1]]; !ok {
charMatch = 0
} else {
charMatch = s[text[j-1]]
}
if d == 0 {
// First pass: exact match.
rd[j] = ((rd[j+1] << 1) | 1) & charMatch
} else {
// Subsequent passes: fuzzy match.
rd[j] = ((rd[j+1]<<1)|1)&charMatch | (((lastRd[j+1] | lastRd[j]) << 1) | 1) | lastRd[j+1]
}
if (rd[j] & matchmask) != 0 {
score := dmp.matchBitapScore(d, j-1, loc, pattern)
// This match will almost certainly be better than any existing match. But check anyway.
if score <= scoreThreshold {
// Told you so.
scoreThreshold = score
bestLoc = j - 1
if bestLoc > loc {
// When passing loc, don't exceed our current distance from loc.
start = int(math.Max(1, float64(2*loc-bestLoc)))
} else {
// Already passed loc, downhill from here on in.
break
}
}
}
}
if dmp.matchBitapScore(d+1, loc, loc, pattern) > scoreThreshold {
// No hope for a (better) match at greater error levels.
break
}
lastRd = rd
}
return bestLoc
}
// matchBitapScore computes and returns the score for a match with e errors and x location.
func (dmp *DiffMatchPatch) matchBitapScore(e, x, loc int, pattern string) float64 {
accuracy := float64(e) / float64(len(pattern))
proximity := math.Abs(float64(loc - x))
if dmp.MatchDistance == 0 {
// Dodge divide by zero error.
if proximity == 0 {
return accuracy
}
return 1.0
}
return accuracy + (proximity / float64(dmp.MatchDistance))
}
// MatchAlphabet initialises the alphabet for the Bitap algorithm.
func (dmp *DiffMatchPatch) MatchAlphabet(pattern string) map[byte]int {
s := map[byte]int{}
charPattern := []byte(pattern)
for _, c := range charPattern {
_, ok := s[c]
if !ok {
s[c] = 0
}
}
i := 0
for _, c := range charPattern {
value := s[c] | int(uint(1)<<uint((len(pattern)-i-1)))
s[c] = value
i++
}
return s
}

View File

@ -1,23 +0,0 @@
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
// https://github.com/sergi/go-diff
// See the included LICENSE file for license details.
//
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
// Original library is Copyright (c) 2006 Google Inc.
// http://code.google.com/p/google-diff-match-patch/
package diffmatchpatch
func min(x, y int) int {
if x < y {
return x
}
return y
}
func max(x, y int) int {
if x > y {
return x
}
return y
}

View File

@ -1,17 +0,0 @@
// Code generated by "stringer -type=Operation -trimprefix=Diff"; DO NOT EDIT.
package diffmatchpatch
import "fmt"
const _Operation_name = "DeleteEqualInsert"
var _Operation_index = [...]uint8{0, 6, 11, 17}
func (i Operation) String() string {
i -= -1
if i < 0 || i >= Operation(len(_Operation_index)-1) {
return fmt.Sprintf("Operation(%d)", i+-1)
}
return _Operation_name[_Operation_index[i]:_Operation_index[i+1]]
}

View File

@ -1,556 +0,0 @@
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
// https://github.com/sergi/go-diff
// See the included LICENSE file for license details.
//
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
// Original library is Copyright (c) 2006 Google Inc.
// http://code.google.com/p/google-diff-match-patch/
package diffmatchpatch
import (
"bytes"
"errors"
"math"
"net/url"
"regexp"
"strconv"
"strings"
)
// Patch represents one patch operation.
type Patch struct {
diffs []Diff
Start1 int
Start2 int
Length1 int
Length2 int
}
// String emulates GNU diff's format.
// Header: @@ -382,8 +481,9 @@
// Indices are printed as 1-based, not 0-based.
func (p *Patch) String() string {
var coords1, coords2 string
if p.Length1 == 0 {
coords1 = strconv.Itoa(p.Start1) + ",0"
} else if p.Length1 == 1 {
coords1 = strconv.Itoa(p.Start1 + 1)
} else {
coords1 = strconv.Itoa(p.Start1+1) + "," + strconv.Itoa(p.Length1)
}
if p.Length2 == 0 {
coords2 = strconv.Itoa(p.Start2) + ",0"
} else if p.Length2 == 1 {
coords2 = strconv.Itoa(p.Start2 + 1)
} else {
coords2 = strconv.Itoa(p.Start2+1) + "," + strconv.Itoa(p.Length2)
}
var text bytes.Buffer
_, _ = text.WriteString("@@ -" + coords1 + " +" + coords2 + " @@\n")
// Escape the body of the patch with %xx notation.
for _, aDiff := range p.diffs {
switch aDiff.Type {
case DiffInsert:
_, _ = text.WriteString("+")
case DiffDelete:
_, _ = text.WriteString("-")
case DiffEqual:
_, _ = text.WriteString(" ")
}
_, _ = text.WriteString(strings.Replace(url.QueryEscape(aDiff.Text), "+", " ", -1))
_, _ = text.WriteString("\n")
}
return unescaper.Replace(text.String())
}
// PatchAddContext increases the context until it is unique, but doesn't let the pattern expand beyond MatchMaxBits.
func (dmp *DiffMatchPatch) PatchAddContext(patch Patch, text string) Patch {
if len(text) == 0 {
return patch
}
pattern := text[patch.Start2 : patch.Start2+patch.Length1]
padding := 0
// Look for the first and last matches of pattern in text. If two different matches are found, increase the pattern length.
for strings.Index(text, pattern) != strings.LastIndex(text, pattern) &&
len(pattern) < dmp.MatchMaxBits-2*dmp.PatchMargin {
padding += dmp.PatchMargin
maxStart := max(0, patch.Start2-padding)
minEnd := min(len(text), patch.Start2+patch.Length1+padding)
pattern = text[maxStart:minEnd]
}
// Add one chunk for good luck.
padding += dmp.PatchMargin
// Add the prefix.
prefix := text[max(0, patch.Start2-padding):patch.Start2]
if len(prefix) != 0 {
patch.diffs = append([]Diff{Diff{DiffEqual, prefix}}, patch.diffs...)
}
// Add the suffix.
suffix := text[patch.Start2+patch.Length1 : min(len(text), patch.Start2+patch.Length1+padding)]
if len(suffix) != 0 {
patch.diffs = append(patch.diffs, Diff{DiffEqual, suffix})
}
// Roll back the start points.
patch.Start1 -= len(prefix)
patch.Start2 -= len(prefix)
// Extend the lengths.
patch.Length1 += len(prefix) + len(suffix)
patch.Length2 += len(prefix) + len(suffix)
return patch
}
// PatchMake computes a list of patches.
func (dmp *DiffMatchPatch) PatchMake(opt ...interface{}) []Patch {
if len(opt) == 1 {
diffs, _ := opt[0].([]Diff)
text1 := dmp.DiffText1(diffs)
return dmp.PatchMake(text1, diffs)
} else if len(opt) == 2 {
text1 := opt[0].(string)
switch t := opt[1].(type) {
case string:
diffs := dmp.DiffMain(text1, t, true)
if len(diffs) > 2 {
diffs = dmp.DiffCleanupSemantic(diffs)
diffs = dmp.DiffCleanupEfficiency(diffs)
}
return dmp.PatchMake(text1, diffs)
case []Diff:
return dmp.patchMake2(text1, t)
}
} else if len(opt) == 3 {
return dmp.PatchMake(opt[0], opt[2])
}
return []Patch{}
}
// patchMake2 computes a list of patches to turn text1 into text2.
// text2 is not provided, diffs are the delta between text1 and text2.
func (dmp *DiffMatchPatch) patchMake2(text1 string, diffs []Diff) []Patch {
// Check for null inputs not needed since null can't be passed in C#.
patches := []Patch{}
if len(diffs) == 0 {
return patches // Get rid of the null case.
}
patch := Patch{}
charCount1 := 0 // Number of characters into the text1 string.
charCount2 := 0 // Number of characters into the text2 string.
// Start with text1 (prepatchText) and apply the diffs until we arrive at text2 (postpatchText). We recreate the patches one by one to determine context info.
prepatchText := text1
postpatchText := text1
for i, aDiff := range diffs {
if len(patch.diffs) == 0 && aDiff.Type != DiffEqual {
// A new patch starts here.
patch.Start1 = charCount1
patch.Start2 = charCount2
}
switch aDiff.Type {
case DiffInsert:
patch.diffs = append(patch.diffs, aDiff)
patch.Length2 += len(aDiff.Text)
postpatchText = postpatchText[:charCount2] +
aDiff.Text + postpatchText[charCount2:]
case DiffDelete:
patch.Length1 += len(aDiff.Text)
patch.diffs = append(patch.diffs, aDiff)
postpatchText = postpatchText[:charCount2] + postpatchText[charCount2+len(aDiff.Text):]
case DiffEqual:
if len(aDiff.Text) <= 2*dmp.PatchMargin &&
len(patch.diffs) != 0 && i != len(diffs)-1 {
// Small equality inside a patch.
patch.diffs = append(patch.diffs, aDiff)
patch.Length1 += len(aDiff.Text)
patch.Length2 += len(aDiff.Text)
}
if len(aDiff.Text) >= 2*dmp.PatchMargin {
// Time for a new patch.
if len(patch.diffs) != 0 {
patch = dmp.PatchAddContext(patch, prepatchText)
patches = append(patches, patch)
patch = Patch{}
// Unlike Unidiff, our patch lists have a rolling context. http://code.google.com/p/google-diff-match-patch/wiki/Unidiff Update prepatch text & pos to reflect the application of the just completed patch.
prepatchText = postpatchText
charCount1 = charCount2
}
}
}
// Update the current character count.
if aDiff.Type != DiffInsert {
charCount1 += len(aDiff.Text)
}
if aDiff.Type != DiffDelete {
charCount2 += len(aDiff.Text)
}
}
// Pick up the leftover patch if not empty.
if len(patch.diffs) != 0 {
patch = dmp.PatchAddContext(patch, prepatchText)
patches = append(patches, patch)
}
return patches
}
// PatchDeepCopy returns an array that is identical to a given an array of patches.
func (dmp *DiffMatchPatch) PatchDeepCopy(patches []Patch) []Patch {
patchesCopy := []Patch{}
for _, aPatch := range patches {
patchCopy := Patch{}
for _, aDiff := range aPatch.diffs {
patchCopy.diffs = append(patchCopy.diffs, Diff{
aDiff.Type,
aDiff.Text,
})
}
patchCopy.Start1 = aPatch.Start1
patchCopy.Start2 = aPatch.Start2
patchCopy.Length1 = aPatch.Length1
patchCopy.Length2 = aPatch.Length2
patchesCopy = append(patchesCopy, patchCopy)
}
return patchesCopy
}
// PatchApply merges a set of patches onto the text. Returns a patched text, as well as an array of true/false values indicating which patches were applied.
func (dmp *DiffMatchPatch) PatchApply(patches []Patch, text string) (string, []bool) {
if len(patches) == 0 {
return text, []bool{}
}
// Deep copy the patches so that no changes are made to originals.
patches = dmp.PatchDeepCopy(patches)
nullPadding := dmp.PatchAddPadding(patches)
text = nullPadding + text + nullPadding
patches = dmp.PatchSplitMax(patches)
x := 0
// delta keeps track of the offset between the expected and actual location of the previous patch. If there are patches expected at positions 10 and 20, but the first patch was found at 12, delta is 2 and the second patch has an effective expected position of 22.
delta := 0
results := make([]bool, len(patches))
for _, aPatch := range patches {
expectedLoc := aPatch.Start2 + delta
text1 := dmp.DiffText1(aPatch.diffs)
var startLoc int
endLoc := -1
if len(text1) > dmp.MatchMaxBits {
// PatchSplitMax will only provide an oversized pattern in the case of a monster delete.
startLoc = dmp.MatchMain(text, text1[:dmp.MatchMaxBits], expectedLoc)
if startLoc != -1 {
endLoc = dmp.MatchMain(text,
text1[len(text1)-dmp.MatchMaxBits:], expectedLoc+len(text1)-dmp.MatchMaxBits)
if endLoc == -1 || startLoc >= endLoc {
// Can't find valid trailing context. Drop this patch.
startLoc = -1
}
}
} else {
startLoc = dmp.MatchMain(text, text1, expectedLoc)
}
if startLoc == -1 {
// No match found. :(
results[x] = false
// Subtract the delta for this failed patch from subsequent patches.
delta -= aPatch.Length2 - aPatch.Length1
} else {
// Found a match. :)
results[x] = true
delta = startLoc - expectedLoc
var text2 string
if endLoc == -1 {
text2 = text[startLoc:int(math.Min(float64(startLoc+len(text1)), float64(len(text))))]
} else {
text2 = text[startLoc:int(math.Min(float64(endLoc+dmp.MatchMaxBits), float64(len(text))))]
}
if text1 == text2 {
// Perfect match, just shove the Replacement text in.
text = text[:startLoc] + dmp.DiffText2(aPatch.diffs) + text[startLoc+len(text1):]
} else {
// Imperfect match. Run a diff to get a framework of equivalent indices.
diffs := dmp.DiffMain(text1, text2, false)
if len(text1) > dmp.MatchMaxBits && float64(dmp.DiffLevenshtein(diffs))/float64(len(text1)) > dmp.PatchDeleteThreshold {
// The end points match, but the content is unacceptably bad.
results[x] = false
} else {
diffs = dmp.DiffCleanupSemanticLossless(diffs)
index1 := 0
for _, aDiff := range aPatch.diffs {
if aDiff.Type != DiffEqual {
index2 := dmp.DiffXIndex(diffs, index1)
if aDiff.Type == DiffInsert {
// Insertion
text = text[:startLoc+index2] + aDiff.Text + text[startLoc+index2:]
} else if aDiff.Type == DiffDelete {
// Deletion
startIndex := startLoc + index2
text = text[:startIndex] +
text[startIndex+dmp.DiffXIndex(diffs, index1+len(aDiff.Text))-index2:]
}
}
if aDiff.Type != DiffDelete {
index1 += len(aDiff.Text)
}
}
}
}
}
x++
}
// Strip the padding off.
text = text[len(nullPadding) : len(nullPadding)+(len(text)-2*len(nullPadding))]
return text, results
}
// PatchAddPadding adds some padding on text start and end so that edges can match something.
// Intended to be called only from within patchApply.
func (dmp *DiffMatchPatch) PatchAddPadding(patches []Patch) string {
paddingLength := dmp.PatchMargin
nullPadding := ""
for x := 1; x <= paddingLength; x++ {
nullPadding += string(x)
}
// Bump all the patches forward.
for i := range patches {
patches[i].Start1 += paddingLength
patches[i].Start2 += paddingLength
}
// Add some padding on start of first diff.
if len(patches[0].diffs) == 0 || patches[0].diffs[0].Type != DiffEqual {
// Add nullPadding equality.
patches[0].diffs = append([]Diff{Diff{DiffEqual, nullPadding}}, patches[0].diffs...)
patches[0].Start1 -= paddingLength // Should be 0.
patches[0].Start2 -= paddingLength // Should be 0.
patches[0].Length1 += paddingLength
patches[0].Length2 += paddingLength
} else if paddingLength > len(patches[0].diffs[0].Text) {
// Grow first equality.
extraLength := paddingLength - len(patches[0].diffs[0].Text)
patches[0].diffs[0].Text = nullPadding[len(patches[0].diffs[0].Text):] + patches[0].diffs[0].Text
patches[0].Start1 -= extraLength
patches[0].Start2 -= extraLength
patches[0].Length1 += extraLength
patches[0].Length2 += extraLength
}
// Add some padding on end of last diff.
last := len(patches) - 1
if len(patches[last].diffs) == 0 || patches[last].diffs[len(patches[last].diffs)-1].Type != DiffEqual {
// Add nullPadding equality.
patches[last].diffs = append(patches[last].diffs, Diff{DiffEqual, nullPadding})
patches[last].Length1 += paddingLength
patches[last].Length2 += paddingLength
} else if paddingLength > len(patches[last].diffs[len(patches[last].diffs)-1].Text) {
// Grow last equality.
lastDiff := patches[last].diffs[len(patches[last].diffs)-1]
extraLength := paddingLength - len(lastDiff.Text)
patches[last].diffs[len(patches[last].diffs)-1].Text += nullPadding[:extraLength]
patches[last].Length1 += extraLength
patches[last].Length2 += extraLength
}
return nullPadding
}
// PatchSplitMax looks through the patches and breaks up any which are longer than the maximum limit of the match algorithm.
// Intended to be called only from within patchApply.
func (dmp *DiffMatchPatch) PatchSplitMax(patches []Patch) []Patch {
patchSize := dmp.MatchMaxBits
for x := 0; x < len(patches); x++ {
if patches[x].Length1 <= patchSize {
continue
}
bigpatch := patches[x]
// Remove the big old patch.
patches = append(patches[:x], patches[x+1:]...)
x--
Start1 := bigpatch.Start1
Start2 := bigpatch.Start2
precontext := ""
for len(bigpatch.diffs) != 0 {
// Create one of several smaller patches.
patch := Patch{}
empty := true
patch.Start1 = Start1 - len(precontext)
patch.Start2 = Start2 - len(precontext)
if len(precontext) != 0 {
patch.Length1 = len(precontext)
patch.Length2 = len(precontext)
patch.diffs = append(patch.diffs, Diff{DiffEqual, precontext})
}
for len(bigpatch.diffs) != 0 && patch.Length1 < patchSize-dmp.PatchMargin {
diffType := bigpatch.diffs[0].Type
diffText := bigpatch.diffs[0].Text
if diffType == DiffInsert {
// Insertions are harmless.
patch.Length2 += len(diffText)
Start2 += len(diffText)
patch.diffs = append(patch.diffs, bigpatch.diffs[0])
bigpatch.diffs = bigpatch.diffs[1:]
empty = false
} else if diffType == DiffDelete && len(patch.diffs) == 1 && patch.diffs[0].Type == DiffEqual && len(diffText) > 2*patchSize {
// This is a large deletion. Let it pass in one chunk.
patch.Length1 += len(diffText)
Start1 += len(diffText)
empty = false
patch.diffs = append(patch.diffs, Diff{diffType, diffText})
bigpatch.diffs = bigpatch.diffs[1:]
} else {
// Deletion or equality. Only take as much as we can stomach.
diffText = diffText[:min(len(diffText), patchSize-patch.Length1-dmp.PatchMargin)]
patch.Length1 += len(diffText)
Start1 += len(diffText)
if diffType == DiffEqual {
patch.Length2 += len(diffText)
Start2 += len(diffText)
} else {
empty = false
}
patch.diffs = append(patch.diffs, Diff{diffType, diffText})
if diffText == bigpatch.diffs[0].Text {
bigpatch.diffs = bigpatch.diffs[1:]
} else {
bigpatch.diffs[0].Text =
bigpatch.diffs[0].Text[len(diffText):]
}
}
}
// Compute the head context for the next patch.
precontext = dmp.DiffText2(patch.diffs)
precontext = precontext[max(0, len(precontext)-dmp.PatchMargin):]
postcontext := ""
// Append the end context for this patch.
if len(dmp.DiffText1(bigpatch.diffs)) > dmp.PatchMargin {
postcontext = dmp.DiffText1(bigpatch.diffs)[:dmp.PatchMargin]
} else {
postcontext = dmp.DiffText1(bigpatch.diffs)
}
if len(postcontext) != 0 {
patch.Length1 += len(postcontext)
patch.Length2 += len(postcontext)
if len(patch.diffs) != 0 && patch.diffs[len(patch.diffs)-1].Type == DiffEqual {
patch.diffs[len(patch.diffs)-1].Text += postcontext
} else {
patch.diffs = append(patch.diffs, Diff{DiffEqual, postcontext})
}
}
if !empty {
x++
patches = append(patches[:x], append([]Patch{patch}, patches[x:]...)...)
}
}
}
return patches
}
// PatchToText takes a list of patches and returns a textual representation.
func (dmp *DiffMatchPatch) PatchToText(patches []Patch) string {
var text bytes.Buffer
for _, aPatch := range patches {
_, _ = text.WriteString(aPatch.String())
}
return text.String()
}
// PatchFromText parses a textual representation of patches and returns a List of Patch objects.
func (dmp *DiffMatchPatch) PatchFromText(textline string) ([]Patch, error) {
patches := []Patch{}
if len(textline) == 0 {
return patches, nil
}
text := strings.Split(textline, "\n")
textPointer := 0
patchHeader := regexp.MustCompile("^@@ -(\\d+),?(\\d*) \\+(\\d+),?(\\d*) @@$")
var patch Patch
var sign uint8
var line string
for textPointer < len(text) {
if !patchHeader.MatchString(text[textPointer]) {
return patches, errors.New("Invalid patch string: " + text[textPointer])
}
patch = Patch{}
m := patchHeader.FindStringSubmatch(text[textPointer])
patch.Start1, _ = strconv.Atoi(m[1])
if len(m[2]) == 0 {
patch.Start1--
patch.Length1 = 1
} else if m[2] == "0" {
patch.Length1 = 0
} else {
patch.Start1--
patch.Length1, _ = strconv.Atoi(m[2])
}
patch.Start2, _ = strconv.Atoi(m[3])
if len(m[4]) == 0 {
patch.Start2--
patch.Length2 = 1
} else if m[4] == "0" {
patch.Length2 = 0
} else {
patch.Start2--
patch.Length2, _ = strconv.Atoi(m[4])
}
textPointer++
for textPointer < len(text) {
if len(text[textPointer]) > 0 {
sign = text[textPointer][0]
} else {
textPointer++
continue
}
line = text[textPointer][1:]
line = strings.Replace(line, "+", "%2b", -1)
line, _ = url.QueryUnescape(line)
if sign == '-' {
// Deletion.
patch.diffs = append(patch.diffs, Diff{DiffDelete, line})
} else if sign == '+' {
// Insertion.
patch.diffs = append(patch.diffs, Diff{DiffInsert, line})
} else if sign == ' ' {
// Minor equality.
patch.diffs = append(patch.diffs, Diff{DiffEqual, line})
} else if sign == '@' {
// Start of next patch.
break
} else {
// WTF?
return patches, errors.New("Invalid patch mode '" + string(sign) + "' in: " + string(line))
}
textPointer++
}
patches = append(patches, patch)
}
return patches, nil
}

View File

@ -1,88 +0,0 @@
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
// https://github.com/sergi/go-diff
// See the included LICENSE file for license details.
//
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
// Original library is Copyright (c) 2006 Google Inc.
// http://code.google.com/p/google-diff-match-patch/
package diffmatchpatch
import (
"strings"
"unicode/utf8"
)
// unescaper unescapes selected chars for compatibility with JavaScript's encodeURI.
// In speed critical applications this could be dropped since the receiving application will certainly decode these fine. Note that this function is case-sensitive. Thus "%3F" would not be unescaped. But this is ok because it is only called with the output of HttpUtility.UrlEncode which returns lowercase hex. Example: "%3f" -> "?", "%24" -> "$", etc.
var unescaper = strings.NewReplacer(
"%21", "!", "%7E", "~", "%27", "'",
"%28", "(", "%29", ")", "%3B", ";",
"%2F", "/", "%3F", "?", "%3A", ":",
"%40", "@", "%26", "&", "%3D", "=",
"%2B", "+", "%24", "$", "%2C", ",", "%23", "#", "%2A", "*")
// indexOf returns the first index of pattern in str, starting at str[i].
func indexOf(str string, pattern string, i int) int {
if i > len(str)-1 {
return -1
}
if i <= 0 {
return strings.Index(str, pattern)
}
ind := strings.Index(str[i:], pattern)
if ind == -1 {
return -1
}
return ind + i
}
// lastIndexOf returns the last index of pattern in str, starting at str[i].
func lastIndexOf(str string, pattern string, i int) int {
if i < 0 {
return -1
}
if i >= len(str) {
return strings.LastIndex(str, pattern)
}
_, size := utf8.DecodeRuneInString(str[i:])
return strings.LastIndex(str[:i+size], pattern)
}
// runesIndexOf returns the index of pattern in target, starting at target[i].
func runesIndexOf(target, pattern []rune, i int) int {
if i > len(target)-1 {
return -1
}
if i <= 0 {
return runesIndex(target, pattern)
}
ind := runesIndex(target[i:], pattern)
if ind == -1 {
return -1
}
return ind + i
}
func runesEqual(r1, r2 []rune) bool {
if len(r1) != len(r2) {
return false
}
for i, c := range r1 {
if c != r2[i] {
return false
}
}
return true
}
// runesIndex is the equivalent of strings.Index for rune slices.
func runesIndex(r1, r2 []rune) int {
last := len(r1) - len(r2)
for i := 0; i <= last; i++ {
if runesEqual(r1[i:i+len(r2)], r2) {
return i
}
}
return -1
}

View File

@ -1,3 +0,0 @@
eval/eval
/target
*.test

View File

@ -1,38 +0,0 @@
language: go
sudo: false
branches:
except:
- release
branches:
only:
- master
- develop
- travis
go:
- 1.9
- tip
matrix:
allow_failures:
- go: tip
before_install:
- if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi;
- if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi;
- go get github.com/mattn/goveralls
before_script:
- make deps
script:
- make qa
after_failure:
- cat ./target/test/report.xml
after_success:
- if [ "$TRAVIS_GO_VERSION" = "1.9" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi;

View File

@ -1,230 +0,0 @@
Copyright (c) 2015, 2016 Damian Gryski <damian@gryski.com>
Copyright (c) 2018 Tomás Senart <tsenart@gmail.com>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-------------------------------------------------------------------------------
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "{}"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright {yyyy} {name of copyright owner}
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -1,203 +0,0 @@
# MAKEFILE
#
# @author Nicola Asuni <info@tecnick.com>
# @link https://github.com/tsenart/go-tsz
#
# This file is intended to be executed in a Linux-compatible system.
# It also assumes that the project has been cloned in the right path under GOPATH:
# $GOPATH/src/github.com/tsenart/go-tsz
#
# ------------------------------------------------------------------------------
# List special make targets that are not associated with files
.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke
# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS).
SHELL=/bin/bash
# CVS path (path to the parent dir containing the project)
CVSPATH=github.com/tsenart
# Project owner
OWNER=tsenart
# Project vendor
VENDOR=tsenart
# Project name
PROJECT=go-tsz
# Project version
VERSION=$(shell cat VERSION)
# Name of RPM or DEB package
PKGNAME=${VENDOR}-${PROJECT}
# Current directory
CURRENTDIR=$(shell pwd)
# GO lang path
ifneq ($(GOPATH),)
ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),)
# the defined GOPATH is not valid
GOPATH=
endif
endif
ifeq ($(GOPATH),)
# extract the GOPATH
GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR)))
endif
# --- MAKE TARGETS ---
# Display general help about this command
help:
@echo ""
@echo "$(PROJECT) Makefile."
@echo "GOPATH=$(GOPATH)"
@echo "The following commands are available:"
@echo ""
@echo " make qa : Run all the tests"
@echo " make test : Run the unit tests"
@echo ""
@echo " make format : Format the source code"
@echo " make fmtcheck : Check if the source code has been formatted"
@echo " make vet : Check for suspicious constructs"
@echo " make lint : Check for style errors"
@echo " make coverage : Generate the coverage report"
@echo " make cyclo : Generate the cyclomatic complexity report"
@echo " make ineffassign : Detect ineffectual assignments"
@echo " make misspell : Detect commonly misspelled words in source files"
@echo " make structcheck : Find unused struct fields"
@echo " make varcheck : Find unused global variables and constants"
@echo " make errcheck : Check that error return values are used"
@echo " make gosimple : Suggest code simplifications"
@echo " make astscan : GO AST scanner"
@echo ""
@echo " make docs : Generate source code documentation"
@echo ""
@echo " make deps : Get the dependencies"
@echo " make clean : Remove any build artifact"
@echo " make nuke : Deletes any intermediate file"
@echo ""
# Alias for help target
all: help
# Run the unit tests
test:
@mkdir -p target/test
@mkdir -p target/report
GOPATH=$(GOPATH) \
go test \
-covermode=atomic \
-bench=. \
-race \
-cpuprofile=target/report/cpu.out \
-memprofile=target/report/mem.out \
-mutexprofile=target/report/mutex.out \
-coverprofile=target/report/coverage.out \
-v . | \
tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \
test $${PIPESTATUS[0]} -eq 0
# Format the source code
format:
@find . -type f -name "*.go" -exec gofmt -s -w {} \;
# Check if the source code has been formatted
fmtcheck:
@mkdir -p target
@find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff
@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
# Check for syntax errors
vet:
GOPATH=$(GOPATH) go vet .
# Check for style errors
lint:
GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint .
# Generate the coverage report
coverage:
@mkdir -p target/report
GOPATH=$(GOPATH) \
go tool cover -html=target/report/coverage.out -o target/report/coverage.html
# Report cyclomatic complexity
cyclo:
@mkdir -p target/report
GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0
# Detect ineffectual assignments
ineffassign:
@mkdir -p target/report
GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0
# Detect commonly misspelled words in source files
misspell:
@mkdir -p target/report
GOPATH=$(GOPATH) misspell -error ./ | tee target/report/misspell.txt ; test $${PIPESTATUS[0]} -eq 0
# Find unused struct fields
structcheck:
@mkdir -p target/report
GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt
# Find unused global variables and constants
varcheck:
@mkdir -p target/report
GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt
# Check that error return values are used
errcheck:
@mkdir -p target/report
GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt
# Suggest code simplifications
gosimple:
@mkdir -p target/report
GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt
# AST scanner
astscan:
@mkdir -p target/report
GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt
# Generate source docs
docs:
@mkdir -p target/docs
nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 &
wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060`
@echo '<html><head><meta http-equiv="refresh" content="0;./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html"/></head><a href="./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html">'${PKGNAME}' Documentation ...</a></html>' > target/docs/index.html
# Alias to run all quality-assurance checks
qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan
# --- INSTALL ---
# Get the dependencies
deps:
GOPATH=$(GOPATH) go get ./...
GOPATH=$(GOPATH) go get github.com/golang/lint/golint
GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report
GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov
GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo
GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign
GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell
GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck
GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck
GOPATH=$(GOPATH) go get github.com/kisielk/errcheck
GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple
GOPATH=$(GOPATH) go get github.com/GoASTScanner/gas
# Remove any build artifact
clean:
GOPATH=$(GOPATH) go clean ./...
# Deletes any intermediate file
nuke:
rm -rf ./target
GOPATH=$(GOPATH) go clean -i ./...

View File

@ -1,41 +0,0 @@
# go-tsz
* Package tsz implement time-series compression http://www.vldb.org/pvldb/vol8/p1816-teller.pdf in Go*
[![Master Branch](https://img.shields.io/badge/branch-master-lightgray.svg)](https://github.com/tsenart/go-tsz/tree/master)
[![Master Build Status](https://secure.travis-ci.org/tsenart/go-tsz.svg?branch=master)](https://travis-ci.org/tsenart/go-tsz?branch=master)
[![Master Coverage Status](https://coveralls.io/repos/tsenart/go-tsz/badge.svg?branch=master&service=github)](https://coveralls.io/github/tsenart/go-tsz?branch=master)
[![Go Report Card](https://goreportcard.com/badge/github.com/tsenart/go-tsz)](https://goreportcard.com/report/github.com/tsenart/go-tsz)
[![GoDoc](https://godoc.org/github.com/tsenart/go-tsz?status.svg)](http://godoc.org/github.com/tsenart/go-tsz)
## Description
Package tsz is a fork of [github.com/dgryski/go-tsz](https://github.com/dgryski/go-tsz) that implements
improvements over the [original Gorilla paper](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) developed by @burmann
in his [Master Thesis](https://aaltodoc.aalto.fi/bitstream/handle/123456789/29099/master_Burman_Michael_2017.pdf?sequence=1)
and released in https://github.com/burmanm/gorilla-tsc.
### Differences from the original paper
- Maximum number of leading zeros is stored with 6 bits to allow up to 63 leading zeros, which are necessary when storing long values.
- Timestamp delta-of-delta is stored by first turning it to a positive integer with ZigZag encoding and then reduced by one to fit in the necessary bits. In the decoding phase all the values are incremented by one to fetch the original value.
- The compressed blocks are created with a 27 bit delta header (unlike in the original paper, which uses a 14 bit delta header). This allows to use up to one day block size using millisecond precision.
## Getting started
This library is written in Go language, please refer to the guides in https://golang.org for getting started.
This project include a Makefile that allows you to test and build the project with simple commands.
To see all available options:
```bash
make help
```
## Running all tests
Before committing the code, please check if it passes all tests using
```bash
make qa
```

View File

@ -1 +0,0 @@
1.0.0

View File

@ -1,222 +0,0 @@
package tsz
import (
"bytes"
"encoding/binary"
"io"
)
// bstream is a stream of bits
type bstream struct {
// the data stream
stream []byte
// how many bits are valid in current byte
count uint8
}
func newBReader(b []byte) *bstream {
return &bstream{stream: b, count: 8}
}
func newBWriter(size int) *bstream {
return &bstream{stream: make([]byte, 0, size), count: 0}
}
func (b *bstream) clone() *bstream {
d := make([]byte, len(b.stream))
copy(d, b.stream)
return &bstream{stream: d, count: b.count}
}
func (b *bstream) bytes() []byte {
return b.stream
}
type bit bool
const (
zero bit = false
one bit = true
)
func (b *bstream) writeBit(bit bit) {
if b.count == 0 {
b.stream = append(b.stream, 0)
b.count = 8
}
i := len(b.stream) - 1
if bit {
b.stream[i] |= 1 << (b.count - 1)
}
b.count--
}
func (b *bstream) writeByte(byt byte) {
if b.count == 0 {
b.stream = append(b.stream, 0)
b.count = 8
}
i := len(b.stream) - 1
// fill up b.b with b.count bits from byt
b.stream[i] |= byt >> (8 - b.count)
b.stream = append(b.stream, 0)
i++
b.stream[i] = byt << b.count
}
func (b *bstream) writeBits(u uint64, nbits int) {
u <<= (64 - uint(nbits))
for nbits >= 8 {
byt := byte(u >> 56)
b.writeByte(byt)
u <<= 8
nbits -= 8
}
for nbits > 0 {
b.writeBit((u >> 63) == 1)
u <<= 1
nbits--
}
}
func (b *bstream) readBit() (bit, error) {
if len(b.stream) == 0 {
return false, io.EOF
}
if b.count == 0 {
b.stream = b.stream[1:]
// did we just run out of stuff to read?
if len(b.stream) == 0 {
return false, io.EOF
}
b.count = 8
}
b.count--
d := b.stream[0] & 0x80
b.stream[0] <<= 1
return d != 0, nil
}
func (b *bstream) readByte() (byte, error) {
if len(b.stream) == 0 {
return 0, io.EOF
}
if b.count == 0 {
b.stream = b.stream[1:]
if len(b.stream) == 0 {
return 0, io.EOF
}
b.count = 8
}
if b.count == 8 {
b.count = 0
return b.stream[0], nil
}
byt := b.stream[0]
b.stream = b.stream[1:]
if len(b.stream) == 0 {
return 0, io.EOF
}
byt |= b.stream[0] >> b.count
b.stream[0] <<= (8 - b.count)
return byt, nil
}
func (b *bstream) readBits(nbits int) (uint64, error) {
var u uint64
for nbits >= 8 {
byt, err := b.readByte()
if err != nil {
return 0, err
}
u = (u << 8) | uint64(byt)
nbits -= 8
}
if nbits == 0 {
return u, nil
}
if nbits > int(b.count) {
u = (u << uint(b.count)) | uint64(b.stream[0]>>(8-b.count))
nbits -= int(b.count)
b.stream = b.stream[1:]
if len(b.stream) == 0 {
return 0, io.EOF
}
b.count = 8
}
u = (u << uint(nbits)) | uint64(b.stream[0]>>(8-uint(nbits)))
b.stream[0] <<= uint(nbits)
b.count -= uint8(nbits)
return u, nil
}
// Read until next unset bit is found or until nbits bits have been read.
func (b *bstream) readUntilZero(nbits int) (uint64, error) {
var u uint64
for i := 0; i < nbits; i++ {
u <<= 1
bit, err := b.readBit()
if err != nil {
return 0, err
}
if bit == zero {
break
}
u |= 1
}
return u, nil
}
// MarshalBinary implements the encoding.BinaryMarshaler interface
func (b *bstream) MarshalBinary() ([]byte, error) {
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.BigEndian, b.count)
if err != nil {
return nil, err
}
err = binary.Write(buf, binary.BigEndian, b.stream)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (b *bstream) UnmarshalBinary(bIn []byte) error {
buf := bytes.NewReader(bIn)
err := binary.Read(buf, binary.BigEndian, &b.count)
if err != nil {
return err
}
b.stream = make([]byte, buf.Len())
return binary.Read(buf, binary.BigEndian, &b.stream)
}

View File

@ -1,69 +0,0 @@
// +build gofuzz
package tsz
import (
"encoding/binary"
"fmt"
"math"
"github.com/tsenart/go-tsz/testdata"
)
func Fuzz(data []byte) int {
fuzzUnpack(data)
if len(data) < 9 {
return 0
}
t0 := uint32(1456236677)
v := float64(10000)
var vals []testdata.Point
s := New(t0)
t := t0
for len(data) >= 10 {
tdelta := uint32(binary.LittleEndian.Uint16(data))
if t == t0 {
tdelta &= (1 << 14) - 1
}
t += tdelta
data = data[2:]
v += float64(int16(binary.LittleEndian.Uint16(data))) + float64(binary.LittleEndian.Uint16(data[2:]))/float64(math.MaxUint16)
data = data[8:]
vals = append(vals, testdata.Point{V: v, T: t})
s.Push(t, v)
}
it := s.Iter()
var i int
for it.Next() {
gt, gv := it.Values()
if gt != vals[i].T || (gv != vals[i].V || math.IsNaN(gv) && math.IsNaN(vals[i].V)) {
panic(fmt.Sprintf("failure: gt=%v vals[i].T=%v gv=%v vals[i].V=%v", gt, vals[i].T, gv, vals[i].V))
}
i++
}
if i != len(vals) {
panic("extra data")
}
return 1
}
func fuzzUnpack(data []byte) {
it, err := NewIterator(data)
if err != nil {
return
}
for it.Next() {
_, _ = it.Values()
}
}

View File

@ -1,264 +0,0 @@
package testdata
type Point struct {
T uint64
V float64
}
var Data = []Point{
{1500400800000, 60000},
{1500405481623, 69087},
{1500405488693, 65640},
{1500405495993, 58155},
{1500405503743, 61025},
{1500405511623, 91156},
{1500405519803, 37516},
{1500405528313, 93515},
{1500405537233, 96226},
{1500405546453, 23833},
{1500405556103, 73186},
{1500405566143, 96947},
{1500405576163, 46927},
{1500405586173, 77954},
{1500405596183, 29302},
{1500405606213, 6700},
{1500405616163, 71971},
{1500405625813, 8528},
{1500405635763, 85321},
{1500405645634, 83229},
{1500405655633, 78298},
{1500405665623, 87122},
{1500405675623, 82055},
{1500405685723, 75067},
{1500405695663, 33680},
{1500405705743, 17576},
{1500405715813, 89701},
{1500405725773, 21427},
{1500405735883, 58255},
{1500405745903, 3768},
{1500405755863, 62086},
{1500405765843, 66965},
{1500405775773, 35801},
{1500405785883, 72169},
{1500405795843, 43089},
{1500405805733, 31418},
{1500405815853, 84781},
{1500405825963, 36103},
{1500405836004, 87431},
{1500405845953, 7379},
{1500405855913, 66919},
{1500405865963, 30906},
{1500405875953, 88630},
{1500405885943, 27546},
{1500405896033, 43813},
{1500405906094, 2124},
{1500405916063, 49399},
{1500405926143, 94577},
{1500405936123, 98459},
{1500405946033, 49457},
{1500405956023, 92838},
{1500405966023, 15628},
{1500405976043, 53916},
{1500405986063, 90387},
{1500405996123, 43176},
{1500406006123, 18838},
{1500406016174, 78847},
{1500406026173, 39591},
{1500406036004, 77070},
{1500406045964, 56788},
{1500406056043, 96706},
{1500406066123, 20756},
{1500406076113, 64433},
{1500406086133, 45791},
{1500406096123, 75028},
{1500406106193, 55403},
{1500406116213, 36991},
{1500406126073, 92929},
{1500406136103, 60416},
{1500406146183, 55485},
{1500406156383, 53525},
{1500406166313, 96021},
{1500406176414, 22705},
{1500406186613, 89801},
{1500406196543, 51975},
{1500406206483, 86741},
{1500406216483, 22440},
{1500406226433, 51818},
{1500406236403, 61965},
{1500406246413, 19074},
{1500406256494, 54521},
{1500406266413, 59315},
{1500406276303, 19171},
{1500406286213, 98800},
{1500406296183, 7086},
{1500406306103, 60578},
{1500406316073, 96828},
{1500406326143, 83746},
{1500406336153, 85481},
{1500406346113, 22346},
{1500406356133, 80976},
{1500406366065, 43586},
{1500406376074, 82500},
{1500406386184, 13576},
{1500406396113, 77871},
{1500406406094, 60978},
{1500406416203, 35264},
{1500406426323, 79733},
{1500406436343, 29140},
{1500406446323, 7237},
{1500406456344, 52866},
{1500406466393, 88456},
{1500406476493, 33533},
{1500406486524, 96961},
{1500406496453, 16389},
{1500406506453, 31181},
{1500406516433, 63282},
{1500406526433, 92857},
{1500406536413, 4582},
{1500406546383, 46832},
{1500406556473, 6335},
{1500406566413, 44367},
{1500406576513, 84640},
{1500406586523, 36174},
{1500406596553, 40075},
{1500406606603, 80886},
{1500406616623, 43784},
{1500406626623, 25077},
{1500406636723, 18617},
{1500406646723, 72681},
{1500406656723, 84811},
{1500406666783, 90053},
{1500406676685, 25708},
{1500406686713, 57134},
{1500406696673, 87193},
{1500406706743, 66057},
{1500406716724, 51404},
{1500406726753, 90141},
{1500406736813, 10434},
{1500406746803, 29056},
{1500406756833, 48160},
{1500406766924, 96652},
{1500406777113, 64141},
{1500406787113, 22143},
{1500406797093, 20561},
{1500406807113, 66401},
{1500406817283, 76802},
{1500406827284, 37555},
{1500406837323, 63169},
{1500406847463, 45712},
{1500406857513, 44751},
{1500406867523, 98891},
{1500406877523, 38122},
{1500406887623, 46202},
{1500406897703, 5875},
{1500406907663, 17397},
{1500406917603, 39994},
{1500406927633, 82385},
{1500406937623, 15598},
{1500406947693, 36235},
{1500406957703, 97536},
{1500406967673, 28557},
{1500406977723, 13985},
{1500406987663, 64304},
{1500406997573, 83693},
{1500407007494, 6574},
{1500407017493, 25134},
{1500407027503, 50383},
{1500407037523, 55922},
{1500407047603, 73436},
{1500407057473, 68235},
{1500407067553, 1469},
{1500407077463, 44315},
{1500407087463, 95064},
{1500407097443, 1997},
{1500407107473, 17247},
{1500407117453, 42454},
{1500407127413, 73631},
{1500407137363, 96890},
{1500407147343, 43450},
{1500407157363, 42042},
{1500407167403, 83014},
{1500407177473, 32051},
{1500407187523, 69280},
{1500407197495, 21425},
{1500407207453, 93748},
{1500407217413, 64151},
{1500407227443, 38791},
{1500407237463, 5248},
{1500407247523, 92935},
{1500407257513, 18516},
{1500407267584, 98870},
{1500407277573, 82244},
{1500407287723, 65464},
{1500407297723, 33801},
{1500407307673, 18331},
{1500407317613, 89744},
{1500407327553, 98460},
{1500407337503, 24709},
{1500407347423, 8407},
{1500407357383, 69451},
{1500407367333, 51100},
{1500407377373, 25309},
{1500407387443, 16148},
{1500407397453, 98974},
{1500407407543, 80284},
{1500407417583, 170},
{1500407427453, 34706},
{1500407437433, 39681},
{1500407447603, 6140},
{1500407457513, 64595},
{1500407467564, 59862},
{1500407477563, 53795},
{1500407487593, 83493},
{1500407497584, 90639},
{1500407507623, 16777},
{1500407517613, 11096},
{1500407527673, 38512},
{1500407537963, 52759},
{1500407548023, 79567},
{1500407558033, 48664},
{1500407568113, 10710},
{1500407578164, 25635},
{1500407588213, 40985},
{1500407598163, 94089},
{1500407608163, 50056},
{1500407618223, 15550},
{1500407628143, 78823},
{1500407638223, 9044},
{1500407648173, 20782},
{1500407658023, 86390},
{1500407667903, 79444},
{1500407677903, 84051},
{1500407687923, 91554},
{1500407697913, 58777},
{1500407708003, 89474},
{1500407718083, 94026},
{1500407728034, 41613},
{1500407738083, 64667},
{1500407748034, 5160},
{1500407758003, 45140},
{1500407768033, 53704},
{1500407778083, 68097},
{1500407788043, 81137},
{1500407798023, 59657},
{1500407808033, 56572},
{1500407817983, 1993},
{1500407828063, 62608},
{1500407838213, 76489},
{1500407848203, 22147},
{1500407858253, 92829},
{1500407868073, 48499},
{1500407878053, 89152},
{1500407888073, 9191},
{1500407898033, 49881},
{1500407908113, 96020},
{1500407918213, 90203},
{1500407928234, 32217},
{1500407938253, 94302},
{1500407948293, 83111},
{1500407958234, 75576},
{1500407968073, 5973},
{1500407978023, 5175},
{1500407987923, 63350},
{1500407997833, 44081},
}

View File

@ -1,385 +0,0 @@
// Package tsz implement time-series compression
/*
http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
*/
package tsz
import (
"bytes"
"encoding/binary"
"io"
"math"
"math/bits"
"sync"
)
// Series is the basic series primitive
// you can concurrently put values, finish the stream, and create iterators
type Series struct {
sync.Mutex
T0 uint64
t uint64
val float64
bw bstream
leading uint8
trailing uint8
finished bool
tDelta uint32
}
// New series
func New(t0 uint64) *Series {
s := Series{
T0: t0,
leading: ^uint8(0),
}
// block header
s.bw.writeBits(uint64(t0), 64)
return &s
}
// Bytes value of the series stream
func (s *Series) Bytes() []byte {
s.Lock()
defer s.Unlock()
return s.bw.bytes()
}
func finish(w *bstream) {
// write an end-of-stream record
w.writeBits(0x0f, 4)
w.writeBits(0xffffffff, 32)
w.writeBit(zero)
}
// Finish the series by writing an end-of-stream record
func (s *Series) Finish() {
s.Lock()
if !s.finished {
finish(&s.bw)
s.finished = true
}
s.Unlock()
}
// Push a timestamp and value to the series.
// Values must be inserted in monotonically increasing time order.
func (s *Series) Push(t uint64, v float64) {
s.Lock()
defer s.Unlock()
if s.t == 0 {
// first point
s.t = t
s.val = v
s.tDelta = uint32(t - s.T0)
s.bw.writeBits(uint64(s.tDelta), 27)
s.bw.writeBits(math.Float64bits(v), 64)
return
}
// Difference to the original Facebook paper, we store the first delta as 27
// bits to allow millisecond accuracy for a one day block.
tDelta := uint32(t - s.t)
d := int32(tDelta - s.tDelta)
if d == 0 {
s.bw.writeBit(zero)
} else {
// Increase by one in the decompressing phase as we have one free bit
switch dod := encodeZigZag32(d) - 1; 32 - bits.LeadingZeros32(dod) {
case 1, 2, 3, 4, 5, 6, 7:
s.bw.writeBits(uint64(dod|256), 9) // dod | 00000000000000000000000100000000
case 8, 9:
s.bw.writeBits(uint64(dod|3072), 12) // dod | 00000000000000000000110000000000
case 10, 11, 12:
s.bw.writeBits(uint64(dod|57344), 16) // dod | 00000000000000001110000000000000
default:
s.bw.writeBits(0x0f, 4) // '1111'
s.bw.writeBits(uint64(dod), 32)
}
}
vDelta := math.Float64bits(s.val) ^ math.Float64bits(v)
if vDelta == 0 {
s.bw.writeBit(zero)
} else {
leading := uint8(bits.LeadingZeros64(vDelta))
trailing := uint8(bits.TrailingZeros64(vDelta))
s.bw.writeBit(one)
if leading >= s.leading && trailing >= s.trailing {
s.bw.writeBit(zero)
s.bw.writeBits(vDelta>>s.trailing, 64-int(s.leading)-int(s.trailing))
} else {
s.bw.writeBit(one)
// Different from version 1.x, use (significantBits - 1) in storage - avoids a branch
sigbits := 64 - leading - trailing
// Different from original, bits 5 -> 6, avoids a branch, allows storing small longs
s.bw.writeBits(uint64(leading), 6) // Number of leading zeros in the next 6 bits
s.bw.writeBits(uint64(sigbits-1), 6) // Length of meaningful bits in the next 6 bits
s.bw.writeBits(vDelta>>trailing, int(sigbits)) // Store the meaningful bits of XOR
s.leading, s.trailing = leading, trailing
}
}
s.tDelta = tDelta
s.t = t
s.val = v
}
// Iter lets you iterate over a series. It is not concurrency-safe.
func (s *Series) Iter() *Iter {
s.Lock()
w := s.bw.clone()
s.Unlock()
finish(w)
iter, _ := bstreamIterator(w)
return iter
}
// Iter lets you iterate over a series. It is not concurrency-safe.
type Iter struct {
T0 uint64
t uint64
val float64
br bstream
leading uint8
trailing uint8
finished bool
tDelta uint32
err error
}
func bstreamIterator(br *bstream) (*Iter, error) {
br.count = 8
t0, err := br.readBits(64)
if err != nil {
return nil, err
}
return &Iter{
T0: uint64(t0),
br: *br,
}, nil
}
// NewIterator for the series
func NewIterator(b []byte) (*Iter, error) {
return bstreamIterator(newBReader(b))
}
// Next iteration of the series iterator
func (it *Iter) Next() bool {
if it.err != nil || it.finished {
return false
}
if it.t == 0 {
// read first t and v
tDelta, err := it.br.readBits(27)
if err != nil {
it.err = err
return false
}
if tDelta == (1<<27)-1 {
it.finished = true
return false
}
it.tDelta = uint32(tDelta)
it.t = it.T0 + tDelta
v, err := it.br.readBits(64)
if err != nil {
it.err = err
return false
}
it.val = math.Float64frombits(v)
return true
}
// read delta-of-delta
d, err := it.br.readUntilZero(4)
if err != nil {
it.err = err
return false
}
if d != 0 {
var sz uint
switch d {
case 0x02:
sz = 7
case 0x06:
sz = 9
case 0x0e:
sz = 12
case 0x0f:
sz = 32
}
bits, err := it.br.readBits(int(sz))
if err != nil {
it.err = err
return false
}
if sz == 32 && bits == 0xffffffff {
it.finished = true
return false
}
dod := decodeZigZag32(uint32(int64(bits) + 1))
it.tDelta += uint32(dod)
}
it.t += uint64(it.tDelta)
val, err := it.br.readUntilZero(2)
if err != nil {
it.err = err
return false
}
switch val {
case 3:
bits, err := it.br.readBits(6)
if err != nil {
it.err = err
return false
}
it.leading = uint8(bits)
bits, err = it.br.readBits(6)
if err != nil {
it.err = err
return false
}
it.trailing = 64 - (uint8(bits) + 1) - it.leading
fallthrough
case 2:
bits, err := it.br.readBits(int(64 - it.leading - it.trailing))
if err != nil {
it.err = err
return false
}
it.val = math.Float64frombits(math.Float64bits(it.val) ^ (bits << it.trailing))
}
return true
}
// Values at the current iterator position
func (it *Iter) Values() (uint64, float64) {
return it.t, it.val
}
// Err error at the current iterator position
func (it *Iter) Err() error {
return it.err
}
type errMarshal struct {
w io.Writer
r io.Reader
err error
}
func (em *errMarshal) write(t interface{}) {
if em.err != nil {
return
}
em.err = binary.Write(em.w, binary.BigEndian, t)
}
func (em *errMarshal) read(t interface{}) {
if em.err != nil {
return
}
em.err = binary.Read(em.r, binary.BigEndian, t)
}
// MarshalBinary implements the encoding.BinaryMarshaler interface
func (s *Series) MarshalBinary() ([]byte, error) {
buf := new(bytes.Buffer)
em := &errMarshal{w: buf}
em.write(s.T0)
em.write(s.leading)
em.write(s.t)
em.write(s.tDelta)
em.write(s.trailing)
em.write(s.val)
bStream, err := s.bw.MarshalBinary()
if err != nil {
return nil, err
}
em.write(bStream)
if em.err != nil {
return nil, em.err
}
return buf.Bytes(), nil
}
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (s *Series) UnmarshalBinary(b []byte) error {
buf := bytes.NewReader(b)
em := &errMarshal{r: buf}
em.read(&s.T0)
em.read(&s.leading)
em.read(&s.t)
em.read(&s.tDelta)
em.read(&s.trailing)
em.read(&s.val)
outBuf := make([]byte, buf.Len())
em.read(outBuf)
err := s.bw.UnmarshalBinary(outBuf)
if err != nil {
return err
}
if em.err != nil {
return em.err
}
return nil
}
// Maps negative values to positive values while going back and
// forth (0 = 0, -1 = 1, 1 = 2, -2 = 3, 2 = 4, -3 = 5, 3 = 6 ...)
// Encodes signed integers into unsigned integers that can be efficiently
// encoded with varint because negative values must be sign-extended to 64 bits to
// be varint encoded, thus always taking 10 bytes on the wire.
//
// Read more: https://gist.github.com/mfuerstenau/ba870a29e16536fdbaba
func encodeZigZag32(n int32) uint32 {
// Note: The right-shift must be arithmetic which it is in Go.
return uint32(n>>31) ^ (uint32(n) << 1)
}
func decodeZigZag32(n uint32) int32 {
return int32((n >> 1) ^ uint32((int32(n&1)<<31)>>31))
}

View File

@ -1,34 +0,0 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
vegeta
vegeta.test
vegeta-*.tar.gz
lib/lib.test
dist
vendor
*.gob
*.lz
.DS_Store

View File

@ -1,35 +0,0 @@
builds:
- goos:
- freebsd
- linux
- darwin
- windows
goarch:
- amd64
- arm64
- arm
goarm:
- 7
env:
- CGO_ENABLED=0
flags:
- -a
- -tags="netgo"
ldflags:
- -s -w -extldflags "-static" -X main.Version={{.Version}} -X main.Commit={{.Commit}} -X main.Date={{.Date}}"
release:
name_template: "v{{.Version}}"
archive:
name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}-{{.Arch}}"
format: tar.gz
format_overrides:
- goos: windows
format: zip
checksum:
name_template: "{{ .ProjectName }}-checksums.txt"
sign:
artifacts: checksum

View File

@ -1,12 +0,0 @@
language: go
sudo: false
go:
- tip
install:
- go get -v golang.org/x/lint/golint
- go get -d -t -v ./...
- go build -v ./...
script:
- go vet ./...
- $HOME/gopath/bin/golint -set_exit_status $(go list ./... | grep -v /vendor/)
- go test -v ./...

View File

@ -1,87 +0,0 @@
2018-05-18: v7.0.0
Include response body in hit results (#279)
Added support for h2c requests (HTTP/2 without TLS) (#261)
Prevent "null" Metrics.Errors JSON encoding (#277)
Add option to override HTTP Proxy on Attacker (#234)
2017-03-19: v6.3.0
Mark responses as success in no redirect following mode (#222)
2017-03-04: v6.2.0
Allow any upper-case ASCII word to be an HTTP method (#217)
Correctly compute Metrics.Rate with sub-second duration results (#208)
2016-08-26: v6.1.1
Respect case sensitivity in target file header names (#195, #191)
2016-04-03: v6.1.0
Add HTTP2 support
2015-11-27: v6.0.0
Insecure attack flag (#160)
Client certificates (#156)
Infinite attacks (#155)
Allow empty lines between targets (#147)
2015-09-19: v5.9.0
Bounded memory streaming reporters (#136)
2015-09-04: v5.8.1
Fix support for DELETE methods in targets
2015-08-11: v5.8.0
Change reporters quantile estimation method to match R's 8th type.
2015-05-23: v5.7.1
Revert end-to-end attack timeout change
2015-05-23: v5.7.0
Allow case sensitve headers in attacks
2015-04-15: v5.6.3
Expose connections flag in the attack command
Add global cpu and heap profiling flags
Measure actual attack rate and print it in relevant reporters
Major performance improvements that allow much higher attack rates
2015-04-02: v5.6.2
Update dygraph to latest version
Improve plot reporter screenshot rendering by using html2canvas.js
Improve plot reporter performance
2015-03-23: v5.6.1
Allow spaces in hist reporter flag format
2015-03-12: v5.6.0
Set default dumper to "json" in the dump command.
Add --version to global vegeta command flags.
Fix response body leak regression introduced in v5.5.3.
2015-03-11: v5.5.3
Always read response bodies for each request.
Homebrew install instructions.
2015-01-3: v5.5.2
Refactor core request logic and simplify tests with a 4x speedup.
2015-01-2: v5.5.1
Treat bad status codes as errors.
2014-11-21: v5.5.0
Implement dump command with CSV and JSON record format.
Optionally ignore redirects and treat them as successes.
2014-11-16: v5.4.0
Add histogram reporter to the report command.
2014-11-16: v5.3.0
Add support for extended targets dsl that supports per-target headers and body.
Target file comments support has been removed.
2014-11-7: v5.2.0
Don't treat 3xx status codes as errors.
Add -keepalive flag to the attack command.
2014-11-3: v5.1.1
Add FreeBSD and Windows releases.
Fix non termination bug in the report command. #85

View File

@ -1,169 +0,0 @@
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
[[projects]]
branch = "master"
digest = "1:c312c313bb8040fcfb5698f38b51ec279e4635c7619773e6f018c908586c64b1"
name = "github.com/alecthomas/jsonschema"
packages = ["."]
pruneopts = "UT"
revision = "f2c93856175a7dd6abe88c5c3900b67ad054adcc"
[[projects]]
branch = "master"
digest = "1:f31b92722c6eca05df9ba28e9cb38d94de0865f034fff164557f103d6d13b612"
name = "github.com/bmizerany/perks"
packages = ["quantile"]
pruneopts = "UT"
revision = "d9a9656a3a4b1c2864fdb44db2ef8619772d92aa"
[[projects]]
branch = "master"
digest = "1:e8891cd354f2af940345f895d5698ab1ed439196d4f69eb44817fc91e80354c1"
name = "github.com/c2h5oh/datasize"
packages = ["."]
pruneopts = "UT"
revision = "4eba002a5eaea69cf8d235a388fc6b65ae68d2dd"
[[projects]]
branch = "master"
digest = "1:5c4cefdd3ba39aee4e4b6f9aad121a79fd2a989b9b54acbe4f530cc3a700cd75"
name = "github.com/dgryski/go-gk"
packages = ["."]
pruneopts = "UT"
revision = "201884a44051d8544b65e6b1c05ae71c0c45b9e0"
[[projects]]
branch = "master"
digest = "1:bc4e2b6c1af97c03fe850864bf22574abc6ad0fcbf0e5dda0fd246d3d43f9ccb"
name = "github.com/dgryski/go-lttb"
packages = ["."]
pruneopts = "UT"
revision = "318fcdf10a77c2df09d83777e41f5e18d236ec9d"
[[projects]]
digest = "1:2e3c336fc7fde5c984d2841455a658a6d626450b1754a854b3b32e7a8f49a07a"
name = "github.com/google/go-cmp"
packages = [
"cmp",
"cmp/internal/diff",
"cmp/internal/function",
"cmp/internal/value",
]
pruneopts = "UT"
revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
version = "v0.2.0"
[[projects]]
branch = "master"
digest = "1:b07e7ba2bb99a70ce3318ea387a62666371c1ab4f9bfcae192a59e3f4b8ffa56"
name = "github.com/influxdata/tdigest"
packages = ["."]
pruneopts = "UT"
revision = "a7d76c6f093a59b94a01c6c2b8429122d444a8cc"
[[projects]]
branch = "master"
digest = "1:aa3d8d42865c42626b5c1add193692d045b3188b1479f0a0a88690d21fe20083"
name = "github.com/mailru/easyjson"
packages = [
".",
"buffer",
"jlexer",
"jwriter",
]
pruneopts = "UT"
revision = "60711f1a8329503b04e1c88535f419d0bb440bff"
[[projects]]
branch = "master"
digest = "1:7ca2584fa7da0520cd2d1136a10194fe5a5b220bdb215074ab6f7b5ad91115f4"
name = "github.com/shurcooL/httpfs"
packages = ["vfsutil"]
pruneopts = "UT"
revision = "809beceb23714880abc4a382a00c05f89d13b1cc"
[[projects]]
branch = "master"
digest = "1:b5db31f108c72b9de4e36ca3222a4e325beebfa492a9d3f62f313ffa9cdab3b7"
name = "github.com/shurcooL/vfsgen"
packages = ["."]
pruneopts = "UT"
revision = "62bca832be04bd2bcaabd3b68a6b19a7ec044411"
[[projects]]
branch = "master"
digest = "1:383e656d30a7bd2c663514940748c189307134c65ec8c7b2070ed01abb93e73d"
name = "github.com/streadway/quantile"
packages = ["."]
pruneopts = "UT"
revision = "b0c588724d25ae13f5afb3d90efec0edc636432b"
[[projects]]
branch = "master"
digest = "1:be87c1e6f0a15e5b1c45036b9382cdd63057a8b65e10ff2fe2bda23886a99c6a"
name = "github.com/tsenart/go-tsz"
packages = [
".",
"testdata",
]
pruneopts = "UT"
revision = "cdeb9e1e981e85dafc95428f5da9cba59cbcc828"
[[projects]]
branch = "master"
digest = "1:dca4094c3be476189bd5854752edbd1bd3d15e61d9570dd9133744bb1ecd599d"
name = "golang.org/x/net"
packages = [
"http/httpguts",
"http2",
"http2/hpack",
"idna",
]
pruneopts = "UT"
revision = "c39426892332e1bb5ec0a434a079bf82f5d30c54"
[[projects]]
digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18"
name = "golang.org/x/text"
packages = [
"collate",
"collate/build",
"internal/colltab",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"language",
"secure/bidirule",
"transform",
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable",
]
pruneopts = "UT"
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/alecthomas/jsonschema",
"github.com/bmizerany/perks/quantile",
"github.com/c2h5oh/datasize",
"github.com/dgryski/go-gk",
"github.com/dgryski/go-lttb",
"github.com/google/go-cmp/cmp",
"github.com/influxdata/tdigest",
"github.com/mailru/easyjson",
"github.com/mailru/easyjson/jlexer",
"github.com/mailru/easyjson/jwriter",
"github.com/shurcooL/vfsgen",
"github.com/streadway/quantile",
"github.com/tsenart/go-tsz",
"golang.org/x/net/http2",
]
solver-name = "gps-cdcl"
solver-version = 1

View File

@ -1,74 +0,0 @@
# Gopkg.toml example
#
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
# for detailed Gopkg.toml documentation.
#
# required = ["github.com/user/thing/cmd/thing"]
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
#
# [[constraint]]
# name = "github.com/user/project"
# version = "1.0.0"
#
# [[constraint]]
# name = "github.com/user/project2"
# branch = "dev"
# source = "github.com/myfork/project2"
#
# [[override]]
# name = "github.com/x/y"
# version = "2.4.0"
#
# [prune]
# non-go = false
# go-tests = true
# unused-packages = true
[[constraint]]
branch = "master"
name = "github.com/alecthomas/jsonschema"
[[constraint]]
branch = "master"
name = "github.com/bmizerany/perks"
[[constraint]]
branch = "master"
name = "github.com/dgryski/go-gk"
[[constraint]]
branch = "master"
name = "github.com/dgryski/go-lttb"
[[constraint]]
branch = "master"
name = "github.com/tsenart/go-tsz"
[[constraint]]
branch = "master"
name = "github.com/influxdata/tdigest"
[[constraint]]
branch = "master"
name = "github.com/streadway/quantile"
[[constraint]]
branch = "master"
name = "golang.org/x/net"
[[constraint]]
branch = "master"
name = "github.com/shurcooL/vfsgen"
[prune]
go-tests = true
unused-packages = true
[[constraint]]
branch = "master"
name = "github.com/c2h5oh/datasize"
[[constraint]]
branch = "master"
name = "github.com/mailru/easyjson"

View File

@ -1,20 +0,0 @@
The MIT License (MIT)
Copyright (c) 2013-2016 Tomás Senart
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View File

@ -1,25 +0,0 @@
COMMIT=$(shell git rev-parse HEAD)
VERSION=$(shell git describe --tags --exact-match --always)
DATE=$(shell date +'%FT%TZ%z')
vegeta: vendor generate
CGO_ENABLED=0 go build -v -a -tags=netgo \
-ldflags '-s -w -extldflags "-static" -X main.Version=$(VERSION) -X main.Commit=$(COMMIT) -X main.Date=$(DATE)'
clean-vegeta:
rm vegeta
generate: vendor
go generate ./...
vendor:
dep ensure -v
clean-vendor:
rm -rf vendor
dist:
goreleaser release --debug --skip-validate
clean-dist:
rm -rf dist

View File

@ -1,803 +0,0 @@
# Vegeta [![Build Status](https://secure.travis-ci.org/tsenart/vegeta.svg?branch=master)](http://travis-ci.org/tsenart/vegeta) [![Go Report Card](https://goreportcard.com/badge/github.com/tsenart/vegeta)](https://goreportcard.com/report/github.com/tsenart/vegeta) [![GoDoc](https://godoc.org/github.com/tsenart/vegeta?status.svg)](https://godoc.org/github.com/tsenart/vegeta) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/tsenart/vegeta?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Donate](https://img.shields.io/badge/donate-bitcoin-yellow.svg)](#donate)
Vegeta is a versatile HTTP load testing tool built out of a need to drill
HTTP services with a constant request rate.
It can be used both as a command line utility and a library.
![Vegeta](http://fc09.deviantart.net/fs49/i/2009/198/c/c/ssj2_vegeta_by_trunks24.jpg)
## Install
### Pre-compiled executables
Get them [here](http://github.com/tsenart/vegeta/releases).
### Homebrew on Mac OS X
You can install Vegeta using the [Homebrew](https://github.com/Homebrew/homebrew/) package manager on Mac OS X:
```shell
$ brew update && brew install vegeta
```
### Source
You need `go` installed and `GOBIN` in your `PATH`. Once that is done, run the
command:
```shell
$ go get -u github.com/tsenart/vegeta
```
## Versioning
Both the library and the CLI are versioned with [SemVer v2.0.0](https://semver.org/spec/v2.0.0.html).
After [v8.0.0](https://github.com/tsenart/vegeta/tree/v8.0.0), the two components
are versioned separately to better isolate breaking changes to each.
CLI releases are tagged with `cli/vMAJOR.MINOR.PATCH` and published on the [Github releases page](https://github.com/tsenart/vegeta/releases).
As for the library, new versions are tagged with both `lib/vMAJOR.MINOR.PATCH` and `vMAJOR.MINOR.PATCH`.
The latter tag is required for compatibility with `go mod`.
## Contributing
See [CONTRIBUTING.md](.github/CONTRIBUTING.md).
## Usage manual
```console
Usage: vegeta [global flags] <command> [command flags]
global flags:
-cpus int
Number of CPUs to use (defaults to the number of CPUs you have)
-profile string
Enable profiling of [cpu, heap]
-version
Print version and exit
attack command:
-body string
Requests body file
-cert string
TLS client PEM encoded certificate file
-connections int
Max open idle connections per target host (default 10000)
-duration duration
Duration of the test [0 = forever]
-format string
Targets format [http, json] (default "http")
-h2c
Send HTTP/2 requests without TLS encryption
-header value
Request header
-http2
Send HTTP/2 requests when supported by the server (default true)
-insecure
Ignore invalid server TLS certificates
-keepalive
Use persistent connections (default true)
-key string
TLS client PEM encoded private key file
-laddr value
Local IP address (default 0.0.0.0)
-lazy
Read targets lazily
-max-body value
Maximum number of bytes to capture from response bodies. [-1 = no limit] (default -1)
-max-workers uint
Maximum number of workers (default 18446744073709551615)
-name string
Attack name
-output string
Output file (default "stdout")
-rate value
Number of requests per time unit [0 = infinity] (default 50/1s)
-redirects int
Number of redirects to follow. -1 will not follow but marks as success (default 10)
-resolvers value
List of addresses (ip:port) to use for DNS resolution. Disables use of local system DNS. (comma separated list)
-root-certs value
TLS root certificate files (comma separated list)
-targets string
Targets file (default "stdin")
-timeout duration
Requests timeout (default 30s)
-unix-socket string
Connect over a unix socket. This overrides the host address in target URLs
-workers uint
Initial number of workers (default 10)
encode command:
-output string
Output file (default "stdout")
-to string
Output encoding [csv, gob, json] (default "json")
plot command:
-output string
Output file (default "stdout")
-threshold int
Threshold of data points above which series are downsampled. (default 4000)
-title string
Title and header of the resulting HTML page (default "Vegeta Plot")
report command:
-buckets string
Histogram buckets, e.g.: "[0,1ms,10ms]"
-every duration
Report interval
-output string
Output file (default "stdout")
-type string
Report type to generate [text, json, hist[buckets], hdrplot] (default "text")
examples:
echo "GET http://localhost/" | vegeta attack -duration=5s | tee results.bin | vegeta report
vegeta report -type=json results.bin > metrics.json
cat results.bin | vegeta plot > plot.html
cat results.bin | vegeta report -type="hist[0,100ms,200ms,300ms]"
```
#### `-cpus`
Specifies the number of CPUs to be used internally.
It defaults to the amount of CPUs available in the system.
#### `-profile`
Specifies which profiler to enable during execution. Both _cpu_ and
_heap_ profiles are supported. It defaults to none.
#### `-version`
Prints the version and exits.
### `attack` command
#### `-body`
Specifies the file whose content will be set as the body of every
request unless overridden per attack target, see `-targets`.
#### `-cert`
Specifies the PEM encoded TLS client certificate file to be used with HTTPS requests.
If `-key` isn't specified, it will be set to the value of this flag.
#### `-connections`
Specifies the maximum number of idle open connections per target host.
#### `-duration`
Specifies the amount of time to issue request to the targets.
The internal concurrency structure's setup has this value as a variable.
The actual run time of the test can be longer than specified due to the
responses delay. Use 0 for an infinite attack.
#### `-format`
Specifies the targets format to decode.
##### `json` format
The JSON format makes integration with programs that produce targets dynamically easier.
Each target is one JSON object in its own line. The method and url fields are required.
If present, the body field must be base64 encoded. The generated [JSON Schema](lib/target.schema.json)
defines the format in detail.
```bash
jq -ncM '{method: "GET", url: "http://goku", body: "Punch!" | @base64, header: {"Content-Type": ["text/plain"]}}' |
vegeta attack -format=json -rate=100 | vegeta encode
```
##### `http` format
The http format almost resembles the plain-text HTTP message format defined in
[RFC 2616](https://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html) but it
doesn't support in-line HTTP bodies, only references to files that are loaded and used
as request bodies (as exemplified below).
Although targets in this format can be produced by other programs, it was originally
meant to be used by people writing targets by hand for simple use cases.
Here are a few examples of valid targets files in the http format:
###### Simple targets
```
GET http://goku:9090/path/to/dragon?item=ball
GET http://user:password@goku:9090/path/to
HEAD http://goku:9090/path/to/success
```
###### Targets with custom headers
```
GET http://user:password@goku:9090/path/to
X-Account-ID: 8675309
DELETE http://goku:9090/path/to/remove
Confirmation-Token: 90215
Authorization: Token DEADBEEF
```
###### Targets with custom bodies
```
POST http://goku:9090/things
@/path/to/newthing.json
PATCH http://goku:9090/thing/71988591
@/path/to/thing-71988591.json
```
###### Targets with custom bodies and headers
```
POST http://goku:9090/things
X-Account-ID: 99
@/path/to/newthing.json
```
###### Add comments to the targets
Lines starting with `#` are ignored.
```
# get a dragon ball
GET http://goku:9090/path/to/dragon?item=ball
```
#### `-h2c`
Specifies that HTTP2 requests are to be sent over TCP without TLS encryption.
#### `-header`
Specifies a request header to be used in all targets defined, see `-targets`.
You can specify as many as needed by repeating the flag.
#### `-http2`
Specifies whether to enable HTTP/2 requests to servers which support it.
#### `-insecure`
Specifies whether to ignore invalid server TLS certificates.
#### `-keepalive`
Specifies whether to reuse TCP connections between HTTP requests.
#### `-key`
Specifies the PEM encoded TLS client certificate private key file to be
used with HTTPS requests.
#### `-laddr`
Specifies the local IP address to be used.
#### `-lazy`
Specifies whether to read the input targets lazily instead of eagerly.
This allows streaming targets into the attack command and reduces memory
footprint.
The trade-off is one of added latency in each hit against the targets.
#### `-max-body`
Specifies the maximum number of bytes to capture from the body of each
response. Remaining unread bytes will be fully read but discarded.
Set to -1 for no limit. It knows how to intepret values like these:
- `"10 MB"` -> `10MB`
- `"10240 g"` -> `10TB`
- `"2000"` -> `2000B`
- `"1tB"` -> `1TB`
- `"5 peta"` -> `5PB`
- `"28 kilobytes"` -> `28KB`
- `"1 gigabyte"` -> `1GB`
#### `-name`
Specifies the name of the attack to be recorded in responses.
#### `-output`
Specifies the output file to which the binary results will be written
to. Made to be piped to the report command input. Defaults to stdout.
#### `-rate`
Specifies the request rate per time unit to issue against
the targets. The actual request rate can vary slightly due to things like
garbage collection, but overall it should stay very close to the specified.
If no time unit is provided, 1s is used.
A `-rate` of `0` or `infinity` means vegeta will send requests as fast as possible.
Use together with `-max-workers` to model a fixed set of concurrent users sending
requests serially (i.e. waiting for a response before sending the next request).
Setting `-max-workers` to a very high number while setting `-rate=0` can result in
vegeta consuming too many resources and crashing. Use with care.
#### `-redirects`
Specifies the max number of redirects followed on each request. The
default is 10. When the value is -1, redirects are not followed but
the response is marked as successful.
#### `-resolvers`
Specifies custom DNS resolver addresses to use for name resolution instead of
the ones configured by the operating system. Works only on non Windows systems.
#### `-root-certs`
Specifies the trusted TLS root CAs certificate files as a comma separated
list. If unspecified, the default system CAs certificates will be used.
#### `-targets`
Specifies the file from which to read targets, defaulting to stdin.
See the [`-format`](#-format) section to learn about the different target formats.
#### `-timeout`
Specifies the timeout for each request. The default is 0 which disables
timeouts.
#### `-workers`
Specifies the initial number of workers used in the attack. The actual
number of workers will increase if necessary in order to sustain the
requested rate, unless it'd go beyond `-max-workers`.
#### `-max-workers`
Specifies the maximum number of workers used in the attack. It can be used to
control the concurrency level used by an attack.
### `report` command
```console
Usage: vegeta report [options] [<file>...]
Outputs a report of attack results.
Arguments:
<file> A file with vegeta attack results encoded with one of
the supported encodings (gob | json | csv) [default: stdin]
Options:
--type Which report type to generate (text | json | hist[buckets] | hdrplot).
[default: text]
--buckets Histogram buckets, e.g.: '[0,1ms,10ms]'
--every Write the report to --output at every given interval (e.g 100ms)
The default of 0 means the report will only be written after
all results have been processed. [default: 0]
--output Output file [default: stdout]
Examples:
echo "GET http://:80" | vegeta attack -rate=10/s > results.gob
echo "GET http://:80" | vegeta attack -rate=100/s | vegeta encode > results.json
vegeta report results.*
```
#### `report -type=text`
```console
Requests [total, rate, throughput] 1200, 120.00, 65.87
Duration [total, attack, wait] 10.094965987s, 9.949883921s, 145.082066ms
Latencies [mean, 50, 95, 99, max] 113.172398ms, 108.272568ms, 140.18235ms, 247.771566ms, 264.815246ms
Bytes In [total, mean] 3714690, 3095.57
Bytes Out [total, mean] 0, 0.00
Success [ratio] 55.42%
Status Codes [code:count] 0:535 200:665
Error Set:
Get http://localhost:6060: dial tcp 127.0.0.1:6060: connection refused
Get http://localhost:6060: read tcp 127.0.0.1:6060: connection reset by peer
Get http://localhost:6060: dial tcp 127.0.0.1:6060: connection reset by peer
Get http://localhost:6060: write tcp 127.0.0.1:6060: broken pipe
Get http://localhost:6060: net/http: transport closed before response was received
Get http://localhost:6060: http: can't write HTTP request on broken connection
```
The `Requests` row shows:
- The `total` number of issued requests.
- The real request `rate` sustained during the `attack` period.
- The `throughput` of successful requests over the `total` period.
The `Duration` row shows:
- The `attack` time taken issuing all requests (`total` - `wait`)
- The `wait` time waiting for the response to the last issued request (`total` - `attack`)
- The `total` time taken in the attack (`attack` + `wait`)
Latency is the amount of time taken for a response to a request to be read (including the `-max-body` bytes from the response body).
- `mean` is the [arithmetic mean / average](https://en.wikipedia.org/wiki/Arithmetic_mean) of the latencies of all requests in an attack.
- `50`, `95`, `99` are the 50th, 95th an 99th [percentiles](https://en.wikipedia.org/wiki/Percentile), respectively, of the latencies of all requests in an attack. To understand more about why these are useful, I recommend [this article](https://bravenewgeek.com/everything-you-know-about-latency-is-wrong/) from @tylertreat.
- `max` is the maximum latency of all requests in an attack.
The `Bytes In` and `Bytes Out` rows shows:
- The `total` number of bytes sent (out) or received (in) with the request or response bodies.
- The `mean` number of bytes sent (out) or received (in) with the request or response bodies.
The `Success` ratio shows the percentage of requests whose responses didn't error and had status codes between **200** and **400** (non-inclusive).
The `Status Codes` row shows a histogram of status codes. `0` status codes mean a request failed to be sent.
The `Error Set` shows a unique set of errors returned by all issued requests. These include requests that got non-successful response status code.
#### `report -type=json`
All duration like fields are in nanoseconds.
```json
{
"latencies": {
"total": 237119463,
"mean": 2371194,
"50th": 2854306,
"95th": 3478629,
"99th": 3530000,
"max": 3660505
},
"buckets": {"0":9952,"1000000":40,"2000000":6,"3000000":0,"4000000":0,"5000000":2},
"bytes_in": {
"total": 606700,
"mean": 6067
},
"bytes_out": {
"total": 0,
"mean": 0
},
"earliest": "2015-09-19T14:45:50.645818631+02:00",
"latest": "2015-09-19T14:45:51.635818575+02:00",
"end": "2015-09-19T14:45:51.639325797+02:00",
"duration": 989999944,
"wait": 3507222,
"requests": 100,
"rate": 101.01010672380401,
"throughput": 101.00012489812,
"success": 1,
"status_codes": {
"200": 100
},
"errors": []
}
```
In the `buckets` field, each key is a nanosecond value representing the lower bound of a bucket.
The upper bound is implied by the next higher bucket.
Upper bounds are non-inclusive.
The highest bucket is the overflow bucket; it has no upper bound.
The values are counts of how many requests fell into that particular bucket.
If the `-buckets` parameter is not present, the `buckets` field is omitted.
#### `report -type=hist`
Computes and prints a text based histogram for the given buckets.
Each bucket upper bound is non-inclusive.
```console
cat results.bin | vegeta report -type='hist[0,2ms,4ms,6ms]'
Bucket # % Histogram
[0, 2ms] 6007 32.65% ########################
[2ms, 4ms] 5505 29.92% ######################
[4ms, 6ms] 2117 11.51% ########
[6ms, +Inf] 4771 25.93% ###################
```
#### `report -type=hdrplot`
Writes out results in a format plottable by https://hdrhistogram.github.io/HdrHistogram/plotFiles.html.
```
Value(ms) Percentile TotalCount 1/(1-Percentile)
0.076715 0.000000 0 1.000000
0.439370 0.100000 200 1.111111
0.480836 0.200000 400 1.250000
0.495559 0.300000 599 1.428571
0.505101 0.400000 799 1.666667
0.513059 0.500000 999 2.000000
0.516664 0.550000 1099 2.222222
0.520455 0.600000 1199 2.500000
0.525008 0.650000 1299 2.857143
0.530174 0.700000 1399 3.333333
0.534891 0.750000 1499 4.000000
0.537572 0.775000 1548 4.444444
0.540340 0.800000 1598 5.000000
0.543763 0.825000 1648 5.714286
0.547164 0.850000 1698 6.666667
0.551432 0.875000 1748 8.000000
0.553444 0.887500 1773 8.888889
0.555774 0.900000 1798 10.000000
0.558454 0.912500 1823 11.428571
0.562123 0.925000 1848 13.333333
0.565563 0.937500 1873 16.000000
0.567831 0.943750 1886 17.777778
0.570617 0.950000 1898 20.000000
0.574522 0.956250 1911 22.857143
0.579046 0.962500 1923 26.666667
0.584426 0.968750 1936 32.000000
0.586695 0.971875 1942 35.555556
0.590451 0.975000 1948 40.000000
0.597543 0.978125 1954 45.714286
0.605637 0.981250 1961 53.333333
0.613564 0.984375 1967 64.000000
0.620393 0.985938 1970 71.113640
0.629121 0.987500 1973 80.000000
0.638060 0.989062 1976 91.424392
0.648085 0.990625 1979 106.666667
0.659689 0.992188 1982 128.008193
0.665870 0.992969 1984 142.227279
0.672985 0.993750 1986 160.000000
0.680101 0.994531 1987 182.848784
0.687810 0.995313 1989 213.356091
0.695729 0.996094 1990 256.016385
0.730641 0.996484 1991 284.414107
0.785516 0.996875 1992 320.000000
0.840392 0.997266 1993 365.764448
1.009646 0.997656 1993 426.621160
1.347020 0.998047 1994 512.032770
1.515276 0.998242 1994 568.828214
1.683532 0.998437 1995 639.795266
1.887487 0.998633 1995 731.528895
2.106249 0.998828 1996 853.242321
2.325011 0.999023 1996 1023.541453
2.434952 0.999121 1996 1137.656428
2.544894 0.999219 1996 1280.409731
2.589510 0.999316 1997 1461.988304
2.605192 0.999414 1997 1706.484642
2.620873 0.999512 1997 2049.180328
2.628713 0.999561 1997 2277.904328
2.636394 0.999609 1997 2557.544757
2.644234 0.999658 1997 2923.976608
2.652075 0.999707 1997 3412.969283
2.658916 0.999756 1998 4098.360656
2.658916 0.999780 1998 4545.454545
2.658916 0.999805 1998 5128.205128
2.658916 0.999829 1998 5847.953216
2.658916 0.999854 1998 6849.315068
2.658916 0.999878 1998 8196.721311
2.658916 0.999890 1998 9090.909091
2.658916 0.999902 1998 10204.081633
2.658916 0.999915 1998 11764.705882
2.658916 0.999927 1998 13698.630137
2.658916 0.999939 1998 16393.442623
2.658916 0.999945 1998 18181.818182
2.658916 0.999951 1998 20408.163265
2.658916 0.999957 1998 23255.813953
2.658916 0.999963 1998 27027.027027
2.658916 0.999969 1998 32258.064516
2.658916 0.999973 1998 37037.037037
2.658916 0.999976 1998 41666.666667
2.658916 0.999979 1998 47619.047619
2.658916 0.999982 1998 55555.555556
2.658916 0.999985 1998 66666.666667
2.658916 0.999986 1998 71428.571429
2.658916 0.999988 1998 83333.333333
2.658916 0.999989 1998 90909.090909
2.658916 0.999991 1998 111111.111111
2.658916 0.999992 1998 125000.000000
2.658916 0.999993 1998 142857.142858
2.658916 0.999994 1998 166666.666668
2.658916 0.999995 1998 199999.999999
2.658916 0.999996 1998 250000.000000
2.658916 0.999997 1998 333333.333336
2.658916 0.999998 1998 500000.000013
2.658916 0.999999 1998 999999.999971
2.658916 1.000000 1998 10000000.000000
```
### `encode` command
```
Usage: vegeta encode [options] [<file>...]
Encodes vegeta attack results from one encoding to another.
The supported encodings are Gob (binary), CSV and JSON.
Each input file may have a different encoding which is detected
automatically.
The CSV encoder doesn't write a header. The columns written by it are:
1. Unix timestamp in nanoseconds since epoch
2. HTTP status code
3. Request latency in nanoseconds
4. Bytes out
5. Bytes in
6. Error
7. Base64 encoded response body
8. Attack name
9. Sequence number of request
Arguments:
<file> A file with vegeta attack results encoded with one of
the supported encodings (gob | json | csv) [default: stdin]
Options:
--to Output encoding (gob | json | csv) [default: json]
--output Output file [default: stdout]
Examples:
echo "GET http://:80" | vegeta attack -rate=1/s > results.gob
cat results.gob | vegeta encode | jq -c 'del(.body)' | vegeta encode -to gob
```
### `plot` command
![Plot](https://i.imgur.com/Jra1sNH.png)
```
Usage: vegeta plot [options] [<file>...]
Outputs an HTML time series plot of request latencies over time.
The X axis represents elapsed time in seconds from the beginning
of the earliest attack in all input files. The Y axis represents
request latency in milliseconds.
Click and drag to select a region to zoom into. Double click to zoom out.
Choose a different number on the bottom left corner input field
to change the moving average window size (in data points).
Arguments:
<file> A file output by running vegeta attack [default: stdin]
Options:
--title Title and header of the resulting HTML page.
[default: Vegeta Plot]
--threshold Threshold of data points to downsample series to.
Series with less than --threshold number of data
points are not downsampled. [default: 4000]
Examples:
echo "GET http://:80" | vegeta attack -name=50qps -rate=50 -duration=5s > results.50qps.bin
cat results.50qps.bin | vegeta plot > plot.50qps.html
echo "GET http://:80" | vegeta attack -name=100qps -rate=100 -duration=5s > results.100qps.bin
vegeta plot results.50qps.bin results.100qps.bin > plot.html
```
## Usage: Generated targets
Apart from accepting a static list of targets, Vegeta can be used together with another program that generates them in a streaming fashion. Here's an example of that using the `jq` utility that generates targets with an incrementing id in their body.
```console
jq -ncM 'while(true; .+1) | {method: "POST", url: "http://:6060", body: {id: .} | @base64 }' | \
vegeta attack -rate=50/s -lazy -format=json -duration=30s | \
tee results.bin | \
vegeta report
```
## Usage: Distributed attacks
Whenever your load test can't be conducted due to Vegeta hitting machine limits
such as open files, memory, CPU or network bandwidth, it's a good idea to use Vegeta in a distributed manner.
In a hypothetical scenario where the desired attack rate is 60k requests per second,
let's assume we have 3 machines with `vegeta` installed.
Make sure open file descriptor and process limits are set to a high number for your user **on each machine**
using the `ulimit` command.
We're ready to start the attack. All we need to do is to divide the intended rate by the number of machines,
and use that number on each attack. Here we'll use [pdsh](https://code.google.com/p/pdsh/) for orchestration.
```shell
$ PDSH_RCMD_TYPE=ssh pdsh -b -w '10.0.1.1,10.0.2.1,10.0.3.1' \
'echo "GET http://target/" | vegeta attack -rate=20000 -duration=60s > result.bin'
```
After the previous command finishes, we can gather the result files to use on our report.
```shell
$ for machine in 10.0.1.1 10.0.2.1 10.0.3.1; do
scp $machine:~/result.bin $machine.bin &
done
```
The `report` command accepts multiple result files.
It'll read and sort them by timestamp before generating reports.
```console
vegeta report *.bin
```
## Usage: Real-time Analysis
If you are a happy user of iTerm, you can integrate vegeta with [jplot](https://github.com/rs/jplot) using [jaggr](https://github.com/rs/jaggr) to plot a vegeta report in real-time in the comfort of you terminal:
```
echo 'GET http://localhost:8080' | \
vegeta attack -rate 5000 -duration 10m | vegeta encode | \
jaggr @count=rps \
hist\[100,200,300,400,500\]:code \
p25,p50,p95:latency \
sum:bytes_in \
sum:bytes_out | \
jplot rps+code.hist.100+code.hist.200+code.hist.300+code.hist.400+code.hist.500 \
latency.p95+latency.p50+latency.p25 \
bytes_in.sum+bytes_out.sum
```
![](https://i.imgur.com/ttBDsQS.gif)
## Usage (Library)
The library versioning follows [SemVer v2.0.0](https://semver.org/spec/v2.0.0.html).
Since [lib/v9.0.0](https://github.com/tsenart/vegeta/tree/lib/v9.0.0), the library and cli
are versioned separately to better isolate breaking changes to each component.
See [Versioning](#Versioning) for more details on git tag naming schemes and compatibility
with `go mod`.
```go
package main
import (
"fmt"
"time"
vegeta "github.com/tsenart/vegeta/lib"
)
func main() {
rate := vegeta.Rate{Freq: 100, Per: time.Second}
duration := 4 * time.Second
targeter := vegeta.NewStaticTargeter(vegeta.Target{
Method: "GET",
URL: "http://localhost:9100/",
})
attacker := vegeta.NewAttacker()
var metrics vegeta.Metrics
for res := range attacker.Attack(targeter, rate, duration, "Big Bang!") {
metrics.Add(res)
}
metrics.Close()
fmt.Printf("99th percentile: %s\n", metrics.Latencies.P99)
}
```
#### Limitations
There will be an upper bound of the supported `rate` which varies on the
machine being used.
You could be CPU bound (unlikely), memory bound (more likely) or
have system resource limits being reached which ought to be tuned for
the process execution. The important limits for us are file descriptors
and processes. On a UNIX system you can get and set the current
soft-limit values for a user.
```shell
$ ulimit -n # file descriptors
2560
$ ulimit -u # processes / threads
709
```
Just pass a new number as the argument to change it.
## License
See [LICENSE](LICENSE).
## Donate
If you use and love Vegeta, please consider sending some Satoshi to
`1MDmKC51ve7Upxt75KoNM6x1qdXHFK6iW2`. In case you want to be mentioned as a
sponsor, let me know!
[![Donate Bitcoin](https://i.imgur.com/W9Vc51d.png)](#donate)

View File

@ -1,242 +0,0 @@
package main
import (
"crypto/tls"
"crypto/x509"
"errors"
"flag"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"os"
"os/signal"
"strings"
"time"
"github.com/tsenart/vegeta/internal/resolver"
vegeta "github.com/tsenart/vegeta/lib"
)
func attackCmd() command {
fs := flag.NewFlagSet("vegeta attack", flag.ExitOnError)
opts := &attackOpts{
headers: headers{http.Header{}},
laddr: localAddr{&vegeta.DefaultLocalAddr},
rate: vegeta.Rate{Freq: 50, Per: time.Second},
maxBody: vegeta.DefaultMaxBody,
}
fs.StringVar(&opts.name, "name", "", "Attack name")
fs.StringVar(&opts.targetsf, "targets", "stdin", "Targets file")
fs.StringVar(&opts.format, "format", vegeta.HTTPTargetFormat,
fmt.Sprintf("Targets format [%s]", strings.Join(vegeta.TargetFormats, ", ")))
fs.StringVar(&opts.outputf, "output", "stdout", "Output file")
fs.StringVar(&opts.bodyf, "body", "", "Requests body file")
fs.StringVar(&opts.certf, "cert", "", "TLS client PEM encoded certificate file")
fs.StringVar(&opts.keyf, "key", "", "TLS client PEM encoded private key file")
fs.Var(&opts.rootCerts, "root-certs", "TLS root certificate files (comma separated list)")
fs.BoolVar(&opts.http2, "http2", true, "Send HTTP/2 requests when supported by the server")
fs.BoolVar(&opts.h2c, "h2c", false, "Send HTTP/2 requests without TLS encryption")
fs.BoolVar(&opts.insecure, "insecure", false, "Ignore invalid server TLS certificates")
fs.BoolVar(&opts.lazy, "lazy", false, "Read targets lazily")
fs.DurationVar(&opts.duration, "duration", 0, "Duration of the test [0 = forever]")
fs.DurationVar(&opts.timeout, "timeout", vegeta.DefaultTimeout, "Requests timeout")
fs.Uint64Var(&opts.workers, "workers", vegeta.DefaultWorkers, "Initial number of workers")
fs.Uint64Var(&opts.maxWorkers, "max-workers", vegeta.DefaultMaxWorkers, "Maximum number of workers")
fs.IntVar(&opts.connections, "connections", vegeta.DefaultConnections, "Max open idle connections per target host")
fs.IntVar(&opts.redirects, "redirects", vegeta.DefaultRedirects, "Number of redirects to follow. -1 will not follow but marks as success")
fs.Var(&maxBodyFlag{&opts.maxBody}, "max-body", "Maximum number of bytes to capture from response bodies. [-1 = no limit]")
fs.Var(&rateFlag{&opts.rate}, "rate", "Number of requests per time unit [0 = infinity]")
fs.Var(&opts.headers, "header", "Request header")
fs.Var(&opts.laddr, "laddr", "Local IP address")
fs.BoolVar(&opts.keepalive, "keepalive", true, "Use persistent connections")
fs.StringVar(&opts.unixSocket, "unix-socket", "", "Connect over a unix socket. This overrides the host address in target URLs")
systemSpecificFlags(fs, opts)
return command{fs, func(args []string) error {
fs.Parse(args)
return attack(opts)
}}
}
var (
errZeroRate = errors.New("rate frequency and time unit must be bigger than zero")
errBadCert = errors.New("bad certificate")
)
// attackOpts aggregates the attack function command options
type attackOpts struct {
name string
targetsf string
format string
outputf string
bodyf string
certf string
keyf string
rootCerts csl
http2 bool
h2c bool
insecure bool
lazy bool
duration time.Duration
timeout time.Duration
rate vegeta.Rate
workers uint64
maxWorkers uint64
connections int
redirects int
maxBody int64
headers headers
laddr localAddr
keepalive bool
resolvers csl
unixSocket string
}
// attack validates the attack arguments, sets up the
// required resources, launches the attack and writes the results
func attack(opts *attackOpts) (err error) {
if opts.maxWorkers == vegeta.DefaultMaxWorkers && opts.rate.Freq == 0 {
return fmt.Errorf("-rate=0 requires setting -max-workers")
}
if len(opts.resolvers) > 0 {
res, err := resolver.NewResolver(opts.resolvers)
if err != nil {
return err
}
net.DefaultResolver = res
}
files := map[string]io.Reader{}
for _, filename := range []string{opts.targetsf, opts.bodyf} {
if filename == "" {
continue
}
f, err := file(filename, false)
if err != nil {
return fmt.Errorf("error opening %s: %s", filename, err)
}
defer f.Close()
files[filename] = f
}
var body []byte
if bodyf, ok := files[opts.bodyf]; ok {
if body, err = ioutil.ReadAll(bodyf); err != nil {
return fmt.Errorf("error reading %s: %s", opts.bodyf, err)
}
}
var (
tr vegeta.Targeter
src = files[opts.targetsf]
hdr = opts.headers.Header
)
switch opts.format {
case vegeta.JSONTargetFormat:
tr = vegeta.NewJSONTargeter(src, body, hdr)
case vegeta.HTTPTargetFormat:
tr = vegeta.NewHTTPTargeter(src, body, hdr)
default:
return fmt.Errorf("format %q isn't one of [%s]",
opts.format, strings.Join(vegeta.TargetFormats, ", "))
}
if !opts.lazy {
targets, err := vegeta.ReadAllTargets(tr)
if err != nil {
return err
}
tr = vegeta.NewStaticTargeter(targets...)
}
out, err := file(opts.outputf, true)
if err != nil {
return fmt.Errorf("error opening %s: %s", opts.outputf, err)
}
defer out.Close()
tlsc, err := tlsConfig(opts.insecure, opts.certf, opts.keyf, opts.rootCerts)
if err != nil {
return err
}
atk := vegeta.NewAttacker(
vegeta.Redirects(opts.redirects),
vegeta.Timeout(opts.timeout),
vegeta.LocalAddr(*opts.laddr.IPAddr),
vegeta.TLSConfig(tlsc),
vegeta.Workers(opts.workers),
vegeta.MaxWorkers(opts.maxWorkers),
vegeta.KeepAlive(opts.keepalive),
vegeta.Connections(opts.connections),
vegeta.HTTP2(opts.http2),
vegeta.H2C(opts.h2c),
vegeta.MaxBody(opts.maxBody),
vegeta.UnixSocket(opts.unixSocket),
)
res := atk.Attack(tr, opts.rate, opts.duration, opts.name)
enc := vegeta.NewEncoder(out)
sig := make(chan os.Signal, 1)
signal.Notify(sig, os.Interrupt)
for {
select {
case <-sig:
atk.Stop()
return nil
case r, ok := <-res:
if !ok {
return nil
}
if err = enc.Encode(r); err != nil {
return err
}
}
}
}
// tlsConfig builds a *tls.Config from the given options.
func tlsConfig(insecure bool, certf, keyf string, rootCerts []string) (*tls.Config, error) {
var err error
files := map[string][]byte{}
filenames := append([]string{certf, keyf}, rootCerts...)
for _, f := range filenames {
if f != "" {
if files[f], err = ioutil.ReadFile(f); err != nil {
return nil, err
}
}
}
c := tls.Config{InsecureSkipVerify: insecure}
if cert, ok := files[certf]; ok {
key, ok := files[keyf]
if !ok {
key = cert
}
certificate, err := tls.X509KeyPair(cert, key)
if err != nil {
return nil, err
}
c.Certificates = append(c.Certificates, certificate)
c.BuildNameToCertificate()
}
if len(rootCerts) > 0 {
c.RootCAs = x509.NewCertPool()
for _, f := range rootCerts {
if !c.RootCAs.AppendCertsFromPEM(files[f]) {
return nil, errBadCert
}
}
}
return &c, nil
}

View File

@ -1,9 +0,0 @@
// +build !windows
package main
import "flag"
func systemSpecificFlags(fs *flag.FlagSet, opts *attackOpts) {
fs.Var(&opts.resolvers, "resolvers", "List of addresses (ip:port) to use for DNS resolution. Disables use of local system DNS. (comma separated list)")
}

View File

@ -1,5 +0,0 @@
package main
import "flag"
func systemSpecificFlags(fs *flag.FlagSet, opts *attackOpts) {}

View File

@ -1,11 +0,0 @@
package main
import (
"fmt"
)
func dumpCmd() command {
return command{fn: func([]string) error {
return fmt.Errorf("vegeta dump has been deprecated and succeeded by the vegeta encode command")
}}
}

View File

@ -1,119 +0,0 @@
package main
import (
"flag"
"fmt"
"io"
"os"
"os/signal"
"strings"
vegeta "github.com/tsenart/vegeta/lib"
)
const (
encodingCSV = "csv"
encodingGob = "gob"
encodingJSON = "json"
)
const encodeUsage = `Usage: vegeta encode [options] [<file>...]
Encodes vegeta attack results from one encoding to another.
The supported encodings are Gob (binary), CSV and JSON.
Each input file may have a different encoding which is detected
automatically.
The CSV encoder doesn't write a header. The columns written by it are:
1. Unix timestamp in nanoseconds since epoch
2. HTTP status code
3. Request latency in nanoseconds
4. Bytes out
5. Bytes in
6. Error
7. Base64 encoded response body
8. Attack name
9. Sequence number of request
Arguments:
<file> A file with vegeta attack results encoded with one of
the supported encodings (gob | json | csv) [default: stdin]
Options:
--to Output encoding (gob | json | csv) [default: json]
--output Output file [default: stdout]
Examples:
echo "GET http://:80" | vegeta attack -rate=1/s > results.gob
cat results.gob | vegeta encode | jq -c 'del(.body)' | vegeta encode -to gob
`
func encodeCmd() command {
encs := "[" + strings.Join([]string{encodingCSV, encodingGob, encodingJSON}, ", ") + "]"
fs := flag.NewFlagSet("vegeta encode", flag.ExitOnError)
to := fs.String("to", encodingJSON, "Output encoding "+encs)
output := fs.String("output", "stdout", "Output file")
fs.Usage = func() {
fmt.Fprintln(os.Stderr, encodeUsage)
}
return command{fs, func(args []string) error {
fs.Parse(args)
files := fs.Args()
if len(files) == 0 {
files = append(files, "stdin")
}
return encode(files, *to, *output)
}}
}
func encode(files []string, to, output string) error {
dec, mc, err := decoder(files)
defer mc.Close()
if err != nil {
return err
}
out, err := file(output, true)
if err != nil {
return err
}
defer out.Close()
var enc vegeta.Encoder
switch to {
case encodingCSV:
enc = vegeta.NewCSVEncoder(out)
case encodingGob:
enc = vegeta.NewEncoder(out)
case encodingJSON:
enc = vegeta.NewJSONEncoder(out)
default:
return fmt.Errorf("encode: unknown encoding %q", to)
}
sigch := make(chan os.Signal, 1)
signal.Notify(sigch, os.Interrupt)
for {
select {
case <-sigch:
return nil
default:
}
var r vegeta.Result
if err = dec.Decode(&r); err != nil {
if err == io.EOF {
break
}
return err
} else if err = enc.Encode(&r); err != nil {
return err
}
}
return nil
}

Some files were not shown because too many files have changed in this diff Show More