mirror of https://github.com/knative/caching.git
Drop unnecessary dependencies on vegeta and licenseclassifier (#582)
This commit is contained in:
Markus Thömmes
GitHub
parent
a2e8051b64
commit
c29dc56d8f
15
go.mod
15
go.mod
|
|
@ -3,23 +3,8 @@ module knative.dev/caching
|
|||
go 1.15
|
||||
|
||||
require (
|
||||
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee // indirect
|
||||
github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.19.5 // indirect
|
||||
github.com/go-openapi/jsonreference v0.19.5 // indirect
|
||||
github.com/go-openapi/spec v0.19.6 // indirect
|
||||
github.com/go-openapi/swag v0.19.15 // indirect
|
||||
github.com/google/go-cmp v0.5.6
|
||||
github.com/google/licenseclassifier v0.0.0-20200708223521-3d09a0ea2f39
|
||||
github.com/imdario/mergo v0.3.9 // indirect
|
||||
github.com/influxdata/tdigest v0.0.1 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/miekg/dns v1.1.29 // indirect
|
||||
github.com/sergi/go-diff v1.1.0 // indirect
|
||||
github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3 // indirect
|
||||
github.com/tsenart/vegeta v12.7.1-0.20190725001342-b5f4fca92137+incompatible
|
||||
go.uber.org/zap v1.19.1
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
|
||||
k8s.io/api v0.21.4
|
||||
k8s.io/apimachinery v0.21.4
|
||||
k8s.io/client-go v0.21.4
|
||||
|
|
|
|||
56
go.sum
56
go.sum
|
|
@ -95,11 +95,8 @@ github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
|
|||
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
|
||||
github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE=
|
||||
github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc=
|
||||
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE=
|
||||
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
|
||||
github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
|
||||
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee h1:BnPxIde0gjtTnc9Er7cxvBk8DHLWhEux0SxayC8dP6I=
|
||||
github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/census-instrumentation/opencensus-proto v0.3.0 h1:t/LhUZLVitR1Ow2YOnduCsavhwFUklBMoGVYUCqmCqk=
|
||||
github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
|
|
@ -139,9 +136,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
|||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-gk v0.0.0-20140819190930-201884a44051/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E=
|
||||
github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654 h1:XOPLOMn/zT4jIgxfxSsoXPxkrzz0FaCHwp33x5POJ+Q=
|
||||
github.com/dgryski/go-gk v0.0.0-20200319235926-a69029f61654/go.mod h1:qm+vckxRlDt0aOla0RYJJVeqHZlWfOm2UIxHaqPB46E=
|
||||
github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77 h1:iRnqZBF0a1hoOOjOdPKf+IxqlJZOas7A48j77RAc7Yg=
|
||||
github.com/dgryski/go-lttb v0.0.0-20180810165845-318fcdf10a77/go.mod h1:Va5MyIzkU0rAM92tn3hb3Anb7oz7KcnixF49+2wOMe4=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
|
||||
|
|
@ -194,21 +188,17 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
|
|||
github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
|
||||
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
|
||||
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
|
||||
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
|
||||
github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
|
||||
github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o=
|
||||
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
|
||||
github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM=
|
||||
github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
|
||||
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
|
||||
github.com/go-openapi/spec v0.19.5 h1:Xm0Ao53uqnk9QE/LlYV5DEU09UAgpliA85QoT9LzqPw=
|
||||
github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
|
||||
github.com/go-openapi/spec v0.19.6 h1:rMMMj8cV38KVXK7SFc+I2MWClbEfbK705+j+dyqun5g=
|
||||
github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
|
||||
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM=
|
||||
github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
|
||||
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
|
||||
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
|
|
@ -287,8 +277,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
|
|||
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/licenseclassifier v0.0.0-20200708223521-3d09a0ea2f39 h1:nkr7S2ETn5pAuBeeoZggV5nXSwOm4nBLz3vscQfA/A8=
|
||||
github.com/google/licenseclassifier v0.0.0-20200708223521-3d09a0ea2f39/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M=
|
||||
github.com/google/mako v0.0.0-20190821191249-122f8dcef9e3/go.mod h1:YzLcVlL+NqWnmUEPuhS1LxDDwGO9WNbVlEXaF4IH35g=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
|
|
@ -363,13 +351,10 @@ github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/J
|
|||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
|
||||
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
|
||||
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
|
||||
github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
|
||||
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/influxdata/tdigest v0.0.0-20180711151920-a7d76c6f093a/go.mod h1:9GkyshztGufsdPQWjH+ifgnIr3xNUL5syI70g2dzU1o=
|
||||
github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY=
|
||||
github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
|
||||
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
|
||||
|
|
@ -378,8 +363,6 @@ github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aW
|
|||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
|
||||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
|
|
@ -413,10 +396,8 @@ github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0Q
|
|||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/mailru/easyjson v0.7.0 h1:aizVhC/NAAcKWb+5QsU1iNOZb4Yws5UO2I+aIprQITM=
|
||||
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
|
|
@ -426,8 +407,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff
|
|||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.17/go.mod h1:WgzbA6oji13JREwiNsRDNfl7jYdPnmz+VEuLrA+/48M=
|
||||
github.com/miekg/dns v1.1.29 h1:xHBEhR+t5RzcFJjBLJlax2daXOrTYtr9z4WdKEfWFzg=
|
||||
github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
|
||||
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
|
|
@ -524,9 +503,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
|
|||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
|
||||
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
|
|
@ -550,7 +526,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
|
|||
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
|
||||
github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25 h1:7z3LSn867ex6VSaahyKadf4WtSsJIgne6A1WLOAGM8A=
|
||||
github.com/streadway/quantile v0.0.0-20150917103942-b0c588724d25/go.mod h1:lbP8tGiBjZ5YWIc2fzuRpTaz0b/53vT6PEs3QuAWzuU=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
|
|
@ -566,10 +541,6 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
|
|||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tsenart/go-tsz v0.0.0-20180814232043-cdeb9e1e981e/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo=
|
||||
github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3 h1:pcQGQzTwCg//7FgVywqge1sW9Yf8VMsMdG58MI5kd8s=
|
||||
github.com/tsenart/go-tsz v0.0.0-20180814235614-0bd30b3df1c3/go.mod h1:SWZznP1z5Ki7hDT2ioqiFKEse8K9tU2OUvaRI0NeGQo=
|
||||
github.com/tsenart/vegeta v12.7.1-0.20190725001342-b5f4fca92137+incompatible h1:ErZrHhRveAoznVW80gbrxz+qxJNydpA2fcQxTPHkZbU=
|
||||
github.com/tsenart/vegeta v12.7.1-0.20190725001342-b5f4fca92137+incompatible/go.mod h1:Smz/ZWfhKRcyDDChZkG3CyTHdj87lHzio/HOCkbndXM=
|
||||
github.com/tsenart/vegeta/v12 v12.8.4/go.mod h1:ZiJtwLn/9M4fTPdMY7bdbIeyNeFVE8/AHbWFqCsUuho=
|
||||
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
|
||||
|
|
@ -622,9 +593,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
|
|||
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
|
||||
golang.org/x/crypto v0.0.0-20210920023735-84f357641f63 h1:kETrAMYZq6WVGPa8IIixL0CaEcIUNi+1WX7grUoi3y8=
|
||||
golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
|
|
@ -634,7 +603,6 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0
|
|||
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
|
||||
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y=
|
||||
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
|
||||
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
|
|
@ -685,7 +653,6 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL
|
|||
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
|
|
@ -767,7 +734,6 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
|
|
@ -838,11 +804,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb
|
|||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
|
||||
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
|
||||
golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
|
|
@ -868,7 +832,6 @@ golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtn
|
|||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
|
|
@ -913,10 +876,6 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1N
|
|||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
|
||||
gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
|
||||
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 h1:4WsZyVtkthqrHTbDCJfiTs8IWNYE4uvsSDgaV6xpp+o=
|
||||
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
|
||||
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
|
||||
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
|
||||
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
|
|
@ -1094,7 +1053,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
|
||||
|
|
|
|||
|
|
@ -33,10 +33,4 @@ import (
|
|||
_ "k8s.io/code-generator/cmd/lister-gen"
|
||||
_ "k8s.io/kube-openapi/cmd/openapi-gen"
|
||||
_ "knative.dev/pkg/codegen/cmd/injection-gen"
|
||||
|
||||
// Licenseclassifier
|
||||
_ "github.com/google/licenseclassifier"
|
||||
|
||||
// For load testing
|
||||
_ "github.com/tsenart/vegeta"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.exe
|
||||
*.test
|
||||
*.prof
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
sudo: false
|
||||
|
||||
language: go
|
||||
go:
|
||||
- 1.4
|
||||
- 1.5
|
||||
- 1.6
|
||||
- 1.7
|
||||
- 1.8
|
||||
- 1.9
|
||||
- tip
|
||||
|
||||
script:
|
||||
- go test -v
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2016 Maciej Lisiewski
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
# datasize [](https://travis-ci.org/c2h5oh/datasize)
|
||||
|
||||
Golang helpers for data sizes
|
||||
|
||||
### Constants
|
||||
|
||||
Just like `time` package provides `time.Second`, `time.Day` constants `datasize` provides:
|
||||
|
||||
* `datasize.B` 1 byte
|
||||
* `datasize.KB` 1 kilobyte
|
||||
* `datasize.MB` 1 megabyte
|
||||
* `datasize.GB` 1 gigabyte
|
||||
* `datasize.TB` 1 terabyte
|
||||
* `datasize.PB` 1 petabyte
|
||||
* `datasize.EB` 1 exabyte
|
||||
|
||||
### Helpers
|
||||
|
||||
Just like `time` package provides `duration.Nanoseconds() uint64 `, `duration.Hours() float64` helpers `datasize` has.
|
||||
|
||||
* `ByteSize.Bytes() uint64`
|
||||
* `ByteSize.Kilobytes() float4`
|
||||
* `ByteSize.Megabytes() float64`
|
||||
* `ByteSize.Gigabytes() float64`
|
||||
* `ByteSize.Terabytes() float64`
|
||||
* `ByteSize.Petebytes() float64`
|
||||
* `ByteSize.Exabytes() float64`
|
||||
|
||||
Warning: see limitations at the end of this document about a possible precission loss
|
||||
|
||||
### Parsing strings
|
||||
|
||||
`datasize.ByteSize` implements `TextUnmarshaler` interface and will automatically parse human readable strings into correct values where it is used:
|
||||
|
||||
* `"10 MB"` -> `10* datasize.MB`
|
||||
* `"10240 g"` -> `10 * datasize.TB`
|
||||
* `"2000"` -> `2000 * datasize.B`
|
||||
* `"1tB"` -> `datasize.TB`
|
||||
* `"5 peta"` -> `5 * datasize.PB`
|
||||
* `"28 kilobytes"` -> `28 * datasize.KB`
|
||||
* `"1 gigabyte"` -> `1 * datasize.GB`
|
||||
|
||||
You can also do it manually:
|
||||
|
||||
```go
|
||||
var v datasize.ByteSize
|
||||
err := v.UnmarshalText([]byte("100 mb"))
|
||||
```
|
||||
|
||||
### Printing
|
||||
|
||||
`Bytesize.String()` uses largest unit allowing an integer value:
|
||||
|
||||
* `(102400 * datasize.MB).String()` -> `"100GB"`
|
||||
* `(datasize.MB + datasize.KB).String()` -> `"1025KB"`
|
||||
|
||||
Use `%d` format string to get value in bytes without a unit.
|
||||
|
||||
### JSON and other encoding
|
||||
|
||||
Both `TextMarshaler` and `TextUnmarshaler` interfaces are implemented - JSON will just work. Other encoders will work provided they use those interfaces.
|
||||
|
||||
### Human readable
|
||||
|
||||
`ByteSize.HumanReadable()` or `ByteSize.HR()` returns a string with 1-3 digits, followed by 1 decimal place, a space and unit big enough to get 1-3 digits
|
||||
|
||||
* `(102400 * datasize.MB).String()` -> `"100.0 GB"`
|
||||
* `(datasize.MB + 512 * datasize.KB).String()` -> `"1.5 MB"`
|
||||
|
||||
### Limitations
|
||||
|
||||
* The underlying data type for `data.ByteSize` is `uint64`, so values outside of 0 to 2^64-1 range will overflow
|
||||
* size helper functions (like `ByteSize.Kilobytes()`) return `float64`, which can't represent all possible values of `uint64` accurately:
|
||||
* if the returned value is supposed to have no fraction (ie `(10 * datasize.MB).Kilobytes()`) accuracy loss happens when value is more than 2^53 larger than unit: `.Kilobytes()` over 8 petabytes, `.Megabytes()` over 8 exabytes
|
||||
* if the returned value is supposed to have a fraction (ie `(datasize.PB + datasize.B).Megabytes()`) in addition to the above note accuracy loss may occur in fractional part too - larger integer part leaves fewer bytes to store fractional part, the smaller the remainder vs unit the move bytes are required to store the fractional part
|
||||
* Parsing a string with `Mb`, `Tb`, etc units will return a syntax error, because capital followed by lower case is commonly used for bits, not bytes
|
||||
* Parsing a string with value exceeding 2^64-1 bytes will return 2^64-1 and an out of range error
|
||||
|
|
@ -1,217 +0,0 @@
|
|||
package datasize
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type ByteSize uint64
|
||||
|
||||
const (
|
||||
B ByteSize = 1
|
||||
KB = B << 10
|
||||
MB = KB << 10
|
||||
GB = MB << 10
|
||||
TB = GB << 10
|
||||
PB = TB << 10
|
||||
EB = PB << 10
|
||||
|
||||
fnUnmarshalText string = "UnmarshalText"
|
||||
maxUint64 uint64 = (1 << 64) - 1
|
||||
cutoff uint64 = maxUint64 / 10
|
||||
)
|
||||
|
||||
var ErrBits = errors.New("unit with capital unit prefix and lower case unit (b) - bits, not bytes ")
|
||||
|
||||
func (b ByteSize) Bytes() uint64 {
|
||||
return uint64(b)
|
||||
}
|
||||
|
||||
func (b ByteSize) KBytes() float64 {
|
||||
v := b / KB
|
||||
r := b % KB
|
||||
return float64(v) + float64(r)/float64(KB)
|
||||
}
|
||||
|
||||
func (b ByteSize) MBytes() float64 {
|
||||
v := b / MB
|
||||
r := b % MB
|
||||
return float64(v) + float64(r)/float64(MB)
|
||||
}
|
||||
|
||||
func (b ByteSize) GBytes() float64 {
|
||||
v := b / GB
|
||||
r := b % GB
|
||||
return float64(v) + float64(r)/float64(GB)
|
||||
}
|
||||
|
||||
func (b ByteSize) TBytes() float64 {
|
||||
v := b / TB
|
||||
r := b % TB
|
||||
return float64(v) + float64(r)/float64(TB)
|
||||
}
|
||||
|
||||
func (b ByteSize) PBytes() float64 {
|
||||
v := b / PB
|
||||
r := b % PB
|
||||
return float64(v) + float64(r)/float64(PB)
|
||||
}
|
||||
|
||||
func (b ByteSize) EBytes() float64 {
|
||||
v := b / EB
|
||||
r := b % EB
|
||||
return float64(v) + float64(r)/float64(EB)
|
||||
}
|
||||
|
||||
func (b ByteSize) String() string {
|
||||
switch {
|
||||
case b == 0:
|
||||
return fmt.Sprint("0B")
|
||||
case b%EB == 0:
|
||||
return fmt.Sprintf("%dEB", b/EB)
|
||||
case b%PB == 0:
|
||||
return fmt.Sprintf("%dPB", b/PB)
|
||||
case b%TB == 0:
|
||||
return fmt.Sprintf("%dTB", b/TB)
|
||||
case b%GB == 0:
|
||||
return fmt.Sprintf("%dGB", b/GB)
|
||||
case b%MB == 0:
|
||||
return fmt.Sprintf("%dMB", b/MB)
|
||||
case b%KB == 0:
|
||||
return fmt.Sprintf("%dKB", b/KB)
|
||||
default:
|
||||
return fmt.Sprintf("%dB", b)
|
||||
}
|
||||
}
|
||||
|
||||
func (b ByteSize) HR() string {
|
||||
return b.HumanReadable()
|
||||
}
|
||||
|
||||
func (b ByteSize) HumanReadable() string {
|
||||
switch {
|
||||
case b > EB:
|
||||
return fmt.Sprintf("%.1f EB", b.EBytes())
|
||||
case b > PB:
|
||||
return fmt.Sprintf("%.1f PB", b.PBytes())
|
||||
case b > TB:
|
||||
return fmt.Sprintf("%.1f TB", b.TBytes())
|
||||
case b > GB:
|
||||
return fmt.Sprintf("%.1f GB", b.GBytes())
|
||||
case b > MB:
|
||||
return fmt.Sprintf("%.1f MB", b.MBytes())
|
||||
case b > KB:
|
||||
return fmt.Sprintf("%.1f KB", b.KBytes())
|
||||
default:
|
||||
return fmt.Sprintf("%d B", b)
|
||||
}
|
||||
}
|
||||
|
||||
func (b ByteSize) MarshalText() ([]byte, error) {
|
||||
return []byte(b.String()), nil
|
||||
}
|
||||
|
||||
func (b *ByteSize) UnmarshalText(t []byte) error {
|
||||
var val uint64
|
||||
var unit string
|
||||
|
||||
// copy for error message
|
||||
t0 := t
|
||||
|
||||
var c byte
|
||||
var i int
|
||||
|
||||
ParseLoop:
|
||||
for i < len(t) {
|
||||
c = t[i]
|
||||
switch {
|
||||
case '0' <= c && c <= '9':
|
||||
if val > cutoff {
|
||||
goto Overflow
|
||||
}
|
||||
|
||||
c = c - '0'
|
||||
val *= 10
|
||||
|
||||
if val > val+uint64(c) {
|
||||
// val+v overflows
|
||||
goto Overflow
|
||||
}
|
||||
val += uint64(c)
|
||||
i++
|
||||
|
||||
default:
|
||||
if i == 0 {
|
||||
goto SyntaxError
|
||||
}
|
||||
break ParseLoop
|
||||
}
|
||||
}
|
||||
|
||||
unit = strings.TrimSpace(string(t[i:]))
|
||||
switch unit {
|
||||
case "Kb", "Mb", "Gb", "Tb", "Pb", "Eb":
|
||||
goto BitsError
|
||||
}
|
||||
unit = strings.ToLower(unit)
|
||||
switch unit {
|
||||
case "", "b", "byte":
|
||||
// do nothing - already in bytes
|
||||
|
||||
case "k", "kb", "kilo", "kilobyte", "kilobytes":
|
||||
if val > maxUint64/uint64(KB) {
|
||||
goto Overflow
|
||||
}
|
||||
val *= uint64(KB)
|
||||
|
||||
case "m", "mb", "mega", "megabyte", "megabytes":
|
||||
if val > maxUint64/uint64(MB) {
|
||||
goto Overflow
|
||||
}
|
||||
val *= uint64(MB)
|
||||
|
||||
case "g", "gb", "giga", "gigabyte", "gigabytes":
|
||||
if val > maxUint64/uint64(GB) {
|
||||
goto Overflow
|
||||
}
|
||||
val *= uint64(GB)
|
||||
|
||||
case "t", "tb", "tera", "terabyte", "terabytes":
|
||||
if val > maxUint64/uint64(TB) {
|
||||
goto Overflow
|
||||
}
|
||||
val *= uint64(TB)
|
||||
|
||||
case "p", "pb", "peta", "petabyte", "petabytes":
|
||||
if val > maxUint64/uint64(PB) {
|
||||
goto Overflow
|
||||
}
|
||||
val *= uint64(PB)
|
||||
|
||||
case "E", "EB", "e", "eb", "eB":
|
||||
if val > maxUint64/uint64(EB) {
|
||||
goto Overflow
|
||||
}
|
||||
val *= uint64(EB)
|
||||
|
||||
default:
|
||||
goto SyntaxError
|
||||
}
|
||||
|
||||
*b = ByteSize(val)
|
||||
return nil
|
||||
|
||||
Overflow:
|
||||
*b = ByteSize(maxUint64)
|
||||
return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrRange}
|
||||
|
||||
SyntaxError:
|
||||
*b = 0
|
||||
return &strconv.NumError{fnUnmarshalText, string(t0), strconv.ErrSyntax}
|
||||
|
||||
BitsError:
|
||||
*b = 0
|
||||
return &strconv.NumError{fnUnmarshalText, string(t0), ErrBits}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
go:
|
||||
- 1.14.x
|
||||
- 1.15.x
|
||||
- 1.11.x
|
||||
- 1.12.x
|
||||
install:
|
||||
- GO111MODULE=off go get -u gotest.tools/gotestsum
|
||||
env:
|
||||
|
|
|
|||
|
|
@ -114,6 +114,9 @@ func getSingleImpl(node interface{}, decodedToken string, nameProvider *swag.Nam
|
|||
rValue := reflect.Indirect(reflect.ValueOf(node))
|
||||
kind := rValue.Kind()
|
||||
|
||||
switch kind {
|
||||
|
||||
case reflect.Struct:
|
||||
if rValue.Type().Implements(jsonPointableType) {
|
||||
r, err := node.(JSONPointable).JSONLookup(decodedToken)
|
||||
if err != nil {
|
||||
|
|
@ -121,9 +124,6 @@ func getSingleImpl(node interface{}, decodedToken string, nameProvider *swag.Nam
|
|||
}
|
||||
return r, kind, nil
|
||||
}
|
||||
|
||||
switch kind {
|
||||
case reflect.Struct:
|
||||
nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken)
|
||||
if !ok {
|
||||
return nil, kind, fmt.Errorf("object has no field %q", decodedToken)
|
||||
|
|
@ -161,7 +161,9 @@ func getSingleImpl(node interface{}, decodedToken string, nameProvider *swag.Nam
|
|||
|
||||
func setSingleImpl(node, data interface{}, decodedToken string, nameProvider *swag.NameProvider) error {
|
||||
rValue := reflect.Indirect(reflect.ValueOf(node))
|
||||
switch rValue.Kind() {
|
||||
|
||||
case reflect.Struct:
|
||||
if ns, ok := node.(JSONSetable); ok { // pointer impl
|
||||
return ns.JSONSet(decodedToken, data)
|
||||
}
|
||||
|
|
@ -170,8 +172,6 @@ func setSingleImpl(node, data interface{}, decodedToken string, nameProvider *sw
|
|||
return node.(JSONSetable).JSONSet(decodedToken, data)
|
||||
}
|
||||
|
||||
switch rValue.Kind() {
|
||||
case reflect.Struct:
|
||||
nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken)
|
||||
if !ok {
|
||||
return fmt.Errorf("object has no field %q", decodedToken)
|
||||
|
|
@ -270,6 +270,9 @@ func (p *Pointer) set(node, data interface{}, nameProvider *swag.NameProvider) e
|
|||
rValue := reflect.Indirect(reflect.ValueOf(node))
|
||||
kind := rValue.Kind()
|
||||
|
||||
switch kind {
|
||||
|
||||
case reflect.Struct:
|
||||
if rValue.Type().Implements(jsonPointableType) {
|
||||
r, err := node.(JSONPointable).JSONLookup(decodedToken)
|
||||
if err != nil {
|
||||
|
|
@ -283,9 +286,6 @@ func (p *Pointer) set(node, data interface{}, nameProvider *swag.NameProvider) e
|
|||
node = r
|
||||
continue
|
||||
}
|
||||
|
||||
switch kind {
|
||||
case reflect.Struct:
|
||||
nm, ok := nameProvider.GetGoNameForType(rValue.Type(), decodedToken)
|
||||
if !ok {
|
||||
return fmt.Errorf("object has no field %q", decodedToken)
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
linters-settings:
|
||||
govet:
|
||||
check-shadowing: true
|
||||
golint:
|
||||
min-confidence: 0
|
||||
gocyclo:
|
||||
min-complexity: 30
|
||||
maligned:
|
||||
suggest-new: true
|
||||
dupl:
|
||||
threshold: 100
|
||||
goconst:
|
||||
min-len: 2
|
||||
min-occurrences: 4
|
||||
linters:
|
||||
enable-all: true
|
||||
disable:
|
||||
- maligned
|
||||
- lll
|
||||
- gochecknoglobals
|
||||
- godox
|
||||
- gocognit
|
||||
- whitespace
|
||||
- wsl
|
||||
- funlen
|
||||
- gochecknoglobals
|
||||
- gochecknoinits
|
||||
- scopelint
|
||||
- wrapcheck
|
||||
- exhaustivestruct
|
||||
- exhaustive
|
||||
- nlreturn
|
||||
- testpackage
|
||||
- gci
|
||||
- gofumpt
|
||||
- goerr113
|
||||
- gomnd
|
||||
- tparallel
|
||||
- nestif
|
||||
- godot
|
||||
- errorlint
|
||||
|
|
@ -1,19 +1,10 @@
|
|||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
go:
|
||||
- 1.14.x
|
||||
- 1.x
|
||||
- 1.11.x
|
||||
- 1.12.x
|
||||
install:
|
||||
- go get gotest.tools/gotestsum
|
||||
jobs:
|
||||
include:
|
||||
# include linting job, but only for latest go version and amd64 arch
|
||||
- go: 1.x
|
||||
arch: amd64
|
||||
install:
|
||||
go get github.com/golangci/golangci-lint/cmd/golangci-lint
|
||||
script:
|
||||
- golangci-lint run --new-from-rev master
|
||||
- GO111MODULE=off go get -u gotest.tools/gotestsum
|
||||
env:
|
||||
- GO111MODULE=on
|
||||
language: go
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
An implementation of JSON Reference - Go language
|
||||
|
||||
## Status
|
||||
Feature complete. Stable API
|
||||
Work in progress ( 90% done )
|
||||
|
||||
## Dependencies
|
||||
https://github.com/go-openapi/jsonpointer
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ require (
|
|||
github.com/go-openapi/jsonpointer v0.19.3
|
||||
github.com/stretchr/testify v1.3.0
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
|
||||
golang.org/x/text v0.3.3 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
)
|
||||
|
||||
go 1.13
|
||||
|
|
|
|||
|
|
@ -5,8 +5,12 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
|
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-openapi/jsonpointer v0.19.2 h1:A9+F4Dc/MCNB5jibxf6rRvOvR/iFgQdyNx9eIhnGqq0=
|
||||
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
|
||||
github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w=
|
||||
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
|
||||
github.com/go-openapi/swag v0.19.2 h1:jvO6bCMBEilGwMfHhrd61zIID4oIFdwb76V17SM88dE=
|
||||
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
|
||||
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
|
|
@ -24,12 +28,14 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
|
|||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
|
||||
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM=
|
||||
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
|
|
|
|||
|
|
@ -14,41 +14,11 @@
|
|||
|
||||
package spec
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/go-openapi/swag"
|
||||
)
|
||||
|
||||
// ContactInfo contact information for the exposed API.
|
||||
//
|
||||
// For more information: http://goo.gl/8us55a#contactObject
|
||||
type ContactInfo struct {
|
||||
ContactInfoProps
|
||||
VendorExtensible
|
||||
}
|
||||
|
||||
type ContactInfoProps struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
Email string `json:"email,omitempty"`
|
||||
}
|
||||
|
||||
func (c *ContactInfo) UnmarshalJSON(data []byte) error {
|
||||
if err := json.Unmarshal(data, &c.ContactInfoProps); err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, &c.VendorExtensible)
|
||||
}
|
||||
|
||||
func (c ContactInfo) MarshalJSON() ([]byte, error) {
|
||||
b1, err := json.Marshal(c.ContactInfoProps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b2, err := json.Marshal(c.VendorExtensible)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return swag.ConcatJSON(b1, b2), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,40 +14,10 @@
|
|||
|
||||
package spec
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
|
||||
"github.com/go-openapi/swag"
|
||||
)
|
||||
|
||||
// License information for the exposed API.
|
||||
//
|
||||
// For more information: http://goo.gl/8us55a#licenseObject
|
||||
type License struct {
|
||||
LicenseProps
|
||||
VendorExtensible
|
||||
}
|
||||
|
||||
type LicenseProps struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
func (l *License) UnmarshalJSON(data []byte) error {
|
||||
if err := json.Unmarshal(data, &l.LicenseProps); err != nil {
|
||||
return err
|
||||
}
|
||||
return json.Unmarshal(data, &l.VendorExtensible)
|
||||
}
|
||||
|
||||
func (l License) MarshalJSON() ([]byte, error) {
|
||||
b1, err := json.Marshal(l.LicenseProps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b2, err := json.Marshal(l.VendorExtensible)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return swag.ConcatJSON(b1, b2), nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,22 +20,3 @@ linters:
|
|||
- lll
|
||||
- gochecknoinits
|
||||
- gochecknoglobals
|
||||
- nlreturn
|
||||
- testpackage
|
||||
- wrapcheck
|
||||
- gomnd
|
||||
- exhaustive
|
||||
- exhaustivestruct
|
||||
- goerr113
|
||||
- wsl
|
||||
- whitespace
|
||||
- gofumpt
|
||||
- godot
|
||||
- nestif
|
||||
- godox
|
||||
- funlen
|
||||
- gci
|
||||
- gocognit
|
||||
- paralleltest
|
||||
- thelper
|
||||
- ifshort
|
||||
|
|
|
|||
|
|
@ -1,34 +1,12 @@
|
|||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
go:
|
||||
- 1.14.x
|
||||
- 1.x
|
||||
arch:
|
||||
- amd64
|
||||
jobs:
|
||||
include:
|
||||
# include arch ppc, but only for latest go version - skip testing for race
|
||||
- go: 1.x
|
||||
arch: ppc64le
|
||||
install: ~
|
||||
script:
|
||||
- go test -v
|
||||
|
||||
#- go: 1.x
|
||||
# arch: arm
|
||||
# install: ~
|
||||
# script:
|
||||
# - go test -v
|
||||
|
||||
# include linting job, but only for latest go version and amd64 arch
|
||||
- go: 1.x
|
||||
arch: amd64
|
||||
install:
|
||||
go get github.com/golangci/golangci-lint/cmd/golangci-lint
|
||||
script:
|
||||
- golangci-lint run --new-from-rev master
|
||||
- 1.11.x
|
||||
- 1.12.x
|
||||
install:
|
||||
- GO111MODULE=off go get -u gotest.tools/gotestsum
|
||||
env:
|
||||
- GO111MODULE=on
|
||||
language: go
|
||||
notifications:
|
||||
slack:
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
[](https://raw.githubusercontent.com/go-openapi/swag/master/LICENSE)
|
||||
[](http://godoc.org/github.com/go-openapi/swag)
|
||||
[](https://golangci.com)
|
||||
[](https://goreportcard.com/report/github.com/go-openapi/swag)
|
||||
|
||||
Contains a bunch of helper functions for go-openapi and go-swagger projects.
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ func ConvertFloat64(str string) (float64, error) {
|
|||
return strconv.ParseFloat(str, 64)
|
||||
}
|
||||
|
||||
// ConvertInt8 turn a string into an int8
|
||||
// ConvertInt8 turn a string into int8 boolean
|
||||
func ConvertInt8(str string) (int8, error) {
|
||||
i, err := strconv.ParseInt(str, 10, 8)
|
||||
if err != nil {
|
||||
|
|
@ -97,7 +97,7 @@ func ConvertInt8(str string) (int8, error) {
|
|||
return int8(i), nil
|
||||
}
|
||||
|
||||
// ConvertInt16 turn a string into an int16
|
||||
// ConvertInt16 turn a string into a int16
|
||||
func ConvertInt16(str string) (int16, error) {
|
||||
i, err := strconv.ParseInt(str, 10, 16)
|
||||
if err != nil {
|
||||
|
|
@ -106,7 +106,7 @@ func ConvertInt16(str string) (int16, error) {
|
|||
return int16(i), nil
|
||||
}
|
||||
|
||||
// ConvertInt32 turn a string into an int32
|
||||
// ConvertInt32 turn a string into a int32
|
||||
func ConvertInt32(str string) (int32, error) {
|
||||
i, err := strconv.ParseInt(str, 10, 32)
|
||||
if err != nil {
|
||||
|
|
@ -115,12 +115,12 @@ func ConvertInt32(str string) (int32, error) {
|
|||
return int32(i), nil
|
||||
}
|
||||
|
||||
// ConvertInt64 turn a string into an int64
|
||||
// ConvertInt64 turn a string into a int64
|
||||
func ConvertInt64(str string) (int64, error) {
|
||||
return strconv.ParseInt(str, 10, 64)
|
||||
}
|
||||
|
||||
// ConvertUint8 turn a string into an uint8
|
||||
// ConvertUint8 turn a string into a uint8
|
||||
func ConvertUint8(str string) (uint8, error) {
|
||||
i, err := strconv.ParseUint(str, 10, 8)
|
||||
if err != nil {
|
||||
|
|
@ -129,7 +129,7 @@ func ConvertUint8(str string) (uint8, error) {
|
|||
return uint8(i), nil
|
||||
}
|
||||
|
||||
// ConvertUint16 turn a string into an uint16
|
||||
// ConvertUint16 turn a string into a uint16
|
||||
func ConvertUint16(str string) (uint16, error) {
|
||||
i, err := strconv.ParseUint(str, 10, 16)
|
||||
if err != nil {
|
||||
|
|
@ -138,7 +138,7 @@ func ConvertUint16(str string) (uint16, error) {
|
|||
return uint16(i), nil
|
||||
}
|
||||
|
||||
// ConvertUint32 turn a string into an uint32
|
||||
// ConvertUint32 turn a string into a uint32
|
||||
func ConvertUint32(str string) (uint32, error) {
|
||||
i, err := strconv.ParseUint(str, 10, 32)
|
||||
if err != nil {
|
||||
|
|
@ -147,7 +147,7 @@ func ConvertUint32(str string) (uint32, error) {
|
|||
return uint32(i), nil
|
||||
}
|
||||
|
||||
// ConvertUint64 turn a string into an uint64
|
||||
// ConvertUint64 turn a string into a uint64
|
||||
func ConvertUint64(str string) (uint64, error) {
|
||||
return strconv.ParseUint(str, 10, 64)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -181,12 +181,12 @@ func IntValueMap(src map[string]*int) map[string]int {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Int32 returns a pointer to of the int32 value passed in.
|
||||
// Int32 returns a pointer to of the int64 value passed in.
|
||||
func Int32(v int32) *int32 {
|
||||
return &v
|
||||
}
|
||||
|
||||
// Int32Value returns the value of the int32 pointer passed in or
|
||||
// Int32Value returns the value of the int64 pointer passed in or
|
||||
// 0 if the pointer is nil.
|
||||
func Int32Value(v *int32) int32 {
|
||||
if v != nil {
|
||||
|
|
@ -195,7 +195,7 @@ func Int32Value(v *int32) int32 {
|
|||
return 0
|
||||
}
|
||||
|
||||
// Int32Slice converts a slice of int32 values into a slice of
|
||||
// Int32Slice converts a slice of int64 values into a slice of
|
||||
// int32 pointers
|
||||
func Int32Slice(src []int32) []*int32 {
|
||||
dst := make([]*int32, len(src))
|
||||
|
|
@ -299,80 +299,13 @@ func Int64ValueMap(src map[string]*int64) map[string]int64 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint16 returns a pointer to of the uint16 value passed in.
|
||||
func Uint16(v uint16) *uint16 {
|
||||
return &v
|
||||
}
|
||||
|
||||
// Uint16Value returns the value of the uint16 pointer passed in or
|
||||
// 0 if the pointer is nil.
|
||||
func Uint16Value(v *uint16) uint16 {
|
||||
if v != nil {
|
||||
return *v
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// Uint16Slice converts a slice of uint16 values into a slice of
|
||||
// uint16 pointers
|
||||
func Uint16Slice(src []uint16) []*uint16 {
|
||||
dst := make([]*uint16, len(src))
|
||||
for i := 0; i < len(src); i++ {
|
||||
dst[i] = &(src[i])
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Uint16ValueSlice converts a slice of uint16 pointers into a slice of
|
||||
// uint16 values
|
||||
func Uint16ValueSlice(src []*uint16) []uint16 {
|
||||
dst := make([]uint16, len(src))
|
||||
|
||||
for i := 0; i < len(src); i++ {
|
||||
if src[i] != nil {
|
||||
dst[i] = *(src[i])
|
||||
}
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Uint16Map converts a string map of uint16 values into a string
|
||||
// map of uint16 pointers
|
||||
func Uint16Map(src map[string]uint16) map[string]*uint16 {
|
||||
dst := make(map[string]*uint16)
|
||||
|
||||
for k, val := range src {
|
||||
v := val
|
||||
dst[k] = &v
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Uint16ValueMap converts a string map of uint16 pointers into a string
|
||||
// map of uint16 values
|
||||
func Uint16ValueMap(src map[string]*uint16) map[string]uint16 {
|
||||
dst := make(map[string]uint16)
|
||||
|
||||
for k, val := range src {
|
||||
if val != nil {
|
||||
dst[k] = *val
|
||||
}
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Uint returns a pointer to of the uint value passed in.
|
||||
// Uint returns a pouinter to of the uint value passed in.
|
||||
func Uint(v uint) *uint {
|
||||
return &v
|
||||
}
|
||||
|
||||
// UintValue returns the value of the uint pointer passed in or
|
||||
// 0 if the pointer is nil.
|
||||
// UintValue returns the value of the uint pouinter passed in or
|
||||
// 0 if the pouinter is nil.
|
||||
func UintValue(v *uint) uint {
|
||||
if v != nil {
|
||||
return *v
|
||||
|
|
@ -380,8 +313,8 @@ func UintValue(v *uint) uint {
|
|||
return 0
|
||||
}
|
||||
|
||||
// UintSlice converts a slice of uint values into a slice of
|
||||
// uint pointers
|
||||
// UintSlice converts a slice of uint values uinto a slice of
|
||||
// uint pouinters
|
||||
func UintSlice(src []uint) []*uint {
|
||||
dst := make([]*uint, len(src))
|
||||
for i := 0; i < len(src); i++ {
|
||||
|
|
@ -390,7 +323,7 @@ func UintSlice(src []uint) []*uint {
|
|||
return dst
|
||||
}
|
||||
|
||||
// UintValueSlice converts a slice of uint pointers into a slice of
|
||||
// UintValueSlice converts a slice of uint pouinters uinto a slice of
|
||||
// uint values
|
||||
func UintValueSlice(src []*uint) []uint {
|
||||
dst := make([]uint, len(src))
|
||||
|
|
@ -402,8 +335,8 @@ func UintValueSlice(src []*uint) []uint {
|
|||
return dst
|
||||
}
|
||||
|
||||
// UintMap converts a string map of uint values into a string
|
||||
// map of uint pointers
|
||||
// UintMap converts a string map of uint values uinto a string
|
||||
// map of uint pouinters
|
||||
func UintMap(src map[string]uint) map[string]*uint {
|
||||
dst := make(map[string]*uint)
|
||||
for k, val := range src {
|
||||
|
|
@ -413,7 +346,7 @@ func UintMap(src map[string]uint) map[string]*uint {
|
|||
return dst
|
||||
}
|
||||
|
||||
// UintValueMap converts a string map of uint pointers into a string
|
||||
// UintValueMap converts a string map of uint pouinters uinto a string
|
||||
// map of uint values
|
||||
func UintValueMap(src map[string]*uint) map[string]uint {
|
||||
dst := make(map[string]uint)
|
||||
|
|
@ -425,13 +358,13 @@ func UintValueMap(src map[string]*uint) map[string]uint {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint32 returns a pointer to of the uint32 value passed in.
|
||||
// Uint32 returns a pouinter to of the uint64 value passed in.
|
||||
func Uint32(v uint32) *uint32 {
|
||||
return &v
|
||||
}
|
||||
|
||||
// Uint32Value returns the value of the uint32 pointer passed in or
|
||||
// 0 if the pointer is nil.
|
||||
// Uint32Value returns the value of the uint64 pouinter passed in or
|
||||
// 0 if the pouinter is nil.
|
||||
func Uint32Value(v *uint32) uint32 {
|
||||
if v != nil {
|
||||
return *v
|
||||
|
|
@ -439,8 +372,8 @@ func Uint32Value(v *uint32) uint32 {
|
|||
return 0
|
||||
}
|
||||
|
||||
// Uint32Slice converts a slice of uint32 values into a slice of
|
||||
// uint32 pointers
|
||||
// Uint32Slice converts a slice of uint64 values uinto a slice of
|
||||
// uint32 pouinters
|
||||
func Uint32Slice(src []uint32) []*uint32 {
|
||||
dst := make([]*uint32, len(src))
|
||||
for i := 0; i < len(src); i++ {
|
||||
|
|
@ -449,7 +382,7 @@ func Uint32Slice(src []uint32) []*uint32 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint32ValueSlice converts a slice of uint32 pointers into a slice of
|
||||
// Uint32ValueSlice converts a slice of uint32 pouinters uinto a slice of
|
||||
// uint32 values
|
||||
func Uint32ValueSlice(src []*uint32) []uint32 {
|
||||
dst := make([]uint32, len(src))
|
||||
|
|
@ -461,8 +394,8 @@ func Uint32ValueSlice(src []*uint32) []uint32 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint32Map converts a string map of uint32 values into a string
|
||||
// map of uint32 pointers
|
||||
// Uint32Map converts a string map of uint32 values uinto a string
|
||||
// map of uint32 pouinters
|
||||
func Uint32Map(src map[string]uint32) map[string]*uint32 {
|
||||
dst := make(map[string]*uint32)
|
||||
for k, val := range src {
|
||||
|
|
@ -472,7 +405,7 @@ func Uint32Map(src map[string]uint32) map[string]*uint32 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint32ValueMap converts a string map of uint32 pointers into a string
|
||||
// Uint32ValueMap converts a string map of uint32 pouinters uinto a string
|
||||
// map of uint32 values
|
||||
func Uint32ValueMap(src map[string]*uint32) map[string]uint32 {
|
||||
dst := make(map[string]uint32)
|
||||
|
|
@ -484,13 +417,13 @@ func Uint32ValueMap(src map[string]*uint32) map[string]uint32 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint64 returns a pointer to of the uint64 value passed in.
|
||||
// Uint64 returns a pouinter to of the uint64 value passed in.
|
||||
func Uint64(v uint64) *uint64 {
|
||||
return &v
|
||||
}
|
||||
|
||||
// Uint64Value returns the value of the uint64 pointer passed in or
|
||||
// 0 if the pointer is nil.
|
||||
// Uint64Value returns the value of the uint64 pouinter passed in or
|
||||
// 0 if the pouinter is nil.
|
||||
func Uint64Value(v *uint64) uint64 {
|
||||
if v != nil {
|
||||
return *v
|
||||
|
|
@ -498,8 +431,8 @@ func Uint64Value(v *uint64) uint64 {
|
|||
return 0
|
||||
}
|
||||
|
||||
// Uint64Slice converts a slice of uint64 values into a slice of
|
||||
// uint64 pointers
|
||||
// Uint64Slice converts a slice of uint64 values uinto a slice of
|
||||
// uint64 pouinters
|
||||
func Uint64Slice(src []uint64) []*uint64 {
|
||||
dst := make([]*uint64, len(src))
|
||||
for i := 0; i < len(src); i++ {
|
||||
|
|
@ -508,7 +441,7 @@ func Uint64Slice(src []uint64) []*uint64 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint64ValueSlice converts a slice of uint64 pointers into a slice of
|
||||
// Uint64ValueSlice converts a slice of uint64 pouinters uinto a slice of
|
||||
// uint64 values
|
||||
func Uint64ValueSlice(src []*uint64) []uint64 {
|
||||
dst := make([]uint64, len(src))
|
||||
|
|
@ -520,8 +453,8 @@ func Uint64ValueSlice(src []*uint64) []uint64 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint64Map converts a string map of uint64 values into a string
|
||||
// map of uint64 pointers
|
||||
// Uint64Map converts a string map of uint64 values uinto a string
|
||||
// map of uint64 pouinters
|
||||
func Uint64Map(src map[string]uint64) map[string]*uint64 {
|
||||
dst := make(map[string]*uint64)
|
||||
for k, val := range src {
|
||||
|
|
@ -531,7 +464,7 @@ func Uint64Map(src map[string]uint64) map[string]*uint64 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Uint64ValueMap converts a string map of uint64 pointers into a string
|
||||
// Uint64ValueMap converts a string map of uint64 pouinters uinto a string
|
||||
// map of uint64 values
|
||||
func Uint64ValueMap(src map[string]*uint64) map[string]uint64 {
|
||||
dst := make(map[string]uint64)
|
||||
|
|
@ -543,74 +476,6 @@ func Uint64ValueMap(src map[string]*uint64) map[string]uint64 {
|
|||
return dst
|
||||
}
|
||||
|
||||
// Float32 returns a pointer to of the float32 value passed in.
|
||||
func Float32(v float32) *float32 {
|
||||
return &v
|
||||
}
|
||||
|
||||
// Float32Value returns the value of the float32 pointer passed in or
|
||||
// 0 if the pointer is nil.
|
||||
func Float32Value(v *float32) float32 {
|
||||
if v != nil {
|
||||
return *v
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// Float32Slice converts a slice of float32 values into a slice of
|
||||
// float32 pointers
|
||||
func Float32Slice(src []float32) []*float32 {
|
||||
dst := make([]*float32, len(src))
|
||||
|
||||
for i := 0; i < len(src); i++ {
|
||||
dst[i] = &(src[i])
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Float32ValueSlice converts a slice of float32 pointers into a slice of
|
||||
// float32 values
|
||||
func Float32ValueSlice(src []*float32) []float32 {
|
||||
dst := make([]float32, len(src))
|
||||
|
||||
for i := 0; i < len(src); i++ {
|
||||
if src[i] != nil {
|
||||
dst[i] = *(src[i])
|
||||
}
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Float32Map converts a string map of float32 values into a string
|
||||
// map of float32 pointers
|
||||
func Float32Map(src map[string]float32) map[string]*float32 {
|
||||
dst := make(map[string]*float32)
|
||||
|
||||
for k, val := range src {
|
||||
v := val
|
||||
dst[k] = &v
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Float32ValueMap converts a string map of float32 pointers into a string
|
||||
// map of float32 values
|
||||
func Float32ValueMap(src map[string]*float32) map[string]float32 {
|
||||
dst := make(map[string]float32)
|
||||
|
||||
for k, val := range src {
|
||||
if val != nil {
|
||||
dst[k] = *val
|
||||
}
|
||||
}
|
||||
|
||||
return dst
|
||||
}
|
||||
|
||||
// Float64 returns a pointer to of the float64 value passed in.
|
||||
func Float64(v float64) *float64 {
|
||||
return &v
|
||||
|
|
|
|||
|
|
@ -2,17 +2,13 @@ module github.com/go-openapi/swag
|
|||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/mailru/easyjson v0.7.6
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
|
||||
github.com/stretchr/testify v1.6.1
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
|
||||
github.com/kr/pretty v0.1.0 // indirect
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63
|
||||
github.com/stretchr/testify v1.3.0
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
|
||||
gopkg.in/yaml.v2 v2.2.2
|
||||
)
|
||||
|
||||
replace github.com/golang/lint => golang.org/x/lint v0.0.0-20190409202823-959b441ac422
|
||||
|
||||
replace sourcegraph.com/sourcegraph/go-diff => github.com/sourcegraph/go-diff v0.5.1
|
||||
|
||||
go 1.11
|
||||
|
|
|
|||
|
|
@ -1,29 +1,20 @@
|
|||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA=
|
||||
github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63 h1:nTT4s92Dgz2HlrB2NaMgvlfqHH39OgMhA7z3PK7PGD4=
|
||||
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ type ejUnmarshaler interface {
|
|||
UnmarshalEasyJSON(w *jlexer.Lexer)
|
||||
}
|
||||
|
||||
// WriteJSON writes json data, prefers finding an appropriate interface to short-circuit the marshaler
|
||||
// WriteJSON writes json data, prefers finding an appropriate interface to short-circuit the marshaller
|
||||
// so it takes the fastest option available.
|
||||
func WriteJSON(data interface{}) ([]byte, error) {
|
||||
if d, ok := data.(ejMarshaler); ok {
|
||||
|
|
@ -65,8 +65,8 @@ func WriteJSON(data interface{}) ([]byte, error) {
|
|||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
// ReadJSON reads json data, prefers finding an appropriate interface to short-circuit the unmarshaler
|
||||
// so it takes the fastest option available
|
||||
// ReadJSON reads json data, prefers finding an appropriate interface to short-circuit the unmarshaller
|
||||
// so it takes the fastes option available
|
||||
func ReadJSON(data []byte, value interface{}) error {
|
||||
trimmedData := bytes.Trim(data, "\x00")
|
||||
if d, ok := value.(ejUnmarshaler); ok {
|
||||
|
|
@ -189,7 +189,7 @@ func FromDynamicJSON(data, target interface{}) error {
|
|||
return json.Unmarshal(b, target)
|
||||
}
|
||||
|
||||
// NameProvider represents an object capable of translating from go property names
|
||||
// NameProvider represents an object capabale of translating from go property names
|
||||
// to json property names
|
||||
// This type is thread-safe.
|
||||
type NameProvider struct {
|
||||
|
|
|
|||
|
|
@ -19,9 +19,7 @@ import (
|
|||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
|
@ -29,15 +27,6 @@ import (
|
|||
// LoadHTTPTimeout the default timeout for load requests
|
||||
var LoadHTTPTimeout = 30 * time.Second
|
||||
|
||||
// LoadHTTPBasicAuthUsername the username to use when load requests require basic auth
|
||||
var LoadHTTPBasicAuthUsername = ""
|
||||
|
||||
// LoadHTTPBasicAuthPassword the password to use when load requests require basic auth
|
||||
var LoadHTTPBasicAuthPassword = ""
|
||||
|
||||
// LoadHTTPCustomHeaders an optional collection of custom HTTP headers for load requests
|
||||
var LoadHTTPCustomHeaders = map[string]string{}
|
||||
|
||||
// LoadFromFileOrHTTP loads the bytes from a file or a remote http server based on the path passed in
|
||||
func LoadFromFileOrHTTP(path string) ([]byte, error) {
|
||||
return LoadStrategy(path, ioutil.ReadFile, loadHTTPBytes(LoadHTTPTimeout))(path)
|
||||
|
|
@ -59,26 +48,6 @@ func LoadStrategy(path string, local, remote func(string) ([]byte, error)) func(
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if strings.HasPrefix(pth, `file://`) {
|
||||
if runtime.GOOS == "windows" {
|
||||
// support for canonical file URIs on windows.
|
||||
// Zero tolerance here for dodgy URIs.
|
||||
u, _ := url.Parse(upth)
|
||||
if u.Host != "" {
|
||||
// assume UNC name (volume share)
|
||||
// file://host/share/folder\... ==> \\host\share\path\folder
|
||||
// NOTE: UNC port not yet supported
|
||||
upth = strings.Join([]string{`\`, u.Host, u.Path}, `\`)
|
||||
} else {
|
||||
// file:///c:/folder/... ==> just remove the leading slash
|
||||
upth = strings.TrimPrefix(upth, `file:///`)
|
||||
}
|
||||
} else {
|
||||
upth = strings.TrimPrefix(upth, `file://`)
|
||||
}
|
||||
}
|
||||
|
||||
return local(filepath.FromSlash(upth))
|
||||
}
|
||||
}
|
||||
|
|
@ -86,19 +55,10 @@ func LoadStrategy(path string, local, remote func(string) ([]byte, error)) func(
|
|||
func loadHTTPBytes(timeout time.Duration) func(path string) ([]byte, error) {
|
||||
return func(path string) ([]byte, error) {
|
||||
client := &http.Client{Timeout: timeout}
|
||||
req, err := http.NewRequest("GET", path, nil) // nolint: noctx
|
||||
req, err := http.NewRequest("GET", path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if LoadHTTPBasicAuthUsername != "" && LoadHTTPBasicAuthPassword != "" {
|
||||
req.SetBasicAuth(LoadHTTPBasicAuthUsername, LoadHTTPBasicAuthPassword)
|
||||
}
|
||||
|
||||
for key, val := range LoadHTTPCustomHeaders {
|
||||
req.Header.Set(key, val)
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
defer func() {
|
||||
if resp != nil {
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ var isInitialism func(string) bool
|
|||
// GoNamePrefixFunc sets an optional rule to prefix go names
|
||||
// which do not start with a letter.
|
||||
//
|
||||
// e.g. to help convert "123" into "{prefix}123"
|
||||
// e.g. to help converting "123" into "{prefix}123"
|
||||
//
|
||||
// The default is to prefix with "X"
|
||||
var GoNamePrefixFunc func(string) string
|
||||
|
|
|
|||
|
|
@ -1,16 +0,0 @@
|
|||
sudo: false
|
||||
language: go
|
||||
go:
|
||||
- "1.10"
|
||||
- "1.11"
|
||||
- "1.12"
|
||||
matrix:
|
||||
allow_failures:
|
||||
- go: master
|
||||
fast_finish: true
|
||||
script:
|
||||
- go get -t -v ./...
|
||||
- diff -u <(echo -n) <(gofmt -d -s .)
|
||||
- go generate -x ./... && git diff --exit-code; code=$?; git checkout -- .; (exit $code) # Check that go generate ./... produces a zero diff; clean up any changes afterwards.
|
||||
- go vet .
|
||||
- go test -v -race ./...
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
# Change Log
|
||||
# All notable changes to this project will be documented in this file.
|
||||
# This project adheres to [Semantic Versioning](http://semver.org/).
|
||||
|
||||
## [0.1.0] UNRELEASED
|
||||
### Init
|
||||
- Initial development.
|
||||
### Changed
|
||||
- Create a database of forbidden licenses to make matching just forbidden
|
||||
licenses quicker.
|
||||
- Remove non-words from the license texts. It makes the matching more precise.
|
||||
This subsumes what the "remove common prefix" and a few other passes did.
|
||||
- Remove the "lattice" structure in favor of a simpler and smaller list of
|
||||
substrings. This gives us a tremendous speed increase of up to 2x.
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
# How to contribute
|
||||
|
||||
We'd love to accept your patches and contributions to this project. There are
|
||||
just a few small guidelines you need to follow.
|
||||
|
||||
## Contributor License Agreement
|
||||
|
||||
Contributions to this project must be accompanied by a Contributor License
|
||||
Agreement. You (or your employer) retain the copyright to your contribution,
|
||||
this simply gives us permission to use and redistribute your contributions as
|
||||
part of the project. Head over to <https://cla.developers.google.com/> to see
|
||||
your current agreements on file or to sign a new one.
|
||||
|
||||
You generally only need to submit a CLA once, so if you've already submitted
|
||||
one (even if it was for a different project), you probably don't need to do it
|
||||
again.
|
||||
|
||||
## Code reviews
|
||||
|
||||
All submissions, including submissions by project members, require review. We
|
||||
use GitHub pull requests for this purpose. Consult [GitHub Help] for more
|
||||
information on using pull requests.
|
||||
|
||||
[GitHub Help]: https://help.github.com/articles/about-pull-requests/
|
||||
|
|
@ -1,202 +0,0 @@
|
|||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
# License Classifier
|
||||
|
||||
[](https://travis-ci.org/google/licenseclassifier)
|
||||
|
||||
## Introduction
|
||||
|
||||
The license classifier is a library and set of tools that can analyze text to
|
||||
determine what type of license it contains. It searches for license texts in a
|
||||
file and compares them to an archive of known licenses. These files could be,
|
||||
e.g., `LICENSE` files with a single or multiple licenses in it, or source code
|
||||
files with the license text in a comment.
|
||||
|
||||
A "confidence level" is associated with each result indicating how close the
|
||||
match was. A confidence level of `1.0` indicates an exact match, while a
|
||||
confidence level of `0.0` indicates that no license was able to match the text.
|
||||
|
||||
## Adding a new license
|
||||
|
||||
Adding a new license is straight-forward:
|
||||
|
||||
1. Create a file in `licenses/`.
|
||||
|
||||
* The filename should be the name of the license or its abbreviation. If
|
||||
the license is an Open Source license, use the appropriate identifier
|
||||
specified at https://spdx.org/licenses/.
|
||||
* If the license is the "header" version of the license, append the suffix
|
||||
"`.header`" to it. See `licenses/README.md` for more details.
|
||||
|
||||
2. Add the license name to the list in `license_type.go`.
|
||||
|
||||
3. Regenerate the `licenses.db` file by running the license serializer:
|
||||
|
||||
```shell
|
||||
$ license_serializer -output licenseclassifier/licenses
|
||||
```
|
||||
|
||||
4. Create and run appropriate tests to verify that the license is indeed
|
||||
present.
|
||||
|
||||
## Tools
|
||||
|
||||
### Identify license
|
||||
|
||||
`identify_license` is a command line tool that can identify the license(s)
|
||||
within a file.
|
||||
|
||||
```shell
|
||||
$ identify_license LICENSE
|
||||
LICENSE: GPL-2.0 (confidence: 1, offset: 0, extent: 14794)
|
||||
LICENSE: LGPL-2.1 (confidence: 1, offset: 18366, extent: 23829)
|
||||
LICENSE: MIT (confidence: 1, offset: 17255, extent: 1059)
|
||||
```
|
||||
|
||||
### License serializer
|
||||
|
||||
The `license_serializer` tool regenerates the `licenses.db` archive. The archive
|
||||
contains preprocessed license texts for quicker comparisons against unknown
|
||||
texts.
|
||||
|
||||
```shell
|
||||
$ license_serializer -output licenseclassifier/licenses
|
||||
```
|
||||
|
||||
----
|
||||
This is not an official Google product (experimental or otherwise), it is just
|
||||
code that happens to be owned by Google.
|
||||
|
|
@ -1,472 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package licenseclassifier provides methods to identify the open source
|
||||
// license that most closely matches an unknown license.
|
||||
package licenseclassifier
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
|
||||
"github.com/google/licenseclassifier/stringclassifier"
|
||||
"github.com/google/licenseclassifier/stringclassifier/searchset"
|
||||
)
|
||||
|
||||
// DefaultConfidenceThreshold is the minimum confidence percentage we're willing to accept in order
|
||||
// to say that a match is good.
|
||||
const DefaultConfidenceThreshold = 0.80
|
||||
|
||||
var (
|
||||
// Normalizers is a list of functions that get applied to the strings
|
||||
// before they are registered with the string classifier.
|
||||
Normalizers = []stringclassifier.NormalizeFunc{
|
||||
html.UnescapeString,
|
||||
removeShebangLine,
|
||||
RemoveNonWords,
|
||||
NormalizeEquivalentWords,
|
||||
NormalizePunctuation,
|
||||
strings.ToLower,
|
||||
removeIgnorableTexts,
|
||||
stringclassifier.FlattenWhitespace,
|
||||
strings.TrimSpace,
|
||||
}
|
||||
|
||||
// commonLicenseWords are words that are common to all known licenses.
|
||||
// If an unknown text doesn't have at least one of these, then we can
|
||||
// ignore it.
|
||||
commonLicenseWords = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)\bcode\b`),
|
||||
regexp.MustCompile(`(?i)\blicense\b`),
|
||||
regexp.MustCompile(`(?i)\boriginal\b`),
|
||||
regexp.MustCompile(`(?i)\brights\b`),
|
||||
regexp.MustCompile(`(?i)\bsoftware\b`),
|
||||
regexp.MustCompile(`(?i)\bterms\b`),
|
||||
regexp.MustCompile(`(?i)\bversion\b`),
|
||||
regexp.MustCompile(`(?i)\bwork\b`),
|
||||
}
|
||||
)
|
||||
|
||||
// License is a classifier pre-loaded with known open source licenses.
|
||||
type License struct {
|
||||
c *stringclassifier.Classifier
|
||||
|
||||
// Threshold is the lowest confidence percentage acceptable for the
|
||||
// classifier.
|
||||
Threshold float64
|
||||
|
||||
// archive is a function that must return the contents of the license archive.
|
||||
// When archive is nil, ReadLicenseFile(LicenseFile) is used to retrieve the
|
||||
// contents.
|
||||
archive func() ([]byte, error)
|
||||
}
|
||||
|
||||
// OptionFunc set options on a License struct.
|
||||
type OptionFunc func(l *License) error
|
||||
|
||||
// Archive is an OptionFunc to specify the location of the license archive file.
|
||||
func Archive(f string) OptionFunc {
|
||||
return func(l *License) error {
|
||||
l.archive = func() ([]byte, error) { return ReadLicenseFile(f) }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ArchiveBytes is an OptionFunc that provides the contents of the license archive file.
|
||||
// The caller must not overwrite the contents of b as it is not copied.
|
||||
func ArchiveBytes(b []byte) OptionFunc {
|
||||
return func(l *License) error {
|
||||
l.archive = func() ([]byte, error) { return b, nil }
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ArchiveFunc is an OptionFunc that provides a function that must return the contents
|
||||
// of the license archive file.
|
||||
func ArchiveFunc(f func() ([]byte, error)) OptionFunc {
|
||||
return func(l *License) error {
|
||||
l.archive = f
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// New creates a license classifier and pre-loads it with known open source licenses.
|
||||
func New(threshold float64, options ...OptionFunc) (*License, error) {
|
||||
classifier := &License{
|
||||
c: stringclassifier.New(threshold, Normalizers...),
|
||||
Threshold: threshold,
|
||||
}
|
||||
|
||||
for _, o := range options {
|
||||
err := o(classifier)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error setting option %v: %v", o, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := classifier.registerLicenses(); err != nil {
|
||||
return nil, fmt.Errorf("cannot register licenses from archive: %v", err)
|
||||
}
|
||||
return classifier, nil
|
||||
}
|
||||
|
||||
// NewWithForbiddenLicenses creates a license classifier and pre-loads it with
|
||||
// known open source licenses which are forbidden.
|
||||
func NewWithForbiddenLicenses(threshold float64, options ...OptionFunc) (*License, error) {
|
||||
opts := []OptionFunc{Archive(ForbiddenLicenseArchive)}
|
||||
opts = append(opts, options...)
|
||||
return New(threshold, opts...)
|
||||
}
|
||||
|
||||
// WithinConfidenceThreshold returns true if the confidence value is above or
|
||||
// equal to the confidence threshold.
|
||||
func (c *License) WithinConfidenceThreshold(conf float64) bool {
|
||||
return conf > c.Threshold || math.Abs(conf-c.Threshold) < math.SmallestNonzeroFloat64
|
||||
}
|
||||
|
||||
// NearestMatch returns the "nearest" match to the given set of known licenses.
|
||||
// Returned are the name of the license, and a confidence percentage indicating
|
||||
// how confident the classifier is in the result.
|
||||
func (c *License) NearestMatch(contents string) *stringclassifier.Match {
|
||||
if !c.hasCommonLicenseWords(contents) {
|
||||
return nil
|
||||
}
|
||||
m := c.c.NearestMatch(contents)
|
||||
m.Name = strings.TrimSuffix(m.Name, ".header")
|
||||
return m
|
||||
}
|
||||
|
||||
// MultipleMatch matches all licenses within an unknown text.
|
||||
func (c *License) MultipleMatch(contents string, includeHeaders bool) stringclassifier.Matches {
|
||||
norm := normalizeText(contents)
|
||||
if !c.hasCommonLicenseWords(norm) {
|
||||
return nil
|
||||
}
|
||||
|
||||
m := make(map[stringclassifier.Match]bool)
|
||||
var matches stringclassifier.Matches
|
||||
for _, v := range c.c.MultipleMatch(norm) {
|
||||
if !c.WithinConfidenceThreshold(v.Confidence) {
|
||||
continue
|
||||
}
|
||||
|
||||
if !includeHeaders && strings.HasSuffix(v.Name, ".header") {
|
||||
continue
|
||||
}
|
||||
|
||||
v.Name = strings.TrimSuffix(v.Name, ".header")
|
||||
if re, ok := forbiddenRegexps[v.Name]; ok && !re.MatchString(norm) {
|
||||
continue
|
||||
}
|
||||
if _, ok := m[*v]; !ok {
|
||||
m[*v] = true
|
||||
matches = append(matches, v)
|
||||
}
|
||||
}
|
||||
sort.Sort(matches)
|
||||
return matches
|
||||
}
|
||||
|
||||
func normalizeText(s string) string {
|
||||
for _, n := range Normalizers {
|
||||
s = n(s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// hasCommonLicenseWords returns true if the unknown text has at least one word
|
||||
// that's common to all licenses.
|
||||
func (c *License) hasCommonLicenseWords(s string) bool {
|
||||
for _, re := range commonLicenseWords {
|
||||
if re.MatchString(s) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type archivedValue struct {
|
||||
name string
|
||||
normalized string
|
||||
set *searchset.SearchSet
|
||||
}
|
||||
|
||||
// registerLicenses loads all known licenses and adds them to c as known values
|
||||
// for comparison. The allocated space after ingesting the 'licenses.db'
|
||||
// archive is ~167M.
|
||||
func (c *License) registerLicenses() error {
|
||||
var contents []byte
|
||||
var err error
|
||||
if c.archive == nil {
|
||||
contents, err = ReadLicenseFile(LicenseArchive)
|
||||
} else {
|
||||
contents, err = c.archive()
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
reader := bytes.NewReader(contents)
|
||||
gr, err := gzip.NewReader(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gr.Close()
|
||||
|
||||
tr := tar.NewReader(gr)
|
||||
|
||||
var muVals sync.Mutex
|
||||
var vals []archivedValue
|
||||
for i := 0; ; i++ {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
name := strings.TrimSuffix(hdr.Name, ".txt")
|
||||
|
||||
// Read normalized value.
|
||||
var b bytes.Buffer
|
||||
if _, err := io.Copy(&b, tr); err != nil {
|
||||
return err
|
||||
}
|
||||
normalized := b.String()
|
||||
b.Reset()
|
||||
|
||||
// Read precomputed hashes.
|
||||
hdr, err = tr.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := io.Copy(&b, tr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var set searchset.SearchSet
|
||||
searchset.Deserialize(&b, &set)
|
||||
|
||||
muVals.Lock()
|
||||
vals = append(vals, archivedValue{name, normalized, &set})
|
||||
muVals.Unlock()
|
||||
}
|
||||
|
||||
for _, v := range vals {
|
||||
if err = c.c.AddPrecomputedValue(v.name, v.normalized, v.set); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// endOfLicenseText is text commonly associated with the end of a license. We
|
||||
// can remove text that occurs after it.
|
||||
var endOfLicenseText = []string{
|
||||
"END OF TERMS AND CONDITIONS",
|
||||
}
|
||||
|
||||
// TrimExtraneousTrailingText removes text after an obvious end of the license
|
||||
// and does not include substantive text of the license.
|
||||
func TrimExtraneousTrailingText(s string) string {
|
||||
for _, e := range endOfLicenseText {
|
||||
if i := strings.LastIndex(s, e); i != -1 {
|
||||
return s[:i+len(e)]
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
var copyrightRE = regexp.MustCompile(`(?m)(?i:Copyright)\s+(?i:©\s+|\(c\)\s+)?(?:\d{2,4})(?:[-,]\s*\d{2,4})*,?\s*(?i:by)?\s*(.*?(?i:\s+Inc\.)?)[.,]?\s*(?i:All rights reserved\.?)?\s*$`)
|
||||
|
||||
// CopyrightHolder finds a copyright notification, if it exists, and returns
|
||||
// the copyright holder.
|
||||
func CopyrightHolder(contents string) string {
|
||||
matches := copyrightRE.FindStringSubmatch(contents)
|
||||
if len(matches) == 2 {
|
||||
return matches[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var publicDomainRE = regexp.MustCompile("(?i)(this file )?is( in the)? public domain")
|
||||
|
||||
// HasPublicDomainNotice performs a simple regex over the contents to see if a
|
||||
// public domain notice is in there. As you can imagine, this isn't 100%
|
||||
// definitive, but can be useful if a license match isn't found.
|
||||
func (c *License) HasPublicDomainNotice(contents string) bool {
|
||||
return publicDomainRE.FindString(contents) != ""
|
||||
}
|
||||
|
||||
// ignorableTexts is a list of lines at the start of the string we can remove
|
||||
// to get a cleaner match.
|
||||
var ignorableTexts = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)^(?:the )?mit license(?: \(mit\))?$`),
|
||||
regexp.MustCompile(`(?i)^(?:new )?bsd license$`),
|
||||
regexp.MustCompile(`(?i)^copyright and permission notice$`),
|
||||
regexp.MustCompile(`(?i)^copyright (\(c\) )?(\[yyyy\]|\d{4})[,.]? .*$`),
|
||||
regexp.MustCompile(`(?i)^(all|some) rights reserved\.?$`),
|
||||
regexp.MustCompile(`(?i)^@license$`),
|
||||
regexp.MustCompile(`^\s*$`),
|
||||
}
|
||||
|
||||
// removeIgnorableTexts removes common text, which is not important for
|
||||
// classification, that shows up before the body of the license.
|
||||
func removeIgnorableTexts(s string) string {
|
||||
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
|
||||
var start int
|
||||
for ; start < len(lines); start++ {
|
||||
line := strings.TrimSpace(lines[start])
|
||||
var matches bool
|
||||
for _, re := range ignorableTexts {
|
||||
if re.MatchString(line) {
|
||||
matches = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !matches {
|
||||
break
|
||||
}
|
||||
}
|
||||
end := len(lines)
|
||||
if start > end {
|
||||
return "\n"
|
||||
}
|
||||
return strings.Join(lines[start:end], "\n") + "\n"
|
||||
}
|
||||
|
||||
// removeShebangLine removes the '#!...' line if it's the first line in the
|
||||
// file. Note that if it's the only line in a comment, it won't be removed.
|
||||
func removeShebangLine(s string) string {
|
||||
lines := strings.Split(s, "\n")
|
||||
if len(lines) <= 1 || !strings.HasPrefix(lines[0], "#!") {
|
||||
return s
|
||||
}
|
||||
|
||||
return strings.Join(lines[1:], "\n")
|
||||
}
|
||||
|
||||
// isDecorative returns true if the line is made up purely of non-letter and
|
||||
// non-digit characters.
|
||||
func isDecorative(s string) bool {
|
||||
for _, c := range s {
|
||||
if unicode.IsLetter(c) || unicode.IsDigit(c) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var nonWords = regexp.MustCompile("[[:punct:]]+")
|
||||
|
||||
// RemoveNonWords removes non-words from the string.
|
||||
func RemoveNonWords(s string) string {
|
||||
return nonWords.ReplaceAllString(s, " ")
|
||||
}
|
||||
|
||||
// interchangeablePunctutation is punctuation that can be normalized.
|
||||
var interchangeablePunctuation = []struct {
|
||||
interchangeable *regexp.Regexp
|
||||
substitute string
|
||||
}{
|
||||
// Hyphen, Dash, En Dash, and Em Dash.
|
||||
{regexp.MustCompile(`[-‒–—]`), "-"},
|
||||
// Single, Double, Curly Single, and Curly Double.
|
||||
{regexp.MustCompile("['\"`‘’“”]"), "'"},
|
||||
// Copyright.
|
||||
{regexp.MustCompile("©"), "(c)"},
|
||||
// Hyphen-separated words.
|
||||
{regexp.MustCompile(`(\S)-\s+(\S)`), "${1}-${2}"},
|
||||
// Currency and Section. (Different copies of the CDDL use each marker.)
|
||||
{regexp.MustCompile("[§¤]"), "(s)"},
|
||||
// Middle Dot
|
||||
{regexp.MustCompile("·"), "*"},
|
||||
}
|
||||
|
||||
// NormalizePunctuation takes all hyphens and quotes and normalizes them.
|
||||
func NormalizePunctuation(s string) string {
|
||||
for _, iw := range interchangeablePunctuation {
|
||||
s = iw.interchangeable.ReplaceAllString(s, iw.substitute)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// interchangeableWords are words we can substitute for a normalized form
|
||||
// without changing the meaning of the license. See
|
||||
// https://spdx.org/spdx-license-list/matching-guidelines for the list.
|
||||
var interchangeableWords = []struct {
|
||||
interchangeable *regexp.Regexp
|
||||
substitute string
|
||||
}{
|
||||
{regexp.MustCompile("(?i)Acknowledgment"), "Acknowledgement"},
|
||||
{regexp.MustCompile("(?i)Analogue"), "Analog"},
|
||||
{regexp.MustCompile("(?i)Analyse"), "Analyze"},
|
||||
{regexp.MustCompile("(?i)Artefact"), "Artifact"},
|
||||
{regexp.MustCompile("(?i)Authorisation"), "Authorization"},
|
||||
{regexp.MustCompile("(?i)Authorised"), "Authorized"},
|
||||
{regexp.MustCompile("(?i)Calibre"), "Caliber"},
|
||||
{regexp.MustCompile("(?i)Cancelled"), "Canceled"},
|
||||
{regexp.MustCompile("(?i)Capitalisations"), "Capitalizations"},
|
||||
{regexp.MustCompile("(?i)Catalogue"), "Catalog"},
|
||||
{regexp.MustCompile("(?i)Categorise"), "Categorize"},
|
||||
{regexp.MustCompile("(?i)Centre"), "Center"},
|
||||
{regexp.MustCompile("(?i)Emphasised"), "Emphasized"},
|
||||
{regexp.MustCompile("(?i)Favour"), "Favor"},
|
||||
{regexp.MustCompile("(?i)Favourite"), "Favorite"},
|
||||
{regexp.MustCompile("(?i)Fulfil"), "Fulfill"},
|
||||
{regexp.MustCompile("(?i)Fulfilment"), "Fulfillment"},
|
||||
{regexp.MustCompile("(?i)Initialise"), "Initialize"},
|
||||
{regexp.MustCompile("(?i)Judgment"), "Judgement"},
|
||||
{regexp.MustCompile("(?i)Labelling"), "Labeling"},
|
||||
{regexp.MustCompile("(?i)Labour"), "Labor"},
|
||||
{regexp.MustCompile("(?i)Licence"), "License"},
|
||||
{regexp.MustCompile("(?i)Maximise"), "Maximize"},
|
||||
{regexp.MustCompile("(?i)Modelled"), "Modeled"},
|
||||
{regexp.MustCompile("(?i)Modelling"), "Modeling"},
|
||||
{regexp.MustCompile("(?i)Offence"), "Offense"},
|
||||
{regexp.MustCompile("(?i)Optimise"), "Optimize"},
|
||||
{regexp.MustCompile("(?i)Organisation"), "Organization"},
|
||||
{regexp.MustCompile("(?i)Organise"), "Organize"},
|
||||
{regexp.MustCompile("(?i)Practise"), "Practice"},
|
||||
{regexp.MustCompile("(?i)Programme"), "Program"},
|
||||
{regexp.MustCompile("(?i)Realise"), "Realize"},
|
||||
{regexp.MustCompile("(?i)Recognise"), "Recognize"},
|
||||
{regexp.MustCompile("(?i)Signalling"), "Signaling"},
|
||||
{regexp.MustCompile("(?i)Sub[- ]license"), "Sublicense"},
|
||||
{regexp.MustCompile("(?i)Utilisation"), "Utilization"},
|
||||
{regexp.MustCompile("(?i)Whilst"), "While"},
|
||||
{regexp.MustCompile("(?i)Wilful"), "Wilfull"},
|
||||
{regexp.MustCompile("(?i)Non-commercial"), "Noncommercial"},
|
||||
{regexp.MustCompile("(?i)Per cent"), "Percent"},
|
||||
}
|
||||
|
||||
// NormalizeEquivalentWords normalizes equivalent words that are interchangeable.
|
||||
func NormalizeEquivalentWords(s string) string {
|
||||
for _, iw := range interchangeableWords {
|
||||
s = iw.interchangeable.ReplaceAllString(s, iw.substitute)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
@ -1,68 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package licenseclassifier
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
// LicenseDirectory is the directory where the prototype licenses are kept.
|
||||
LicenseDirectory = "src/github.com/google/licenseclassifier/licenses"
|
||||
// LicenseArchive is the name of the archive containing preprocessed
|
||||
// license texts.
|
||||
LicenseArchive = "licenses.db"
|
||||
// ForbiddenLicenseArchive is the name of the archive containing preprocessed
|
||||
// forbidden license texts only.
|
||||
ForbiddenLicenseArchive = "forbidden_licenses.db"
|
||||
)
|
||||
|
||||
// lcRoot computes the location of the licenses data in the licenseclassifier source tree based on the location of this file.
|
||||
func lcRoot() (string, error) {
|
||||
_, filename, _, ok := runtime.Caller(0)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("unable to compute path of licenseclassifier source")
|
||||
}
|
||||
// this file must be in the root of the package, or the relative paths will be wrong.
|
||||
return filepath.Join(filepath.Dir(filename), "licenses"), nil
|
||||
}
|
||||
|
||||
// ReadLicenseFile locates and reads the license archive file. Absolute paths are used unmodified. Relative paths are expected to be in the licenses directory of the licenseclassifier package.
|
||||
func ReadLicenseFile(filename string) ([]byte, error) {
|
||||
if strings.HasPrefix(filename, "/") {
|
||||
return ioutil.ReadFile(filename)
|
||||
}
|
||||
|
||||
root, err := lcRoot()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error locating licenses directory: %v", err)
|
||||
}
|
||||
return ioutil.ReadFile(filepath.Join(root, filename))
|
||||
}
|
||||
|
||||
// ReadLicenseDir reads directory containing the license files.
|
||||
func ReadLicenseDir() ([]os.FileInfo, error) {
|
||||
root, err := lcRoot()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error locating licenses directory: %v", err)
|
||||
}
|
||||
|
||||
return ioutil.ReadDir(root)
|
||||
}
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package licenseclassifier
|
||||
|
||||
import "regexp"
|
||||
|
||||
var (
|
||||
reCCBYNC = regexp.MustCompile(`(?i).*\bAttribution NonCommercial\b.*`)
|
||||
reCCBYNCND = regexp.MustCompile(`(?i).*\bAttribution NonCommercial NoDerivs\b.*`)
|
||||
reCCBYNCSA = regexp.MustCompile(`(?i).*\bAttribution NonCommercial ShareAlike\b.*`)
|
||||
|
||||
// forbiddenRegexps are regular expressions we expect to find in
|
||||
// forbidden licenses. If we think we have a forbidden license but
|
||||
// don't find the equivalent phrase, then it's probably just a
|
||||
// misclassification.
|
||||
forbiddenRegexps = map[string]*regexp.Regexp{
|
||||
AGPL10: regexp.MustCompile(`(?i).*\bAFFERO GENERAL PUBLIC LICENSE\b.*`),
|
||||
AGPL30: regexp.MustCompile(`(?i).*\bGNU AFFERO GENERAL PUBLIC LICENSE\b.*`),
|
||||
CCBYNC10: reCCBYNC,
|
||||
CCBYNC20: reCCBYNC,
|
||||
CCBYNC25: reCCBYNC,
|
||||
CCBYNC30: reCCBYNC,
|
||||
CCBYNC40: reCCBYNC,
|
||||
CCBYNCND10: regexp.MustCompile(`(?i).*\bAttribution NoDerivs NonCommercial\b.*`),
|
||||
CCBYNCND20: reCCBYNCND,
|
||||
CCBYNCND25: reCCBYNCND,
|
||||
CCBYNCND30: reCCBYNCND,
|
||||
CCBYNCND40: regexp.MustCompile(`(?i).*\bAttribution NonCommercial NoDerivatives\b.*`),
|
||||
CCBYNCSA10: reCCBYNCSA,
|
||||
CCBYNCSA20: reCCBYNCSA,
|
||||
CCBYNCSA25: reCCBYNCSA,
|
||||
CCBYNCSA30: reCCBYNCSA,
|
||||
CCBYNCSA40: reCCBYNCSA,
|
||||
WTFPL: regexp.MustCompile(`(?i).*\bDO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE\b.*`),
|
||||
}
|
||||
)
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
module github.com/google/licenseclassifier
|
||||
|
||||
go 1.11
|
||||
|
||||
require (
|
||||
github.com/google/go-cmp v0.2.0
|
||||
github.com/sergi/go-diff v1.0.0
|
||||
github.com/stretchr/testify v1.3.0 // indirect
|
||||
)
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package sets provides sets for storing collections of unique elements.
|
||||
package sets
|
||||
|
||||
// present is an empty struct used as the "value" in the map[int], since
|
||||
// empty structs consume zero bytes (unlike 1 unnecessary byte per bool).
|
||||
type present struct{}
|
||||
|
|
@ -1,228 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
package sets
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// StringSet stores a set of unique string elements.
|
||||
type StringSet struct {
|
||||
set map[string]present
|
||||
}
|
||||
|
||||
// NewStringSet creates a StringSet containing the supplied initial string elements.
|
||||
func NewStringSet(elements ...string) *StringSet {
|
||||
s := &StringSet{}
|
||||
s.set = make(map[string]present)
|
||||
s.Insert(elements...)
|
||||
return s
|
||||
}
|
||||
|
||||
// Copy returns a newly allocated copy of the supplied StringSet.
|
||||
func (s *StringSet) Copy() *StringSet {
|
||||
c := NewStringSet()
|
||||
if s != nil {
|
||||
for e := range s.set {
|
||||
c.set[e] = present{}
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// Insert zero or more string elements into the StringSet.
|
||||
// As expected for a Set, elements already present in the StringSet are
|
||||
// simply ignored.
|
||||
func (s *StringSet) Insert(elements ...string) {
|
||||
for _, e := range elements {
|
||||
s.set[e] = present{}
|
||||
}
|
||||
}
|
||||
|
||||
// Delete zero or more string elements from the StringSet.
|
||||
// Any elements not present in the StringSet are simply ignored.
|
||||
func (s *StringSet) Delete(elements ...string) {
|
||||
for _, e := range elements {
|
||||
delete(s.set, e)
|
||||
}
|
||||
}
|
||||
|
||||
// Intersect returns a new StringSet containing the intersection of the
|
||||
// receiver and argument StringSets. Returns an empty set if the argument is nil.
|
||||
func (s *StringSet) Intersect(other *StringSet) *StringSet {
|
||||
if other == nil {
|
||||
return NewStringSet()
|
||||
}
|
||||
|
||||
// Point a and b to the maps, setting a to the smaller of the two.
|
||||
a, b := s.set, other.set
|
||||
if len(b) < len(a) {
|
||||
a, b = b, a
|
||||
}
|
||||
|
||||
// Perform the intersection.
|
||||
intersect := NewStringSet()
|
||||
for e := range a {
|
||||
if _, ok := b[e]; ok {
|
||||
intersect.set[e] = present{}
|
||||
}
|
||||
}
|
||||
return intersect
|
||||
}
|
||||
|
||||
// Disjoint returns true if the intersection of the receiver and the argument
|
||||
// StringSets is the empty set. Returns true if the argument is nil or either
|
||||
// StringSet is the empty set.
|
||||
func (s *StringSet) Disjoint(other *StringSet) bool {
|
||||
if other == nil || len(other.set) == 0 || len(s.set) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// Point a and b to the maps, setting a to the smaller of the two.
|
||||
a, b := s.set, other.set
|
||||
if len(b) < len(a) {
|
||||
a, b = b, a
|
||||
}
|
||||
|
||||
// Check for non-empty intersection.
|
||||
for e := range a {
|
||||
if _, ok := b[e]; ok {
|
||||
return false // Early-exit because intersecting.
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Difference returns a new StringSet containing the elements in the receiver
|
||||
// that are not present in the argument StringSet. Returns a copy of the
|
||||
// receiver if the argument is nil.
|
||||
func (s *StringSet) Difference(other *StringSet) *StringSet {
|
||||
if other == nil {
|
||||
return s.Copy()
|
||||
}
|
||||
|
||||
// Insert only the elements in the receiver that are not present in the
|
||||
// argument StringSet.
|
||||
diff := NewStringSet()
|
||||
for e := range s.set {
|
||||
if _, ok := other.set[e]; !ok {
|
||||
diff.set[e] = present{}
|
||||
}
|
||||
}
|
||||
return diff
|
||||
}
|
||||
|
||||
// Unique returns a new StringSet containing the elements in the receiver
|
||||
// that are not present in the argument StringSet *and* the elements in the
|
||||
// argument StringSet that are not in the receiver (which is the union of two
|
||||
// disjoint sets). Returns a copy of the
|
||||
// receiver if the argument is nil.
|
||||
func (s *StringSet) Unique(other *StringSet) *StringSet {
|
||||
if other == nil {
|
||||
return s.Copy()
|
||||
}
|
||||
|
||||
sNotInOther := s.Difference(other)
|
||||
otherNotInS := other.Difference(s)
|
||||
|
||||
// Duplicate Union implementation here to avoid extra Copy, since both
|
||||
// sNotInOther and otherNotInS are already copies.
|
||||
unique := sNotInOther
|
||||
for e := range otherNotInS.set {
|
||||
unique.set[e] = present{}
|
||||
}
|
||||
return unique
|
||||
}
|
||||
|
||||
// Equal returns true if the receiver and the argument StringSet contain
|
||||
// exactly the same elements.
|
||||
func (s *StringSet) Equal(other *StringSet) bool {
|
||||
if s == nil || other == nil {
|
||||
return s == nil && other == nil
|
||||
}
|
||||
|
||||
// Two sets of different length cannot have the exact same unique elements.
|
||||
if len(s.set) != len(other.set) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Only one loop is needed. If the two sets are known to be of equal
|
||||
// length, then the two sets are equal only if exactly all of the elements
|
||||
// in the first set are found in the second.
|
||||
for e := range s.set {
|
||||
if _, ok := other.set[e]; !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Union returns a new StringSet containing the union of the receiver and
|
||||
// argument StringSets. Returns a copy of the receiver if the argument is nil.
|
||||
func (s *StringSet) Union(other *StringSet) *StringSet {
|
||||
union := s.Copy()
|
||||
if other != nil {
|
||||
for e := range other.set {
|
||||
union.set[e] = present{}
|
||||
}
|
||||
}
|
||||
return union
|
||||
}
|
||||
|
||||
// Contains returns true if element is in the StringSet.
|
||||
func (s *StringSet) Contains(element string) bool {
|
||||
_, in := s.set[element]
|
||||
return in
|
||||
}
|
||||
|
||||
// Len returns the number of unique elements in the StringSet.
|
||||
func (s *StringSet) Len() int {
|
||||
return len(s.set)
|
||||
}
|
||||
|
||||
// Empty returns true if the receiver is the empty set.
|
||||
func (s *StringSet) Empty() bool {
|
||||
return len(s.set) == 0
|
||||
}
|
||||
|
||||
// Elements returns a []string of the elements in the StringSet, in no
|
||||
// particular (or consistent) order.
|
||||
func (s *StringSet) Elements() []string {
|
||||
elements := []string{} // Return at least an empty slice rather than nil.
|
||||
for e := range s.set {
|
||||
elements = append(elements, e)
|
||||
}
|
||||
return elements
|
||||
}
|
||||
|
||||
// Sorted returns a sorted []string of the elements in the StringSet.
|
||||
func (s *StringSet) Sorted() []string {
|
||||
elements := s.Elements()
|
||||
sort.Strings(elements)
|
||||
return elements
|
||||
}
|
||||
|
||||
// String formats the StringSet elements as sorted strings, representing them
|
||||
// in "array initializer" syntax.
|
||||
func (s *StringSet) String() string {
|
||||
elements := s.Sorted()
|
||||
var quoted []string
|
||||
for _, e := range elements {
|
||||
quoted = append(quoted, fmt.Sprintf("%q", e))
|
||||
}
|
||||
return fmt.Sprintf("{%s}", strings.Join(quoted, ", "))
|
||||
}
|
||||
|
|
@ -1,394 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package licenseclassifier
|
||||
|
||||
// *** NOTE: Update this file when adding a new license. You need to:
|
||||
//
|
||||
// 1. Add the canonical name to the list, and
|
||||
// 2. Categorize the license.
|
||||
|
||||
import "github.com/google/licenseclassifier/internal/sets"
|
||||
|
||||
// Canonical names of the licenses.
|
||||
const (
|
||||
// The names come from the https://spdx.org/licenses website, and are
|
||||
// also the filenames of the licenses in licenseclassifier/licenses.
|
||||
AFL11 = "AFL-1.1"
|
||||
AFL12 = "AFL-1.2"
|
||||
AFL20 = "AFL-2.0"
|
||||
AFL21 = "AFL-2.1"
|
||||
AFL30 = "AFL-3.0"
|
||||
AGPL10 = "AGPL-1.0"
|
||||
AGPL30 = "AGPL-3.0"
|
||||
Apache10 = "Apache-1.0"
|
||||
Apache11 = "Apache-1.1"
|
||||
Apache20 = "Apache-2.0"
|
||||
APSL10 = "APSL-1.0"
|
||||
APSL11 = "APSL-1.1"
|
||||
APSL12 = "APSL-1.2"
|
||||
APSL20 = "APSL-2.0"
|
||||
Artistic10cl8 = "Artistic-1.0-cl8"
|
||||
Artistic10Perl = "Artistic-1.0-Perl"
|
||||
Artistic10 = "Artistic-1.0"
|
||||
Artistic20 = "Artistic-2.0"
|
||||
BCL = "BCL"
|
||||
Beerware = "Beerware"
|
||||
BSD2ClauseFreeBSD = "BSD-2-Clause-FreeBSD"
|
||||
BSD2ClauseNetBSD = "BSD-2-Clause-NetBSD"
|
||||
BSD2Clause = "BSD-2-Clause"
|
||||
BSD3ClauseAttribution = "BSD-3-Clause-Attribution"
|
||||
BSD3ClauseClear = "BSD-3-Clause-Clear"
|
||||
BSD3ClauseLBNL = "BSD-3-Clause-LBNL"
|
||||
BSD3Clause = "BSD-3-Clause"
|
||||
BSD4Clause = "BSD-4-Clause"
|
||||
BSD4ClauseUC = "BSD-4-Clause-UC"
|
||||
BSDProtection = "BSD-Protection"
|
||||
BSL10 = "BSL-1.0"
|
||||
CC010 = "CC0-1.0"
|
||||
CCBY10 = "CC-BY-1.0"
|
||||
CCBY20 = "CC-BY-2.0"
|
||||
CCBY25 = "CC-BY-2.5"
|
||||
CCBY30 = "CC-BY-3.0"
|
||||
CCBY40 = "CC-BY-4.0"
|
||||
CCBYNC10 = "CC-BY-NC-1.0"
|
||||
CCBYNC20 = "CC-BY-NC-2.0"
|
||||
CCBYNC25 = "CC-BY-NC-2.5"
|
||||
CCBYNC30 = "CC-BY-NC-3.0"
|
||||
CCBYNC40 = "CC-BY-NC-4.0"
|
||||
CCBYNCND10 = "CC-BY-NC-ND-1.0"
|
||||
CCBYNCND20 = "CC-BY-NC-ND-2.0"
|
||||
CCBYNCND25 = "CC-BY-NC-ND-2.5"
|
||||
CCBYNCND30 = "CC-BY-NC-ND-3.0"
|
||||
CCBYNCND40 = "CC-BY-NC-ND-4.0"
|
||||
CCBYNCSA10 = "CC-BY-NC-SA-1.0"
|
||||
CCBYNCSA20 = "CC-BY-NC-SA-2.0"
|
||||
CCBYNCSA25 = "CC-BY-NC-SA-2.5"
|
||||
CCBYNCSA30 = "CC-BY-NC-SA-3.0"
|
||||
CCBYNCSA40 = "CC-BY-NC-SA-4.0"
|
||||
CCBYND10 = "CC-BY-ND-1.0"
|
||||
CCBYND20 = "CC-BY-ND-2.0"
|
||||
CCBYND25 = "CC-BY-ND-2.5"
|
||||
CCBYND30 = "CC-BY-ND-3.0"
|
||||
CCBYND40 = "CC-BY-ND-4.0"
|
||||
CCBYSA10 = "CC-BY-SA-1.0"
|
||||
CCBYSA20 = "CC-BY-SA-2.0"
|
||||
CCBYSA25 = "CC-BY-SA-2.5"
|
||||
CCBYSA30 = "CC-BY-SA-3.0"
|
||||
CCBYSA40 = "CC-BY-SA-4.0"
|
||||
CDDL10 = "CDDL-1.0"
|
||||
CDDL11 = "CDDL-1.1"
|
||||
CommonsClause = "Commons-Clause"
|
||||
CPAL10 = "CPAL-1.0"
|
||||
CPL10 = "CPL-1.0"
|
||||
eGenix = "eGenix"
|
||||
EPL10 = "EPL-1.0"
|
||||
EPL20 = "EPL-2.0"
|
||||
EUPL10 = "EUPL-1.0"
|
||||
EUPL11 = "EUPL-1.1"
|
||||
Facebook2Clause = "Facebook-2-Clause"
|
||||
Facebook3Clause = "Facebook-3-Clause"
|
||||
FacebookExamples = "Facebook-Examples"
|
||||
FreeImage = "FreeImage"
|
||||
FTL = "FTL"
|
||||
GPL10 = "GPL-1.0"
|
||||
GPL20 = "GPL-2.0"
|
||||
GPL20withautoconfexception = "GPL-2.0-with-autoconf-exception"
|
||||
GPL20withbisonexception = "GPL-2.0-with-bison-exception"
|
||||
GPL20withclasspathexception = "GPL-2.0-with-classpath-exception"
|
||||
GPL20withfontexception = "GPL-2.0-with-font-exception"
|
||||
GPL20withGCCexception = "GPL-2.0-with-GCC-exception"
|
||||
GPL30 = "GPL-3.0"
|
||||
GPL30withautoconfexception = "GPL-3.0-with-autoconf-exception"
|
||||
GPL30withGCCexception = "GPL-3.0-with-GCC-exception"
|
||||
GUSTFont = "GUST-Font-License"
|
||||
ImageMagick = "ImageMagick"
|
||||
IPL10 = "IPL-1.0"
|
||||
ISC = "ISC"
|
||||
LGPL20 = "LGPL-2.0"
|
||||
LGPL21 = "LGPL-2.1"
|
||||
LGPL30 = "LGPL-3.0"
|
||||
LGPLLR = "LGPLLR"
|
||||
Libpng = "Libpng"
|
||||
Lil10 = "Lil-1.0"
|
||||
LinuxOpenIB = "Linux-OpenIB"
|
||||
LPL102 = "LPL-1.02"
|
||||
LPL10 = "LPL-1.0"
|
||||
LPPL13c = "LPPL-1.3c"
|
||||
MIT = "MIT"
|
||||
MPL10 = "MPL-1.0"
|
||||
MPL11 = "MPL-1.1"
|
||||
MPL20 = "MPL-2.0"
|
||||
MSPL = "MS-PL"
|
||||
NCSA = "NCSA"
|
||||
NPL10 = "NPL-1.0"
|
||||
NPL11 = "NPL-1.1"
|
||||
OFL11 = "OFL-1.1"
|
||||
OpenSSL = "OpenSSL"
|
||||
OpenVision = "OpenVision"
|
||||
OSL10 = "OSL-1.0"
|
||||
OSL11 = "OSL-1.1"
|
||||
OSL20 = "OSL-2.0"
|
||||
OSL21 = "OSL-2.1"
|
||||
OSL30 = "OSL-3.0"
|
||||
PHP301 = "PHP-3.01"
|
||||
PHP30 = "PHP-3.0"
|
||||
PIL = "PIL"
|
||||
PostgreSQL = "PostgreSQL"
|
||||
Python20complete = "Python-2.0-complete"
|
||||
Python20 = "Python-2.0"
|
||||
QPL10 = "QPL-1.0"
|
||||
Ruby = "Ruby"
|
||||
SGIB10 = "SGI-B-1.0"
|
||||
SGIB11 = "SGI-B-1.1"
|
||||
SGIB20 = "SGI-B-2.0"
|
||||
SISSL12 = "SISSL-1.2"
|
||||
SISSL = "SISSL"
|
||||
Sleepycat = "Sleepycat"
|
||||
UnicodeTOU = "Unicode-TOU"
|
||||
UnicodeDFS2015 = "Unicode-DFS-2015"
|
||||
UnicodeDFS2016 = "Unicode-DFS-2016"
|
||||
Unlicense = "Unlicense"
|
||||
UPL10 = "UPL-1.0"
|
||||
W3C19980720 = "W3C-19980720"
|
||||
W3C20150513 = "W3C-20150513"
|
||||
W3C = "W3C"
|
||||
WTFPL = "WTFPL"
|
||||
X11 = "X11"
|
||||
Xnet = "Xnet"
|
||||
Zend20 = "Zend-2.0"
|
||||
ZeroBSD = "0BSD"
|
||||
ZlibAcknowledgement = "zlib-acknowledgement"
|
||||
Zlib = "Zlib"
|
||||
ZPL11 = "ZPL-1.1"
|
||||
ZPL20 = "ZPL-2.0"
|
||||
ZPL21 = "ZPL-2.1"
|
||||
)
|
||||
|
||||
var (
|
||||
// Licenses Categorized by Type
|
||||
|
||||
// restricted - Licenses in this category require mandatory source
|
||||
// distribution if we ships a product that includes third-party code
|
||||
// protected by such a license.
|
||||
restrictedType = sets.NewStringSet(
|
||||
BCL,
|
||||
CCBYND10,
|
||||
CCBYND20,
|
||||
CCBYND25,
|
||||
CCBYND30,
|
||||
CCBYND40,
|
||||
CCBYSA10,
|
||||
CCBYSA20,
|
||||
CCBYSA25,
|
||||
CCBYSA30,
|
||||
CCBYSA40,
|
||||
GPL10,
|
||||
GPL20,
|
||||
GPL20withautoconfexception,
|
||||
GPL20withbisonexception,
|
||||
GPL20withclasspathexception,
|
||||
GPL20withfontexception,
|
||||
GPL20withGCCexception,
|
||||
GPL30,
|
||||
GPL30withautoconfexception,
|
||||
GPL30withGCCexception,
|
||||
LGPL20,
|
||||
LGPL21,
|
||||
LGPL30,
|
||||
NPL10,
|
||||
NPL11,
|
||||
OSL10,
|
||||
OSL11,
|
||||
OSL20,
|
||||
OSL21,
|
||||
OSL30,
|
||||
QPL10,
|
||||
Sleepycat,
|
||||
)
|
||||
|
||||
// reciprocal - These licenses allow usage of software made available
|
||||
// under such licenses freely in *unmodified* form. If the third-party
|
||||
// source code is modified in any way these modifications to the
|
||||
// original third-party source code must be made available.
|
||||
reciprocalType = sets.NewStringSet(
|
||||
APSL10,
|
||||
APSL11,
|
||||
APSL12,
|
||||
APSL20,
|
||||
CDDL10,
|
||||
CDDL11,
|
||||
CPL10,
|
||||
EPL10,
|
||||
EPL20,
|
||||
FreeImage,
|
||||
IPL10,
|
||||
MPL10,
|
||||
MPL11,
|
||||
MPL20,
|
||||
Ruby,
|
||||
)
|
||||
|
||||
// notice - These licenses contain few restrictions, allowing original
|
||||
// or modified third-party software to be shipped in any product
|
||||
// without endangering or encumbering our source code. All of the
|
||||
// licenses in this category do, however, have an "original Copyright
|
||||
// notice" or "advertising clause", wherein any external distributions
|
||||
// must include the notice or clause specified in the license.
|
||||
noticeType = sets.NewStringSet(
|
||||
AFL11,
|
||||
AFL12,
|
||||
AFL20,
|
||||
AFL21,
|
||||
AFL30,
|
||||
Apache10,
|
||||
Apache11,
|
||||
Apache20,
|
||||
Artistic10cl8,
|
||||
Artistic10Perl,
|
||||
Artistic10,
|
||||
Artistic20,
|
||||
BSL10,
|
||||
BSD2ClauseFreeBSD,
|
||||
BSD2ClauseNetBSD,
|
||||
BSD2Clause,
|
||||
BSD3ClauseAttribution,
|
||||
BSD3ClauseClear,
|
||||
BSD3ClauseLBNL,
|
||||
BSD3Clause,
|
||||
BSD4Clause,
|
||||
BSD4ClauseUC,
|
||||
BSDProtection,
|
||||
CCBY10,
|
||||
CCBY20,
|
||||
CCBY25,
|
||||
CCBY30,
|
||||
CCBY40,
|
||||
FTL,
|
||||
ISC,
|
||||
ImageMagick,
|
||||
Libpng,
|
||||
Lil10,
|
||||
LinuxOpenIB,
|
||||
LPL102,
|
||||
LPL10,
|
||||
MSPL,
|
||||
MIT,
|
||||
NCSA,
|
||||
OpenSSL,
|
||||
PHP301,
|
||||
PHP30,
|
||||
PIL,
|
||||
Python20,
|
||||
Python20complete,
|
||||
PostgreSQL,
|
||||
SGIB10,
|
||||
SGIB11,
|
||||
SGIB20,
|
||||
UnicodeDFS2015,
|
||||
UnicodeDFS2016,
|
||||
UnicodeTOU,
|
||||
UPL10,
|
||||
W3C19980720,
|
||||
W3C20150513,
|
||||
W3C,
|
||||
X11,
|
||||
Xnet,
|
||||
Zend20,
|
||||
ZlibAcknowledgement,
|
||||
Zlib,
|
||||
ZPL11,
|
||||
ZPL20,
|
||||
ZPL21,
|
||||
)
|
||||
|
||||
// permissive - These licenses can be used in (relatively rare) cases
|
||||
// where third-party software is under a license (not "Public Domain"
|
||||
// or "free for any use" like 'unencumbered') that is even more lenient
|
||||
// than a 'notice' license. Use the 'permissive' license type when even
|
||||
// a copyright notice is not required for license compliance.
|
||||
permissiveType = sets.NewStringSet()
|
||||
|
||||
// unencumbered - Licenses that basically declare that the code is "free for any use".
|
||||
unencumberedType = sets.NewStringSet(
|
||||
CC010,
|
||||
Unlicense,
|
||||
ZeroBSD,
|
||||
)
|
||||
|
||||
// byexceptiononly - Licenses that are incompatible with all (or most)
|
||||
// uses in combination with our source code. Commercial third-party
|
||||
// packages that are purchased and licensed only for a specific use
|
||||
// fall into this category.
|
||||
byExceptionOnlyType = sets.NewStringSet(
|
||||
Beerware,
|
||||
OFL11,
|
||||
OpenVision,
|
||||
)
|
||||
|
||||
// forbidden - Licenses that are forbidden to be used.
|
||||
forbiddenType = sets.NewStringSet(
|
||||
AGPL10,
|
||||
AGPL30,
|
||||
CCBYNC10,
|
||||
CCBYNC20,
|
||||
CCBYNC25,
|
||||
CCBYNC30,
|
||||
CCBYNC40,
|
||||
CCBYNCND10,
|
||||
CCBYNCND20,
|
||||
CCBYNCND25,
|
||||
CCBYNCND30,
|
||||
CCBYNCND40,
|
||||
CCBYNCSA10,
|
||||
CCBYNCSA20,
|
||||
CCBYNCSA25,
|
||||
CCBYNCSA30,
|
||||
CCBYNCSA40,
|
||||
CommonsClause,
|
||||
Facebook2Clause,
|
||||
Facebook3Clause,
|
||||
FacebookExamples,
|
||||
WTFPL,
|
||||
)
|
||||
|
||||
// LicenseTypes is a set of the types of licenses Google recognizes.
|
||||
LicenseTypes = sets.NewStringSet(
|
||||
"restricted",
|
||||
"reciprocal",
|
||||
"notice",
|
||||
"permissive",
|
||||
"unencumbered",
|
||||
"by_exception_only",
|
||||
)
|
||||
)
|
||||
|
||||
// LicenseType returns the type the license has.
|
||||
func LicenseType(name string) string {
|
||||
switch {
|
||||
case restrictedType.Contains(name):
|
||||
return "restricted"
|
||||
case reciprocalType.Contains(name):
|
||||
return "reciprocal"
|
||||
case noticeType.Contains(name):
|
||||
return "notice"
|
||||
case permissiveType.Contains(name):
|
||||
return "permissive"
|
||||
case unencumberedType.Contains(name):
|
||||
return "unencumbered"
|
||||
case forbiddenType.Contains(name):
|
||||
return "FORBIDDEN"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
# How to contribute
|
||||
|
||||
We'd love to accept your patches and contributions to this project. There are
|
||||
just a few small guidelines you need to follow.
|
||||
|
||||
## Contributor License Agreement
|
||||
|
||||
Contributions to this project must be accompanied by a Contributor License
|
||||
Agreement. You (or your employer) retain the copyright to your contribution,
|
||||
this simply gives us permission to use and redistribute your contributions as
|
||||
part of the project. Head over to <https://cla.developers.google.com/> to see
|
||||
your current agreements on file or to sign a new one.
|
||||
|
||||
You generally only need to submit a CLA once, so if you've already submitted
|
||||
one (even if it was for a different project), you probably don't need to do it
|
||||
again.
|
||||
|
||||
## Code reviews
|
||||
|
||||
All submissions, including submissions by project members, require review. We
|
||||
use GitHub pull requests for this purpose. Consult [GitHub Help] for more
|
||||
information on using pull requests.
|
||||
|
||||
[GitHub Help]: https://help.github.com/articles/about-pull-requests/
|
||||
|
|
@ -1,202 +0,0 @@
|
|||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -1,65 +0,0 @@
|
|||
# StringClassifier
|
||||
|
||||
StringClassifier is a library to classify an unknown text against a set of known
|
||||
texts. The classifier uses the [Levenshtein Distance] algorithm to determine
|
||||
which of the known texts most closely matches the unknown text. The Levenshtein
|
||||
Distance is normalized into a "confidence percentage" between 1 and 0, where 1.0
|
||||
indicates an exact match and 0.0 indicates a complete mismatch.
|
||||
|
||||
[Levenshtein Distance]: https://en.wikipedia.org/wiki/Levenshtein_distance
|
||||
|
||||
## Types of matching
|
||||
|
||||
There are two kinds of matching algorithms the string classifier can perform:
|
||||
|
||||
1. [Nearest matching](#nearest), and
|
||||
2. [Multiple matching](#multiple).
|
||||
|
||||
### Normalization
|
||||
|
||||
To get the best match, normalizing functions can be applied to the texts. For
|
||||
example, flattening whitespaces removes a lot of inconsequential formatting
|
||||
differences that would otherwise lower the matching confidence percentage.
|
||||
|
||||
```go
|
||||
sc := stringclassifier.New(stringclassifier.FlattenWhitespace, strings.ToLower)
|
||||
```
|
||||
|
||||
The normalizating functions are run on all the known texts that are added to the
|
||||
classifier. They're also run on the unknown text before classification.
|
||||
|
||||
### Nearest matching {#nearest}
|
||||
|
||||
A nearest match returns the name of the known text that most closely matches the
|
||||
full unknown text. This is most useful when the unknown text doesn't have
|
||||
extraneous text around it.
|
||||
|
||||
Example:
|
||||
|
||||
```go
|
||||
func IdentifyText(sc *stringclassifier.Classifier, name, unknown string) {
|
||||
m := sc.NearestMatch(unknown)
|
||||
log.Printf("The nearest match to %q is %q (confidence: %v)", name, m.Name, m.Confidence)
|
||||
}
|
||||
```
|
||||
|
||||
## Multiple matching {#multiple}
|
||||
|
||||
Multiple matching identifies all of the known texts which may exist in the
|
||||
unknown text. It can also detect a known text in an unknown text even if there's
|
||||
extraneous text around the unknown text. As with nearest matching, a confidence
|
||||
percentage for each match is given.
|
||||
|
||||
Example:
|
||||
|
||||
```go
|
||||
log.Printf("The text %q contains:", name)
|
||||
for _, m := range sc.MultipleMatch(unknown, false) {
|
||||
log.Printf(" %q (conf: %v, offset: %v)", m.Name, m.Confidence, m.Offset)
|
||||
}
|
||||
```
|
||||
|
||||
## Disclaimer
|
||||
|
||||
This is not an official Google product (experimental or otherwise), it is just
|
||||
code that happens to be owned by Google.
|
||||
|
|
@ -1,560 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package stringclassifier finds the nearest match between a string and a set of known values. It
|
||||
// uses the Levenshtein Distance (LD) algorithm to determine this. A match with a large LD is less
|
||||
// likely to be correct than one with a small LD. A confidence percentage is returned, which
|
||||
// indicates how confident the algorithm is that the match is correct. The higher the percentage,
|
||||
// the greater the confidence that the match is correct.
|
||||
//
|
||||
// Example Usage:
|
||||
//
|
||||
// type Text struct {
|
||||
// Name string
|
||||
// Text string
|
||||
// }
|
||||
//
|
||||
// func NewClassifier(knownTexts []Text) (*stringclassifier.Classifier, error) {
|
||||
// sc := stringclassifier.New(stringclassifier.FlattenWhitespace)
|
||||
// for _, known := range knownTexts {
|
||||
// if err := sc.AddValue(known.Name, known.Text); err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
// }
|
||||
// return sc, nil
|
||||
// }
|
||||
//
|
||||
// func IdentifyTexts(sc *stringclassifier.Classifier, unknownTexts []*Text) {
|
||||
// for _, unknown := range unknownTexts {
|
||||
// m := sc.NearestMatch(unknown.Text)
|
||||
// log.Printf("The nearest match to %q is %q (confidence: %v)",
|
||||
// unknown.Name, m.Name, m.Confidence)
|
||||
// }
|
||||
// }
|
||||
package stringclassifier
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"math"
|
||||
"regexp"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/google/licenseclassifier/stringclassifier/internal/pq"
|
||||
"github.com/google/licenseclassifier/stringclassifier/searchset"
|
||||
"github.com/sergi/go-diff/diffmatchpatch"
|
||||
)
|
||||
|
||||
// The diff/match/patch algorithm.
|
||||
var dmp = diffmatchpatch.New()
|
||||
|
||||
const (
|
||||
// DefaultConfidenceThreshold is the minimum ratio threshold between
|
||||
// the matching range and the full source range that we're willing to
|
||||
// accept in order to say that the matching range will produce a
|
||||
// sufficiently good edit distance. I.e., if the matching range is
|
||||
// below this threshold we won't run the Levenshtein Distance algorithm
|
||||
// on it.
|
||||
DefaultConfidenceThreshold float64 = 0.80
|
||||
|
||||
defaultMinDiffRatio float64 = 0.75
|
||||
)
|
||||
|
||||
// A Classifier matches a string to a set of known values.
|
||||
type Classifier struct {
|
||||
muValues sync.RWMutex
|
||||
values map[string]*knownValue
|
||||
normalizers []NormalizeFunc
|
||||
threshold float64
|
||||
|
||||
// MinDiffRatio defines the minimum ratio of the length difference
|
||||
// allowed to consider a known value a possible match. This is used as
|
||||
// a performance optimization to eliminate values that are unlikely to
|
||||
// be a match.
|
||||
//
|
||||
// For example, a value of 0.75 means that the shorter string must be
|
||||
// at least 75% the length of the longer string to consider it a
|
||||
// possible match.
|
||||
//
|
||||
// Setting this to 1.0 will require that strings are identical length.
|
||||
// Setting this to 0 will consider all known values as possible
|
||||
// matches.
|
||||
MinDiffRatio float64
|
||||
}
|
||||
|
||||
// NormalizeFunc is a function that is used to normalize a string prior to comparison.
|
||||
type NormalizeFunc func(string) string
|
||||
|
||||
// New creates a new Classifier with the provided NormalizeFuncs. Each
|
||||
// NormalizeFunc is applied in order to a string before comparison.
|
||||
func New(threshold float64, funcs ...NormalizeFunc) *Classifier {
|
||||
return &Classifier{
|
||||
values: make(map[string]*knownValue),
|
||||
normalizers: append([]NormalizeFunc(nil), funcs...),
|
||||
threshold: threshold,
|
||||
MinDiffRatio: defaultMinDiffRatio,
|
||||
}
|
||||
}
|
||||
|
||||
// knownValue identifies a value in the corpus to match against.
|
||||
type knownValue struct {
|
||||
key string
|
||||
normalizedValue string
|
||||
reValue *regexp.Regexp
|
||||
set *searchset.SearchSet
|
||||
}
|
||||
|
||||
// AddValue adds a known value to be matched against. If a value already exists
|
||||
// for key, an error is returned.
|
||||
func (c *Classifier) AddValue(key, value string) error {
|
||||
c.muValues.Lock()
|
||||
defer c.muValues.Unlock()
|
||||
if _, ok := c.values[key]; ok {
|
||||
return fmt.Errorf("value already registered with key %q", key)
|
||||
}
|
||||
norm := c.normalize(value)
|
||||
c.values[key] = &knownValue{
|
||||
key: key,
|
||||
normalizedValue: norm,
|
||||
reValue: regexp.MustCompile(norm),
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddPrecomputedValue adds a known value to be matched against. The value has
|
||||
// already been normalized and the SearchSet object deserialized, so no
|
||||
// processing is necessary.
|
||||
func (c *Classifier) AddPrecomputedValue(key, value string, set *searchset.SearchSet) error {
|
||||
c.muValues.Lock()
|
||||
defer c.muValues.Unlock()
|
||||
if _, ok := c.values[key]; ok {
|
||||
return fmt.Errorf("value already registered with key %q", key)
|
||||
}
|
||||
set.GenerateNodeList()
|
||||
c.values[key] = &knownValue{
|
||||
key: key,
|
||||
normalizedValue: value,
|
||||
reValue: regexp.MustCompile(value),
|
||||
set: set,
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalize a string by applying each of the registered NormalizeFuncs.
|
||||
func (c *Classifier) normalize(s string) string {
|
||||
for _, fn := range c.normalizers {
|
||||
s = fn(s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// Match identifies the result of matching a string against a knownValue.
|
||||
type Match struct {
|
||||
Name string // Name of knownValue that was matched
|
||||
Confidence float64 // Confidence percentage
|
||||
Offset int // The offset into the unknown string the match was made
|
||||
Extent int // The length from the offset into the unknown string
|
||||
}
|
||||
|
||||
// Matches is a list of Match-es. This is here mainly so that the list can be
|
||||
// sorted.
|
||||
type Matches []*Match
|
||||
|
||||
func (m Matches) Len() int { return len(m) }
|
||||
func (m Matches) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
|
||||
func (m Matches) Less(i, j int) bool {
|
||||
if math.Abs(m[j].Confidence-m[i].Confidence) < math.SmallestNonzeroFloat64 {
|
||||
if m[i].Name == m[j].Name {
|
||||
if m[i].Offset > m[j].Offset {
|
||||
return false
|
||||
}
|
||||
if m[i].Offset == m[j].Offset {
|
||||
return m[i].Extent > m[j].Extent
|
||||
}
|
||||
return true
|
||||
}
|
||||
return m[i].Name < m[j].Name
|
||||
}
|
||||
return m[i].Confidence > m[j].Confidence
|
||||
}
|
||||
|
||||
// Names returns an unsorted slice of the names of the matched licenses.
|
||||
func (m Matches) Names() []string {
|
||||
var names []string
|
||||
for _, n := range m {
|
||||
names = append(names, n.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// uniquify goes through the matches and removes any that are contained within
|
||||
// one with a higher confidence. This assumes that Matches is sorted.
|
||||
func (m Matches) uniquify() Matches {
|
||||
type matchedRange struct {
|
||||
offset, extent int
|
||||
}
|
||||
|
||||
var matched []matchedRange
|
||||
var matches Matches
|
||||
OUTER:
|
||||
for _, match := range m {
|
||||
for _, mr := range matched {
|
||||
if match.Offset >= mr.offset && match.Offset <= mr.offset+mr.extent {
|
||||
continue OUTER
|
||||
}
|
||||
}
|
||||
matched = append(matched, matchedRange{match.Offset, match.Extent})
|
||||
matches = append(matches, match)
|
||||
}
|
||||
|
||||
return matches
|
||||
}
|
||||
|
||||
// NearestMatch returns the name of the known value that most closely matches
|
||||
// the unknown string and a confidence percentage is returned indicating how
|
||||
// confident the classifier is in the result. A percentage of "1.0" indicates
|
||||
// an exact match, while a percentage of "0.0" indicates a complete mismatch.
|
||||
//
|
||||
// If the string is equidistant from multiple known values, it is undefined
|
||||
// which will be returned.
|
||||
func (c *Classifier) NearestMatch(s string) *Match {
|
||||
pq := c.nearestMatch(s)
|
||||
if pq.Len() == 0 {
|
||||
return &Match{}
|
||||
}
|
||||
return pq.Pop().(*Match)
|
||||
}
|
||||
|
||||
// MultipleMatch tries to determine which known strings are found within an
|
||||
// unknown string. This differs from "NearestMatch" in that it looks only at
|
||||
// those areas within the unknown string that are likely to match. A list of
|
||||
// potential matches are returned. It's up to the caller to determine which
|
||||
// ones are acceptable.
|
||||
func (c *Classifier) MultipleMatch(s string) (matches Matches) {
|
||||
pq := c.multipleMatch(s)
|
||||
if pq == nil {
|
||||
return matches
|
||||
}
|
||||
|
||||
// A map to remove duplicate entries.
|
||||
m := make(map[Match]bool)
|
||||
|
||||
for pq.Len() != 0 {
|
||||
v := pq.Pop().(*Match)
|
||||
if _, ok := m[*v]; !ok {
|
||||
m[*v] = true
|
||||
matches = append(matches, v)
|
||||
}
|
||||
}
|
||||
|
||||
sort.Sort(matches)
|
||||
return matches.uniquify()
|
||||
}
|
||||
|
||||
// possibleMatch identifies a known value and it's diffRatio to a given string.
|
||||
type possibleMatch struct {
|
||||
value *knownValue
|
||||
diffRatio float64
|
||||
}
|
||||
|
||||
// likelyMatches is a slice of possibleMatches that can be sorted by their
|
||||
// diffRatio to a given string, such that the most likely matches (based on
|
||||
// length) are at the beginning.
|
||||
type likelyMatches []possibleMatch
|
||||
|
||||
func (m likelyMatches) Len() int { return len(m) }
|
||||
func (m likelyMatches) Less(i, j int) bool { return m[i].diffRatio > m[j].diffRatio }
|
||||
func (m likelyMatches) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
|
||||
|
||||
// nearestMatch returns a Queue of values that the unknown string may be. The
|
||||
// values are compared via their Levenshtein Distance and ranked with the
|
||||
// nearest match at the beginning.
|
||||
func (c *Classifier) nearestMatch(unknown string) *pq.Queue {
|
||||
var mu sync.Mutex // Protect the priority queue.
|
||||
pq := pq.NewQueue(func(x, y interface{}) bool {
|
||||
return x.(*Match).Confidence > y.(*Match).Confidence
|
||||
}, nil)
|
||||
|
||||
unknown = c.normalize(unknown)
|
||||
if len(unknown) == 0 {
|
||||
return pq
|
||||
}
|
||||
|
||||
c.muValues.RLock()
|
||||
var likely likelyMatches
|
||||
for _, v := range c.values {
|
||||
dr := diffRatio(unknown, v.normalizedValue)
|
||||
if dr < c.MinDiffRatio {
|
||||
continue
|
||||
}
|
||||
if unknown == v.normalizedValue {
|
||||
// We found an exact match.
|
||||
pq.Push(&Match{Name: v.key, Confidence: 1.0, Offset: 0, Extent: len(unknown)})
|
||||
c.muValues.RUnlock()
|
||||
return pq
|
||||
}
|
||||
likely = append(likely, possibleMatch{value: v, diffRatio: dr})
|
||||
}
|
||||
c.muValues.RUnlock()
|
||||
sort.Sort(likely)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
classifyString := func(name, unknown, known string) {
|
||||
defer wg.Done()
|
||||
|
||||
diffs := dmp.DiffMain(unknown, known, true)
|
||||
distance := dmp.DiffLevenshtein(diffs)
|
||||
confidence := confidencePercentage(len(unknown), len(known), distance)
|
||||
if confidence > 0.0 {
|
||||
mu.Lock()
|
||||
pq.Push(&Match{Name: name, Confidence: confidence, Offset: 0, Extent: len(unknown)})
|
||||
mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
wg.Add(len(likely))
|
||||
for _, known := range likely {
|
||||
go classifyString(known.value.key, unknown, known.value.normalizedValue)
|
||||
}
|
||||
wg.Wait()
|
||||
return pq
|
||||
}
|
||||
|
||||
// matcher finds all potential matches of "known" in "unknown". The results are
|
||||
// placed in "queue".
|
||||
type matcher struct {
|
||||
unknown *searchset.SearchSet
|
||||
normUnknown string
|
||||
threshold float64
|
||||
|
||||
mu sync.Mutex
|
||||
queue *pq.Queue
|
||||
}
|
||||
|
||||
// newMatcher creates a "matcher" object.
|
||||
func newMatcher(unknown string, threshold float64) *matcher {
|
||||
return &matcher{
|
||||
unknown: searchset.New(unknown, searchset.DefaultGranularity),
|
||||
normUnknown: unknown,
|
||||
threshold: threshold,
|
||||
queue: pq.NewQueue(func(x, y interface{}) bool {
|
||||
return x.(*Match).Confidence > y.(*Match).Confidence
|
||||
}, nil),
|
||||
}
|
||||
}
|
||||
|
||||
// findMatches takes a known text and finds all potential instances of it in
|
||||
// the unknown text. The resulting matches can then filtered to determine which
|
||||
// are the best matches.
|
||||
func (m *matcher) findMatches(known *knownValue) {
|
||||
var mrs []searchset.MatchRanges
|
||||
if all := known.reValue.FindAllStringIndex(m.normUnknown, -1); all != nil {
|
||||
// We found exact matches. Just use those!
|
||||
for _, a := range all {
|
||||
var start, end int
|
||||
for i, tok := range m.unknown.Tokens {
|
||||
if tok.Offset == a[0] {
|
||||
start = i
|
||||
} else if tok.Offset >= a[len(a)-1]-len(tok.Text) {
|
||||
end = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
mrs = append(mrs, searchset.MatchRanges{{
|
||||
SrcStart: 0,
|
||||
SrcEnd: len(known.set.Tokens),
|
||||
TargetStart: start,
|
||||
TargetEnd: end + 1,
|
||||
}})
|
||||
}
|
||||
} else {
|
||||
// No exact match. Perform a more thorough match.
|
||||
mrs = searchset.FindPotentialMatches(known.set, m.unknown)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, mr := range mrs {
|
||||
if !m.withinConfidenceThreshold(known.set, mr) {
|
||||
continue
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(mr searchset.MatchRanges) {
|
||||
start, end := mr.TargetRange(m.unknown)
|
||||
conf := levDist(m.normUnknown[start:end], known.normalizedValue)
|
||||
if conf > 0.0 {
|
||||
m.mu.Lock()
|
||||
m.queue.Push(&Match{Name: known.key, Confidence: conf, Offset: start, Extent: end - start})
|
||||
m.mu.Unlock()
|
||||
}
|
||||
wg.Done()
|
||||
}(mr)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// withinConfidenceThreshold returns the Confidence we have in the potential
|
||||
// match. It does this by calculating the ratio of what's matching to the
|
||||
// original known text.
|
||||
func (m *matcher) withinConfidenceThreshold(known *searchset.SearchSet, mr searchset.MatchRanges) bool {
|
||||
return float64(mr.Size())/float64(len(known.Tokens)) >= m.threshold
|
||||
}
|
||||
|
||||
// multipleMatch returns a Queue of values that might be within the unknown
|
||||
// string. The values are compared via their Levenshtein Distance and ranked
|
||||
// with the nearest match at the beginning.
|
||||
func (c *Classifier) multipleMatch(unknown string) *pq.Queue {
|
||||
normUnknown := c.normalize(unknown)
|
||||
if normUnknown == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
m := newMatcher(normUnknown, c.threshold)
|
||||
|
||||
c.muValues.RLock()
|
||||
var kvals []*knownValue
|
||||
for _, known := range c.values {
|
||||
kvals = append(kvals, known)
|
||||
}
|
||||
c.muValues.RUnlock()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(kvals))
|
||||
for _, known := range kvals {
|
||||
go func(known *knownValue) {
|
||||
if known.set == nil {
|
||||
k := searchset.New(known.normalizedValue, searchset.DefaultGranularity)
|
||||
c.muValues.Lock()
|
||||
c.values[known.key].set = k
|
||||
c.muValues.Unlock()
|
||||
}
|
||||
m.findMatches(known)
|
||||
wg.Done()
|
||||
}(known)
|
||||
}
|
||||
wg.Wait()
|
||||
return m.queue
|
||||
}
|
||||
|
||||
// levDist runs the Levenshtein Distance algorithm on the known and unknown
|
||||
// texts to measure how well they match.
|
||||
func levDist(unknown, known string) float64 {
|
||||
if len(known) == 0 || len(unknown) == 0 {
|
||||
log.Printf("Zero-sized texts in Levenshtein Distance algorithm: known==%d, unknown==%d", len(known), len(unknown))
|
||||
return 0.0
|
||||
}
|
||||
|
||||
// Calculate the differences between the potentially matching known
|
||||
// text and the unknown text.
|
||||
diffs := dmp.DiffMain(unknown, known, false)
|
||||
end := diffRangeEnd(known, diffs)
|
||||
|
||||
// Now execute the Levenshtein Distance algorithm to see how much it
|
||||
// does match.
|
||||
distance := dmp.DiffLevenshtein(diffs[:end])
|
||||
return confidencePercentage(unknownTextLength(unknown, diffs), len(known), distance)
|
||||
}
|
||||
|
||||
// unknownTextLength returns the length of the unknown text based on the diff range.
|
||||
func unknownTextLength(unknown string, diffs []diffmatchpatch.Diff) int {
|
||||
last := len(diffs) - 1
|
||||
for ; last >= 0; last-- {
|
||||
if diffs[last].Type == diffmatchpatch.DiffEqual {
|
||||
break
|
||||
}
|
||||
}
|
||||
ulen := 0
|
||||
for i := 0; i < last+1; i++ {
|
||||
switch diffs[i].Type {
|
||||
case diffmatchpatch.DiffEqual, diffmatchpatch.DiffDelete:
|
||||
ulen += len(diffs[i].Text)
|
||||
}
|
||||
}
|
||||
return ulen
|
||||
}
|
||||
|
||||
// diffRangeEnd returns the end index for the "Diff" objects that constructs
|
||||
// (or nearly constructs) the "known" value.
|
||||
func diffRangeEnd(known string, diffs []diffmatchpatch.Diff) (end int) {
|
||||
var seen string
|
||||
for end = 0; end < len(diffs); end++ {
|
||||
if seen == known {
|
||||
// Once we've constructed the "known" value, then we've
|
||||
// reached the point in the diff list where more
|
||||
// "Diff"s would just make the Levenshtein Distance
|
||||
// less valid. There shouldn't be further "DiffEqual"
|
||||
// nodes, because there's nothing further to match in
|
||||
// the "known" text.
|
||||
break
|
||||
}
|
||||
switch diffs[end].Type {
|
||||
case diffmatchpatch.DiffEqual, diffmatchpatch.DiffInsert:
|
||||
seen += diffs[end].Text
|
||||
}
|
||||
}
|
||||
return end
|
||||
}
|
||||
|
||||
// confidencePercentage calculates how confident we are in the result of the
|
||||
// match. A percentage of "1.0" means an identical match. A confidence of "0.0"
|
||||
// means a complete mismatch.
|
||||
func confidencePercentage(ulen, klen, distance int) float64 {
|
||||
if ulen == 0 && klen == 0 {
|
||||
return 1.0
|
||||
}
|
||||
if ulen == 0 || klen == 0 || (distance > ulen && distance > klen) {
|
||||
return 0.0
|
||||
}
|
||||
return 1.0 - float64(distance)/float64(max(ulen, klen))
|
||||
}
|
||||
|
||||
// diffRatio calculates the ratio of the length of s1 and s2, returned as a
|
||||
// percentage of the length of the longer string. E.g., diffLength("abcd", "e")
|
||||
// would return 0.25 because "e" is 25% of the size of "abcd". Comparing
|
||||
// strings of equal length will return 1.
|
||||
func diffRatio(s1, s2 string) float64 {
|
||||
x, y := len(s1), len(s2)
|
||||
if x == 0 && y == 0 {
|
||||
// Both strings are zero length
|
||||
return 1.0
|
||||
}
|
||||
if x < y {
|
||||
return float64(x) / float64(y)
|
||||
}
|
||||
return float64(y) / float64(x)
|
||||
}
|
||||
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// wsRegexp is a regexp used to identify blocks of whitespace.
|
||||
var wsRegexp = regexp.MustCompile(`\s+`)
|
||||
|
||||
// FlattenWhitespace will flatten contiguous blocks of whitespace down to a single space.
|
||||
var FlattenWhitespace NormalizeFunc = func(s string) string {
|
||||
return wsRegexp.ReplaceAllString(s, " ")
|
||||
}
|
||||
111
vendor/github.com/google/licenseclassifier/stringclassifier/internal/pq/priority.go
generated
vendored
111
vendor/github.com/google/licenseclassifier/stringclassifier/internal/pq/priority.go
generated
vendored
|
|
@ -1,111 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package pq provides a priority queue.
|
||||
package pq
|
||||
|
||||
import "container/heap"
|
||||
|
||||
// NewQueue returns an unbounded priority queue that compares elements using
|
||||
// less; the minimal element is at the top of the queue.
|
||||
//
|
||||
// If setIndex is not nil, the queue calls setIndex to inform each element of
|
||||
// its position in the queue. If an element's priority changes, its position in
|
||||
// the queue may be incorrect. Call Fix on the element's index to update the
|
||||
// queue. Call Remove on the element's index to remove it from the queue.
|
||||
func NewQueue(less func(x, y interface{}) bool, setIndex func(x interface{}, idx int)) *Queue {
|
||||
return &Queue{
|
||||
heap: pqHeap{
|
||||
less: less,
|
||||
setIndex: setIndex,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Queue is a priority queue that supports updating the priority of an element.
|
||||
// A Queue must be created with NewQueue.
|
||||
type Queue struct {
|
||||
heap pqHeap
|
||||
}
|
||||
|
||||
// Len returns the number of elements in the queue.
|
||||
func (pq *Queue) Len() int {
|
||||
return pq.heap.Len()
|
||||
}
|
||||
|
||||
// Push adds x to the queue.
|
||||
func (pq *Queue) Push(x interface{}) {
|
||||
heap.Push(&pq.heap, x)
|
||||
}
|
||||
|
||||
// Min returns the minimal element.
|
||||
// Min panics if the queue is empty.
|
||||
func (pq *Queue) Min() interface{} {
|
||||
return pq.heap.a[0]
|
||||
}
|
||||
|
||||
// Pop removes and returns the minimal element.
|
||||
// Pop panics if the queue is empty.
|
||||
func (pq *Queue) Pop() interface{} {
|
||||
return heap.Pop(&pq.heap)
|
||||
}
|
||||
|
||||
// Fix adjusts the heap to reflect that the element at index has changed priority.
|
||||
func (pq *Queue) Fix(index int) {
|
||||
heap.Fix(&pq.heap, index)
|
||||
}
|
||||
|
||||
// Remove removes the element at index i from the heap.
|
||||
func (pq *Queue) Remove(index int) {
|
||||
heap.Remove(&pq.heap, index)
|
||||
}
|
||||
|
||||
// pqHeap implements heap.Interface.
|
||||
type pqHeap struct {
|
||||
a []interface{}
|
||||
less func(x, y interface{}) bool
|
||||
setIndex func(x interface{}, idx int)
|
||||
}
|
||||
|
||||
func (h pqHeap) Len() int {
|
||||
return len(h.a)
|
||||
}
|
||||
|
||||
func (h pqHeap) Less(i, j int) bool {
|
||||
return h.less(h.a[i], h.a[j])
|
||||
}
|
||||
|
||||
func (h pqHeap) Swap(i, j int) {
|
||||
h.a[i], h.a[j] = h.a[j], h.a[i]
|
||||
if h.setIndex != nil {
|
||||
h.setIndex(h.a[i], i)
|
||||
h.setIndex(h.a[j], j)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *pqHeap) Push(x interface{}) {
|
||||
n := len(h.a)
|
||||
if h.setIndex != nil {
|
||||
h.setIndex(x, n)
|
||||
}
|
||||
h.a = append(h.a, x)
|
||||
}
|
||||
|
||||
func (h *pqHeap) Pop() interface{} {
|
||||
old := h.a
|
||||
n := len(old)
|
||||
x := old[n-1]
|
||||
h.a = old[:n-1]
|
||||
return x
|
||||
}
|
||||
491
vendor/github.com/google/licenseclassifier/stringclassifier/searchset/searchset.go
generated
vendored
491
vendor/github.com/google/licenseclassifier/stringclassifier/searchset/searchset.go
generated
vendored
|
|
@ -1,491 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package searchset generates hashes for all substrings of a text. Potential
|
||||
// matches between two SearchSet objects can then be determined quickly.
|
||||
// Generating the hashes can be expensive, so it's best to perform it once. If
|
||||
// the text is part of a known corpus, then the SearchSet can be serialized and
|
||||
// kept in an archive.
|
||||
//
|
||||
// Matching occurs by "mapping" ranges from the source text into the target
|
||||
// text but still retaining the source order:
|
||||
//
|
||||
// SOURCE: |-----------------------------|
|
||||
//
|
||||
// TARGET: |*****************************************|
|
||||
//
|
||||
// MAP SOURCE SECTIONS ONTO TARGET IN SOURCE ORDER:
|
||||
//
|
||||
// S: |-[--]-----[---]------[----]------|
|
||||
// / | \
|
||||
// |---| |---------| |-------------|
|
||||
// T: |*****************************************|
|
||||
//
|
||||
// Note that a single source range may match many different ranges in the
|
||||
// target. The matching algorithm untangles these so that all matched ranges
|
||||
// are in order with respect to the source ranges. This is especially important
|
||||
// since the source text may occur more than once in the target text. The
|
||||
// algorithm finds each potential occurrence of S in T and returns all as
|
||||
// potential matched ranges.
|
||||
package searchset
|
||||
|
||||
import (
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
|
||||
"github.com/google/licenseclassifier/stringclassifier/searchset/tokenizer"
|
||||
)
|
||||
|
||||
// DefaultGranularity is the minimum size (in words) of the hash chunks.
|
||||
const DefaultGranularity = 3
|
||||
|
||||
// SearchSet is a set of substrings that have hashes associated with them,
|
||||
// making it fast to search for potential matches.
|
||||
type SearchSet struct {
|
||||
// Tokens is a tokenized list of the original input string.
|
||||
Tokens tokenizer.Tokens
|
||||
// Hashes is a map of checksums to a range of tokens.
|
||||
Hashes tokenizer.Hash
|
||||
// Checksums is a list of checksums ordered from longest range to
|
||||
// shortest.
|
||||
Checksums []uint32
|
||||
// ChecksumRanges are the token ranges for the above checksums.
|
||||
ChecksumRanges tokenizer.TokenRanges
|
||||
|
||||
nodes []*node
|
||||
}
|
||||
|
||||
// node consists of a range of tokens along with the checksum for those tokens.
|
||||
type node struct {
|
||||
checksum uint32
|
||||
tokens *tokenizer.TokenRange
|
||||
}
|
||||
|
||||
func (n *node) String() string {
|
||||
return fmt.Sprintf("[%d:%d]", n.tokens.Start, n.tokens.End)
|
||||
}
|
||||
|
||||
// New creates a new SearchSet object. It generates a hash for each substring of "s".
|
||||
func New(s string, granularity int) *SearchSet {
|
||||
toks := tokenizer.Tokenize(s)
|
||||
|
||||
// Start generating hash values for all substrings within the text.
|
||||
h := make(tokenizer.Hash)
|
||||
checksums, tokenRanges := toks.GenerateHashes(h, func(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}(len(toks), granularity))
|
||||
sset := &SearchSet{
|
||||
Tokens: toks,
|
||||
Hashes: h,
|
||||
Checksums: checksums,
|
||||
ChecksumRanges: tokenRanges,
|
||||
}
|
||||
sset.GenerateNodeList()
|
||||
return sset
|
||||
}
|
||||
|
||||
// GenerateNodeList creates a node list out of the search set.
|
||||
func (s *SearchSet) GenerateNodeList() {
|
||||
if len(s.Tokens) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for i := 0; i < len(s.Checksums); i++ {
|
||||
s.nodes = append(s.nodes, &node{
|
||||
checksum: s.Checksums[i],
|
||||
tokens: s.ChecksumRanges[i],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize emits the SearchSet out so that it can be recreated at a later
|
||||
// time.
|
||||
func (s *SearchSet) Serialize(w io.Writer) error {
|
||||
return gob.NewEncoder(w).Encode(s)
|
||||
}
|
||||
|
||||
// Deserialize reads a file with a serialized SearchSet in it and reconstructs it.
|
||||
func Deserialize(r io.Reader, s *SearchSet) error {
|
||||
if err := gob.NewDecoder(r).Decode(&s); err != nil {
|
||||
return err
|
||||
}
|
||||
s.GenerateNodeList()
|
||||
return nil
|
||||
}
|
||||
|
||||
// MatchRange is the range within the source text that is a match to the range
|
||||
// in the target text.
|
||||
type MatchRange struct {
|
||||
// Offsets into the source tokens.
|
||||
SrcStart, SrcEnd int
|
||||
// Offsets into the target tokens.
|
||||
TargetStart, TargetEnd int
|
||||
}
|
||||
|
||||
// in returns true if the start and end are enclosed in the match range.
|
||||
func (m *MatchRange) in(start, end int) bool {
|
||||
return start >= m.TargetStart && end <= m.TargetEnd
|
||||
}
|
||||
|
||||
func (m *MatchRange) String() string {
|
||||
return fmt.Sprintf("[%v, %v)->[%v, %v)", m.SrcStart, m.SrcEnd, m.TargetStart, m.TargetEnd)
|
||||
}
|
||||
|
||||
// MatchRanges is a list of "MatchRange"s. The ranges are monotonically
|
||||
// increasing in value and indicate a single potential occurrence of the source
|
||||
// text in the target text.
|
||||
type MatchRanges []*MatchRange
|
||||
|
||||
func (m MatchRanges) Len() int { return len(m) }
|
||||
func (m MatchRanges) Swap(i, j int) { m[i], m[j] = m[j], m[i] }
|
||||
func (m MatchRanges) Less(i, j int) bool {
|
||||
if m[i].TargetStart < m[j].TargetStart {
|
||||
return true
|
||||
}
|
||||
return m[i].TargetStart == m[j].TargetStart && m[i].SrcStart < m[j].SrcStart
|
||||
}
|
||||
|
||||
// TargetRange is the start and stop token offsets into the target text.
|
||||
func (m MatchRanges) TargetRange(target *SearchSet) (start, end int) {
|
||||
start = target.Tokens[m[0].TargetStart].Offset
|
||||
end = target.Tokens[m[len(m)-1].TargetEnd-1].Offset + len(target.Tokens[m[len(m)-1].TargetEnd-1].Text)
|
||||
return start, end
|
||||
}
|
||||
|
||||
// Size is the number of source tokens that were matched.
|
||||
func (m MatchRanges) Size() int {
|
||||
sum := 0
|
||||
for _, mr := range m {
|
||||
sum += mr.SrcEnd - mr.SrcStart
|
||||
}
|
||||
return sum
|
||||
}
|
||||
|
||||
// FindPotentialMatches returns the ranges in the target (unknown) text that
|
||||
// are best potential matches to the source (known) text.
|
||||
func FindPotentialMatches(src, target *SearchSet) []MatchRanges {
|
||||
matchedRanges := getMatchedRanges(src, target)
|
||||
if len(matchedRanges) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Cleanup the matching ranges so that we get the longest contiguous ranges.
|
||||
for i := 0; i < len(matchedRanges); i++ {
|
||||
matchedRanges[i] = coalesceMatchRanges(matchedRanges[i])
|
||||
}
|
||||
return matchedRanges
|
||||
}
|
||||
|
||||
// getMatchedRanges finds the ranges in the target text that match the source
|
||||
// text. There can be multiple occurrences of the source text within the target
|
||||
// text. Each separate occurrence is an entry in the returned slice.
|
||||
func getMatchedRanges(src, target *SearchSet) []MatchRanges {
|
||||
matched := targetMatchedRanges(src, target)
|
||||
if len(matched) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Sort(matched)
|
||||
matched = untangleSourceRanges(matched)
|
||||
matchedRanges := splitRanges(matched)
|
||||
return mergeConsecutiveRanges(matchedRanges)
|
||||
}
|
||||
|
||||
func extendsAny(tr tokenizer.TokenRanges, mr []MatchRanges) bool {
|
||||
if len(mr) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, tv := range tr {
|
||||
for _, mv := range mr {
|
||||
if tv.Start >= mv[0].TargetStart && tv.Start <= mv[len(mv)-1].TargetEnd {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// targetMatchedRanges finds matching sequences in target and src ordered by target position
|
||||
func targetMatchedRanges(src, target *SearchSet) MatchRanges {
|
||||
if src.nodes == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var matched MatchRanges
|
||||
var previous *node
|
||||
var possible []MatchRanges
|
||||
for _, tgtNode := range target.nodes {
|
||||
sr, ok := src.Hashes[tgtNode.checksum]
|
||||
if !ok || (previous != nil && tgtNode.tokens.Start > previous.tokens.End) || !extendsAny(sr, possible) {
|
||||
for _, r := range possible {
|
||||
matched = append(matched, r...)
|
||||
}
|
||||
possible = possible[:0]
|
||||
previous = nil
|
||||
}
|
||||
if !ok {
|
||||
// There isn't a match in the source.
|
||||
continue
|
||||
}
|
||||
|
||||
// Maps index within `possible` to the slice of ranges extended by a new range
|
||||
extended := make(map[int]*MatchRanges)
|
||||
// Go over the set of source ranges growing lists of `possible` match ranges.
|
||||
tv := tgtNode.tokens
|
||||
for _, sv := range sr {
|
||||
r := &MatchRange{
|
||||
SrcStart: sv.Start,
|
||||
SrcEnd: sv.End,
|
||||
TargetStart: tv.Start,
|
||||
TargetEnd: tv.End,
|
||||
}
|
||||
found := false
|
||||
// Grow or extend each abutting `possible` match range.
|
||||
for i, p := range possible {
|
||||
last := p[len(p)-1]
|
||||
if sv.Start >= last.SrcStart && sv.Start <= last.SrcEnd && tv.Start >= last.TargetStart && tv.Start <= last.TargetEnd {
|
||||
found = true
|
||||
possible[i] = append(possible[i], r)
|
||||
extended[i] = &possible[i]
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
// Did not abut any existing ranges, start a new `possible` match range.
|
||||
mrs := make(MatchRanges, 0, 2)
|
||||
mrs = append(mrs, r)
|
||||
possible = append(possible, mrs)
|
||||
extended[len(possible)-1] = &possible[len(possible)-1]
|
||||
}
|
||||
}
|
||||
if len(extended) < len(possible) {
|
||||
// Ranges not extended--add to `matched` if not included in other range.
|
||||
for i := 0; i < len(possible); {
|
||||
_, updated := extended[i]
|
||||
if updated {
|
||||
i++ // Keep in `possible` and advance to next index.
|
||||
continue
|
||||
}
|
||||
p1 := possible[i]
|
||||
found := false // whether found as subrange of another `possible` match.
|
||||
for _, p2 := range extended {
|
||||
if p1[0].SrcStart >= (*p2)[0].SrcStart && p1[0].TargetStart >= (*p2)[0].TargetStart {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
matched = append(matched, p1...)
|
||||
} // else included in other match.
|
||||
// Finished -- delete from `possible` and continue from same index.
|
||||
possible = append(possible[:i], possible[i+1:]...)
|
||||
}
|
||||
}
|
||||
previous = tgtNode
|
||||
}
|
||||
// At end of file, terminate all `possible` match ranges.
|
||||
for i := 0; i < len(possible); i++ {
|
||||
p1 := possible[i]
|
||||
found := false // whether found as subrange of another `possible` match.
|
||||
for j := i + 1; j < len(possible); {
|
||||
p2 := possible[j]
|
||||
if p1[0].SrcStart <= p2[0].SrcStart && p1[0].TargetStart <= p2[0].TargetStart {
|
||||
// Delete later sub-ranges included in this range.
|
||||
possible = append(possible[:j], possible[j+1:]...)
|
||||
continue
|
||||
}
|
||||
// Skip if subrange of a later range
|
||||
if p1[0].SrcStart >= p2[0].SrcStart && p1[0].TargetStart >= p2[0].TargetStart {
|
||||
found = true
|
||||
}
|
||||
j++
|
||||
}
|
||||
if !found {
|
||||
matched = append(matched, p1...)
|
||||
}
|
||||
}
|
||||
return matched
|
||||
}
|
||||
|
||||
// untangleSourceRanges goes through the ranges and removes any whose source
|
||||
// ranges are "out of order". A source range is "out of order" if the source
|
||||
// range is out of sequence with the source ranges before and after it. This
|
||||
// happens when more than one source range maps to the same target range.
|
||||
// E.g.:
|
||||
//
|
||||
// SrcStart: 20, SrcEnd: 30, TargetStart: 127, TargetEnd: 137
|
||||
// 1: SrcStart: 12, SrcEnd: 17, TargetStart: 138, TargetEnd: 143
|
||||
// 2: SrcStart: 32, SrcEnd: 37, TargetStart: 138, TargetEnd: 143
|
||||
// SrcStart: 38, SrcEnd: 40, TargetStart: 144, TargetEnd: 146
|
||||
//
|
||||
// Here (1) is out of order, because the source range [12, 17) is out of
|
||||
// sequence with the surrounding source sequences, but [32, 37) is.
|
||||
func untangleSourceRanges(matched MatchRanges) MatchRanges {
|
||||
mr := MatchRanges{matched[0]}
|
||||
NEXT:
|
||||
for i := 1; i < len(matched); i++ {
|
||||
if mr[len(mr)-1].TargetStart == matched[i].TargetStart && mr[len(mr)-1].TargetEnd == matched[i].TargetEnd {
|
||||
// The matched range has already been added.
|
||||
continue
|
||||
}
|
||||
|
||||
if i+1 < len(matched) && equalTargetRange(matched[i], matched[i+1]) {
|
||||
// A sequence of ranges match the same target range.
|
||||
// Find the first one that has a source range greater
|
||||
// than the currently matched range. Omit all others.
|
||||
if matched[i].SrcStart > mr[len(mr)-1].SrcStart {
|
||||
mr = append(mr, matched[i])
|
||||
continue
|
||||
}
|
||||
|
||||
for j := i + 1; j < len(matched) && equalTargetRange(matched[i], matched[j]); j++ {
|
||||
// Check subsequent ranges to see if we can
|
||||
// find one that matches in the correct order.
|
||||
if matched[j].SrcStart > mr[len(mr)-1].SrcStart {
|
||||
mr = append(mr, matched[j])
|
||||
i = j
|
||||
continue NEXT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mr = append(mr, matched[i])
|
||||
}
|
||||
return mr
|
||||
}
|
||||
|
||||
// equalTargetRange returns true if the two MatchRange's cover the same target range.
|
||||
func equalTargetRange(this, that *MatchRange) bool {
|
||||
return this.TargetStart == that.TargetStart && this.TargetEnd == that.TargetEnd
|
||||
}
|
||||
|
||||
// splitRanges splits the matched ranges so that a single match range has a
|
||||
// monotonically increasing source range (indicating a single, potential
|
||||
// instance of the source in the target).
|
||||
func splitRanges(matched MatchRanges) []MatchRanges {
|
||||
var matchedRanges []MatchRanges
|
||||
mr := MatchRanges{matched[0]}
|
||||
for i := 1; i < len(matched); i++ {
|
||||
if mr[len(mr)-1].SrcStart > matched[i].SrcStart {
|
||||
matchedRanges = append(matchedRanges, mr)
|
||||
mr = MatchRanges{matched[i]}
|
||||
} else {
|
||||
mr = append(mr, matched[i])
|
||||
}
|
||||
}
|
||||
matchedRanges = append(matchedRanges, mr)
|
||||
return matchedRanges
|
||||
}
|
||||
|
||||
// mergeConsecutiveRanges goes through the matched ranges and merges
|
||||
// consecutive ranges. Two ranges are consecutive if the end of the previous
|
||||
// matched range and beginning of the next matched range overlap. "matched"
|
||||
// should have 1 or more MatchRanges, each with one or more MatchRange objects.
|
||||
func mergeConsecutiveRanges(matched []MatchRanges) []MatchRanges {
|
||||
mr := []MatchRanges{matched[0]}
|
||||
|
||||
// Convenience functions.
|
||||
prevMatchedRange := func() MatchRanges {
|
||||
return mr[len(mr)-1]
|
||||
}
|
||||
prevMatchedRangeLastElem := func() *MatchRange {
|
||||
return prevMatchedRange()[len(prevMatchedRange())-1]
|
||||
}
|
||||
|
||||
// This algorithm compares the start of each MatchRanges object to the
|
||||
// end of the previous MatchRanges object. If they overlap, then it
|
||||
// tries to combine them. Note that a 0 offset into a MatchRanges
|
||||
// object (e.g., matched[i][0]) is its first MatchRange, which
|
||||
// indicates the start of the whole matched range.
|
||||
NEXT:
|
||||
for i := 1; i < len(matched); i++ {
|
||||
if prevMatchedRangeLastElem().TargetEnd > matched[i][0].TargetStart {
|
||||
// Consecutive matched ranges overlap. Merge them.
|
||||
if prevMatchedRangeLastElem().TargetStart < matched[i][0].TargetStart {
|
||||
// The last element of the previous matched
|
||||
// range overlaps with the first element of the
|
||||
// current matched range. Concatenate them.
|
||||
if prevMatchedRangeLastElem().TargetEnd < matched[i][0].TargetEnd {
|
||||
prevMatchedRangeLastElem().SrcEnd += matched[i][0].TargetEnd - prevMatchedRangeLastElem().TargetEnd
|
||||
prevMatchedRangeLastElem().TargetEnd = matched[i][0].TargetEnd
|
||||
}
|
||||
mr[len(mr)-1] = append(prevMatchedRange(), matched[i][1:]...)
|
||||
continue
|
||||
}
|
||||
|
||||
for j := 1; j < len(matched[i]); j++ {
|
||||
// Find the positions in the ranges where the
|
||||
// tail end of the previous matched range
|
||||
// overlaps with the start of the next matched
|
||||
// range.
|
||||
for k := len(prevMatchedRange()) - 1; k > 0; k-- {
|
||||
if prevMatchedRange()[k].SrcStart < matched[i][j].SrcStart &&
|
||||
prevMatchedRange()[k].TargetStart < matched[i][j].TargetStart {
|
||||
// Append the next range to the previous range.
|
||||
if prevMatchedRange()[k].TargetEnd < matched[i][j].TargetStart {
|
||||
// Coalesce the ranges.
|
||||
prevMatchedRange()[k].SrcEnd += matched[i][j-1].TargetEnd - prevMatchedRange()[k].TargetEnd
|
||||
prevMatchedRange()[k].TargetEnd = matched[i][j-1].TargetEnd
|
||||
}
|
||||
mr[len(mr)-1] = append(prevMatchedRange()[:k+1], matched[i][j:]...)
|
||||
continue NEXT
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
mr = append(mr, matched[i])
|
||||
}
|
||||
return mr
|
||||
}
|
||||
|
||||
// coalesceMatchRanges coalesces overlapping match ranges into a single
|
||||
// contiguous match range.
|
||||
func coalesceMatchRanges(matchedRanges MatchRanges) MatchRanges {
|
||||
coalesced := MatchRanges{matchedRanges[0]}
|
||||
for i := 1; i < len(matchedRanges); i++ {
|
||||
c := coalesced[len(coalesced)-1]
|
||||
mr := matchedRanges[i]
|
||||
|
||||
if mr.SrcStart <= c.SrcEnd && mr.SrcStart >= c.SrcStart {
|
||||
var se, ts, te int
|
||||
if mr.SrcEnd > c.SrcEnd {
|
||||
se = mr.SrcEnd
|
||||
} else {
|
||||
se = c.SrcEnd
|
||||
}
|
||||
if mr.TargetStart < c.TargetStart {
|
||||
ts = mr.TargetStart
|
||||
} else {
|
||||
ts = c.TargetStart
|
||||
}
|
||||
if mr.TargetEnd > c.TargetEnd {
|
||||
te = mr.TargetEnd
|
||||
} else {
|
||||
te = c.TargetEnd
|
||||
}
|
||||
coalesced[len(coalesced)-1] = &MatchRange{
|
||||
SrcStart: c.SrcStart,
|
||||
SrcEnd: se,
|
||||
TargetStart: ts,
|
||||
TargetEnd: te,
|
||||
}
|
||||
} else {
|
||||
coalesced = append(coalesced, mr)
|
||||
}
|
||||
}
|
||||
return coalesced
|
||||
}
|
||||
175
vendor/github.com/google/licenseclassifier/stringclassifier/searchset/tokenizer/tokenizer.go
generated
vendored
175
vendor/github.com/google/licenseclassifier/stringclassifier/searchset/tokenizer/tokenizer.go
generated
vendored
|
|
@ -1,175 +0,0 @@
|
|||
// Copyright 2017 Google Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package tokenizer converts a text into a stream of tokens.
|
||||
package tokenizer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"sort"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Token is a non-whitespace sequence (i.e., word or punctuation) in the
|
||||
// original string. This is not meant for use outside of this package.
|
||||
type token struct {
|
||||
Text string
|
||||
Offset int
|
||||
}
|
||||
|
||||
// Tokens is a list of Token objects.
|
||||
type Tokens []*token
|
||||
|
||||
// newToken creates a new token object with an invalid (negative) offset, which
|
||||
// will be set before the token's used.
|
||||
func newToken() *token {
|
||||
return &token{Offset: -1}
|
||||
}
|
||||
|
||||
// Tokenize converts a string into a stream of tokens.
|
||||
func Tokenize(s string) (toks Tokens) {
|
||||
tok := newToken()
|
||||
for i := 0; i < len(s); {
|
||||
r, size := utf8.DecodeRuneInString(s[i:])
|
||||
switch {
|
||||
case unicode.IsSpace(r):
|
||||
if tok.Offset >= 0 {
|
||||
toks = append(toks, tok)
|
||||
tok = newToken()
|
||||
}
|
||||
case unicode.IsPunct(r):
|
||||
if tok.Offset >= 0 {
|
||||
toks = append(toks, tok)
|
||||
tok = newToken()
|
||||
}
|
||||
toks = append(toks, &token{
|
||||
Text: string(r),
|
||||
Offset: i,
|
||||
})
|
||||
default:
|
||||
if tok.Offset == -1 {
|
||||
tok.Offset = i
|
||||
}
|
||||
tok.Text += string(r)
|
||||
}
|
||||
i += size
|
||||
}
|
||||
if tok.Offset != -1 {
|
||||
// Add any remaining token that wasn't yet included in the list.
|
||||
toks = append(toks, tok)
|
||||
}
|
||||
return toks
|
||||
}
|
||||
|
||||
// GenerateHashes generates hashes for "size" length substrings. The
|
||||
// "stringifyTokens" call takes a long time to run, so not all substrings have
|
||||
// hashes, i.e. we skip some of the smaller substrings.
|
||||
func (t Tokens) GenerateHashes(h Hash, size int) ([]uint32, TokenRanges) {
|
||||
if size == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
var css []uint32
|
||||
var tr TokenRanges
|
||||
for offset := 0; offset+size <= len(t); offset += size / 2 {
|
||||
var b bytes.Buffer
|
||||
t.stringifyTokens(&b, offset, size)
|
||||
cs := crc32.ChecksumIEEE(b.Bytes())
|
||||
css = append(css, cs)
|
||||
tr = append(tr, &TokenRange{offset, offset + size})
|
||||
h.add(cs, offset, offset+size)
|
||||
if size <= 1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return css, tr
|
||||
}
|
||||
|
||||
// stringifyTokens serializes a sublist of tokens into a bytes buffer.
|
||||
func (t Tokens) stringifyTokens(b *bytes.Buffer, offset, size int) {
|
||||
for j := offset; j < offset+size; j++ {
|
||||
if j != offset {
|
||||
b.WriteRune(' ')
|
||||
}
|
||||
b.WriteString(t[j].Text)
|
||||
}
|
||||
}
|
||||
|
||||
// TokenRange indicates the range of tokens that map to a particular checksum.
|
||||
type TokenRange struct {
|
||||
Start int
|
||||
End int
|
||||
}
|
||||
|
||||
func (t *TokenRange) String() string {
|
||||
return fmt.Sprintf("[%v, %v)", t.Start, t.End)
|
||||
}
|
||||
|
||||
// TokenRanges is a list of TokenRange objects. The chance that two different
|
||||
// strings map to the same checksum is very small, but unfortunately isn't
|
||||
// zero, so we use this instead of making the assumption that they will all be
|
||||
// unique.
|
||||
type TokenRanges []*TokenRange
|
||||
|
||||
func (t TokenRanges) Len() int { return len(t) }
|
||||
func (t TokenRanges) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||
func (t TokenRanges) Less(i, j int) bool { return t[i].Start < t[j].Start }
|
||||
|
||||
// CombineUnique returns the combination of both token ranges with no duplicates.
|
||||
func (t TokenRanges) CombineUnique(other TokenRanges) TokenRanges {
|
||||
if len(other) == 0 {
|
||||
return t
|
||||
}
|
||||
if len(t) == 0 {
|
||||
return other
|
||||
}
|
||||
|
||||
cu := append(t, other...)
|
||||
sort.Sort(cu)
|
||||
|
||||
if len(cu) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
res := TokenRanges{cu[0]}
|
||||
for prev, i := cu[0], 1; i < len(cu); i++ {
|
||||
if prev.Start != cu[i].Start || prev.End != cu[i].End {
|
||||
res = append(res, cu[i])
|
||||
prev = cu[i]
|
||||
}
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// Hash is a map of the hashes of a section of text to the token range covering that text.
|
||||
type Hash map[uint32]TokenRanges
|
||||
|
||||
// add associates a token range, [start, end], to a checksum.
|
||||
func (h Hash) add(checksum uint32, start, end int) {
|
||||
ntr := &TokenRange{Start: start, End: end}
|
||||
if r, ok := h[checksum]; ok {
|
||||
for _, tr := range r {
|
||||
if tr.Start == ntr.Start && tr.End == ntr.End {
|
||||
// The token range already exists at this
|
||||
// checksum. No need to re-add it.
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
h[checksum] = append(h[checksum], ntr)
|
||||
}
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
version = 1
|
||||
|
||||
test_patterns = [
|
||||
"*_test.go"
|
||||
]
|
||||
|
||||
[[analyzers]]
|
||||
name = "go"
|
||||
enabled = true
|
||||
|
||||
[analyzers.meta]
|
||||
import_path = "github.com/imdario/mergo"
|
||||
|
|
@ -4,6 +4,4 @@ install:
|
|||
- go get golang.org/x/tools/cmd/cover
|
||||
- go get github.com/mattn/goveralls
|
||||
script:
|
||||
- go test -race -v ./...
|
||||
after_script:
|
||||
- $HOME/gopath/bin/goveralls -service=travis-ci -repotoken $COVERALLS_TOKEN
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ It is ready for production use. [It is used in several projects by Docker, Googl
|
|||
[![Build Status][1]][2]
|
||||
[![Coverage Status][7]][8]
|
||||
[![Sourcegraph][9]][10]
|
||||
[](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield)
|
||||
|
||||
[1]: https://travis-ci.org/imdario/mergo.png
|
||||
[2]: https://travis-ci.org/imdario/mergo
|
||||
|
|
@ -28,7 +27,7 @@ It is ready for production use. [It is used in several projects by Docker, Googl
|
|||
|
||||
### Latest release
|
||||
|
||||
[Release v0.3.7](https://github.com/imdario/mergo/releases/tag/v0.3.7).
|
||||
[Release v0.3.4](https://github.com/imdario/mergo/releases/tag/v0.3.4).
|
||||
|
||||
### Important note
|
||||
|
||||
|
|
@ -218,21 +217,6 @@ If I can help you, you have an idea or you are using Mergo in your projects, don
|
|||
|
||||
Written by [Dario Castañé](http://dario.im).
|
||||
|
||||
## Top Contributors
|
||||
|
||||
[](https://sourcerer.io/fame/imdario/imdario/mergo/links/0)
|
||||
[](https://sourcerer.io/fame/imdario/imdario/mergo/links/1)
|
||||
[](https://sourcerer.io/fame/imdario/imdario/mergo/links/2)
|
||||
[](https://sourcerer.io/fame/imdario/imdario/mergo/links/3)
|
||||
[](https://sourcerer.io/fame/imdario/imdario/mergo/links/4)
|
||||
[](https://sourcerer.io/fame/imdario/imdario/mergo/links/5)
|
||||
[](https://sourcerer.io/fame/imdario/imdario/mergo/links/6)
|
||||
[](https://sourcerer.io/fame/imdario/imdario/mergo/links/7)
|
||||
|
||||
|
||||
## License
|
||||
|
||||
[BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE).
|
||||
|
||||
|
||||
[](https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_large)
|
||||
|
|
|
|||
|
|
@ -72,7 +72,6 @@ func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, conf
|
|||
case reflect.Struct:
|
||||
srcMap := src.Interface().(map[string]interface{})
|
||||
for key := range srcMap {
|
||||
config.overwriteWithEmptyValue = true
|
||||
srcValue := srcMap[key]
|
||||
fieldName := changeInitialCase(key, unicode.ToUpper)
|
||||
dstElement := dst.FieldByName(fieldName)
|
||||
|
|
@ -99,11 +98,11 @@ func deepMap(dst, src reflect.Value, visited map[uintptr]*visit, depth int, conf
|
|||
continue
|
||||
}
|
||||
if srcKind == dstKind {
|
||||
if _, err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
|
||||
if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
|
||||
return
|
||||
}
|
||||
} else if dstKind == reflect.Interface && dstElement.Kind() == reflect.Interface {
|
||||
if _, err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
|
||||
if err = deepMerge(dstElement, srcElement, visited, depth+1, config); err != nil {
|
||||
return
|
||||
}
|
||||
} else if srcKind == reflect.Map {
|
||||
|
|
@ -157,8 +156,7 @@ func _map(dst, src interface{}, opts ...func(*Config)) error {
|
|||
// To be friction-less, we redirect equal-type arguments
|
||||
// to deepMerge. Only because arguments can be anything.
|
||||
if vSrc.Kind() == vDst.Kind() {
|
||||
_, err := deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
|
||||
return err
|
||||
return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
|
||||
}
|
||||
switch vSrc.Kind() {
|
||||
case reflect.Struct:
|
||||
|
|
|
|||
|
|
@ -9,41 +9,25 @@
|
|||
package mergo
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func hasExportedField(dst reflect.Value) (exported bool) {
|
||||
for i, n := 0, dst.NumField(); i < n; i++ {
|
||||
field := dst.Type().Field(i)
|
||||
if isExportedComponent(&field) {
|
||||
return true
|
||||
if field.Anonymous && dst.Field(i).Kind() == reflect.Struct {
|
||||
exported = exported || hasExportedField(dst.Field(i))
|
||||
} else {
|
||||
exported = exported || len(field.PkgPath) == 0
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func isExportedComponent(field *reflect.StructField) bool {
|
||||
name := field.Name
|
||||
pkgPath := field.PkgPath
|
||||
if len(pkgPath) > 0 {
|
||||
return false
|
||||
}
|
||||
c := name[0]
|
||||
if 'a' <= c && c <= 'z' || c == '_' {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Overwrite bool
|
||||
AppendSlice bool
|
||||
TypeCheck bool
|
||||
Transformers Transformers
|
||||
overwriteWithEmptyValue bool
|
||||
overwriteSliceWithEmptyValue bool
|
||||
}
|
||||
|
||||
type Transformers interface {
|
||||
|
|
@ -53,17 +37,12 @@ type Transformers interface {
|
|||
// Traverses recursively both values, assigning src's fields values to dst.
|
||||
// The map argument tracks comparisons that have already been seen, which allows
|
||||
// short circuiting on recursive types.
|
||||
func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (dst reflect.Value, err error) {
|
||||
dst = dstIn
|
||||
func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, config *Config) (err error) {
|
||||
overwrite := config.Overwrite
|
||||
typeCheck := config.TypeCheck
|
||||
overwriteWithEmptySrc := config.overwriteWithEmptyValue
|
||||
overwriteSliceWithEmptySrc := config.overwriteSliceWithEmptyValue
|
||||
|
||||
if !src.IsValid() {
|
||||
return
|
||||
}
|
||||
|
||||
if dst.CanAddr() {
|
||||
addr := dst.UnsafeAddr()
|
||||
h := 17 * addr
|
||||
|
|
@ -71,7 +50,7 @@ func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int,
|
|||
typ := dst.Type()
|
||||
for p := seen; p != nil; p = p.next {
|
||||
if p.ptr == addr && p.typ == typ {
|
||||
return dst, nil
|
||||
return nil
|
||||
}
|
||||
}
|
||||
// Remember, remember...
|
||||
|
|
@ -85,170 +64,130 @@ func deepMerge(dstIn, src reflect.Value, visited map[uintptr]*visit, depth int,
|
|||
}
|
||||
}
|
||||
|
||||
if dst.IsValid() && src.IsValid() && src.Type() != dst.Type() {
|
||||
err = fmt.Errorf("cannot append two different types (%s, %s)", src.Kind(), dst.Kind())
|
||||
return
|
||||
}
|
||||
|
||||
switch dst.Kind() {
|
||||
case reflect.Struct:
|
||||
if hasExportedField(dst) {
|
||||
dstCp := reflect.New(dst.Type()).Elem()
|
||||
for i, n := 0, dst.NumField(); i < n; i++ {
|
||||
dstField := dst.Field(i)
|
||||
structField := dst.Type().Field(i)
|
||||
// copy un-exported struct fields
|
||||
if !isExportedComponent(&structField) {
|
||||
rf := dstCp.Field(i)
|
||||
rf = reflect.NewAt(rf.Type(), unsafe.Pointer(rf.UnsafeAddr())).Elem() //nolint:gosec
|
||||
dstRF := dst.Field(i)
|
||||
if !dst.Field(i).CanAddr() {
|
||||
continue
|
||||
}
|
||||
|
||||
dstRF = reflect.NewAt(dstRF.Type(), unsafe.Pointer(dstRF.UnsafeAddr())).Elem() //nolint:gosec
|
||||
rf.Set(dstRF)
|
||||
continue
|
||||
}
|
||||
dstField, err = deepMerge(dstField, src.Field(i), visited, depth+1, config)
|
||||
if err != nil {
|
||||
if err = deepMerge(dst.Field(i), src.Field(i), visited, depth+1, config); err != nil {
|
||||
return
|
||||
}
|
||||
dstCp.Field(i).Set(dstField)
|
||||
}
|
||||
|
||||
if dst.CanSet() {
|
||||
dst.Set(dstCp)
|
||||
} else {
|
||||
dst = dstCp
|
||||
}
|
||||
return
|
||||
} else {
|
||||
if (isReflectNil(dst) || overwrite) && (!isEmptyValue(src) || overwriteWithEmptySrc) {
|
||||
dst = src
|
||||
if dst.CanSet() && !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) {
|
||||
dst.Set(src)
|
||||
}
|
||||
}
|
||||
|
||||
case reflect.Map:
|
||||
if dst.IsNil() && !src.IsNil() {
|
||||
if dst.CanSet() {
|
||||
dst.Set(reflect.MakeMap(dst.Type()))
|
||||
} else {
|
||||
dst = src
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, key := range src.MapKeys() {
|
||||
srcElement := src.MapIndex(key)
|
||||
dstElement := dst.MapIndex(key)
|
||||
if !srcElement.IsValid() {
|
||||
continue
|
||||
}
|
||||
if dst.MapIndex(key).IsValid() {
|
||||
k := dstElement.Interface()
|
||||
dstElement = reflect.ValueOf(k)
|
||||
}
|
||||
if isReflectNil(srcElement) {
|
||||
if overwrite || isReflectNil(dstElement) {
|
||||
dst.SetMapIndex(key, srcElement)
|
||||
}
|
||||
dstElement := dst.MapIndex(key)
|
||||
switch srcElement.Kind() {
|
||||
case reflect.Chan, reflect.Func, reflect.Map, reflect.Interface, reflect.Slice:
|
||||
if srcElement.IsNil() {
|
||||
continue
|
||||
}
|
||||
fallthrough
|
||||
default:
|
||||
if !srcElement.CanInterface() {
|
||||
continue
|
||||
}
|
||||
|
||||
if srcElement.CanInterface() {
|
||||
srcElement = reflect.ValueOf(srcElement.Interface())
|
||||
if dstElement.IsValid() {
|
||||
dstElement = reflect.ValueOf(dstElement.Interface())
|
||||
switch reflect.TypeOf(srcElement.Interface()).Kind() {
|
||||
case reflect.Struct:
|
||||
fallthrough
|
||||
case reflect.Ptr:
|
||||
fallthrough
|
||||
case reflect.Map:
|
||||
srcMapElm := srcElement
|
||||
dstMapElm := dstElement
|
||||
if srcMapElm.CanInterface() {
|
||||
srcMapElm = reflect.ValueOf(srcMapElm.Interface())
|
||||
if dstMapElm.IsValid() {
|
||||
dstMapElm = reflect.ValueOf(dstMapElm.Interface())
|
||||
}
|
||||
}
|
||||
dstElement, err = deepMerge(dstElement, srcElement, visited, depth+1, config)
|
||||
if err != nil {
|
||||
if err = deepMerge(dstMapElm, srcMapElm, visited, depth+1, config); err != nil {
|
||||
return
|
||||
}
|
||||
dst.SetMapIndex(key, dstElement)
|
||||
|
||||
}
|
||||
case reflect.Slice:
|
||||
newSlice := dst
|
||||
if (!isEmptyValue(src) || overwriteWithEmptySrc || overwriteSliceWithEmptySrc) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
|
||||
if typeCheck && src.Type() != dst.Type() {
|
||||
return dst, fmt.Errorf("cannot override two slices with different type (%s, %s)", src.Type(), dst.Type())
|
||||
srcSlice := reflect.ValueOf(srcElement.Interface())
|
||||
|
||||
var dstSlice reflect.Value
|
||||
if !dstElement.IsValid() || dstElement.IsNil() {
|
||||
dstSlice = reflect.MakeSlice(srcSlice.Type(), 0, srcSlice.Len())
|
||||
} else {
|
||||
dstSlice = reflect.ValueOf(dstElement.Interface())
|
||||
}
|
||||
newSlice = src
|
||||
|
||||
if !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
|
||||
dstSlice = srcSlice
|
||||
} else if config.AppendSlice {
|
||||
if typeCheck && src.Type() != dst.Type() {
|
||||
err = fmt.Errorf("cannot append two slice with different type (%s, %s)", src.Type(), dst.Type())
|
||||
return
|
||||
dstSlice = reflect.AppendSlice(dstSlice, srcSlice)
|
||||
}
|
||||
newSlice = reflect.AppendSlice(dst, src)
|
||||
dst.SetMapIndex(key, dstSlice)
|
||||
}
|
||||
if dst.CanSet() {
|
||||
dst.Set(newSlice)
|
||||
} else {
|
||||
dst = newSlice
|
||||
}
|
||||
case reflect.Ptr, reflect.Interface:
|
||||
if isReflectNil(src) {
|
||||
break
|
||||
if dstElement.IsValid() && reflect.TypeOf(srcElement.Interface()).Kind() == reflect.Map {
|
||||
continue
|
||||
}
|
||||
|
||||
if dst.Kind() != reflect.Ptr && src.Type().AssignableTo(dst.Type()) {
|
||||
if dst.IsNil() || overwrite {
|
||||
if overwrite || isEmptyValue(dst) {
|
||||
if dst.CanSet() {
|
||||
if srcElement.IsValid() && (overwrite || (!dstElement.IsValid() || isEmptyValue(dstElement))) {
|
||||
if dst.IsNil() {
|
||||
dst.Set(reflect.MakeMap(dst.Type()))
|
||||
}
|
||||
dst.SetMapIndex(key, srcElement)
|
||||
}
|
||||
}
|
||||
case reflect.Slice:
|
||||
if !dst.CanSet() {
|
||||
break
|
||||
}
|
||||
if !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) && !config.AppendSlice {
|
||||
dst.Set(src)
|
||||
} else {
|
||||
dst = src
|
||||
}
|
||||
}
|
||||
} else if config.AppendSlice {
|
||||
dst.Set(reflect.AppendSlice(dst, src))
|
||||
}
|
||||
case reflect.Ptr:
|
||||
fallthrough
|
||||
case reflect.Interface:
|
||||
if src.IsNil() {
|
||||
break
|
||||
}
|
||||
|
||||
if src.Kind() != reflect.Interface {
|
||||
if dst.IsNil() || (src.Kind() != reflect.Ptr && overwrite) {
|
||||
if dst.IsNil() || overwrite {
|
||||
if dst.CanSet() && (overwrite || isEmptyValue(dst)) {
|
||||
dst.Set(src)
|
||||
}
|
||||
} else if src.Kind() == reflect.Ptr {
|
||||
if dst, err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
|
||||
if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
|
||||
return
|
||||
}
|
||||
dst = dst.Addr()
|
||||
} else if dst.Elem().Type() == src.Type() {
|
||||
if dst, err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil {
|
||||
if err = deepMerge(dst.Elem(), src, visited, depth+1, config); err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
return dst, ErrDifferentArgumentsTypes
|
||||
return ErrDifferentArgumentsTypes
|
||||
}
|
||||
break
|
||||
}
|
||||
if dst.IsNil() || overwrite {
|
||||
if (overwrite || isEmptyValue(dst)) && (overwriteWithEmptySrc || !isEmptyValue(src)) {
|
||||
if dst.CanSet() {
|
||||
if dst.CanSet() && (overwrite || isEmptyValue(dst)) {
|
||||
dst.Set(src)
|
||||
} else {
|
||||
dst = src
|
||||
}
|
||||
}
|
||||
} else if _, err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
|
||||
} else if err = deepMerge(dst.Elem(), src.Elem(), visited, depth+1, config); err != nil {
|
||||
return
|
||||
}
|
||||
default:
|
||||
overwriteFull := (!isEmptyValue(src) || overwriteWithEmptySrc) && (overwrite || isEmptyValue(dst))
|
||||
if overwriteFull {
|
||||
if dst.CanSet() {
|
||||
if dst.CanSet() && !isEmptyValue(src) && (overwrite || isEmptyValue(dst)) {
|
||||
dst.Set(src)
|
||||
} else {
|
||||
dst = src
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -260,7 +199,7 @@ func Merge(dst, src interface{}, opts ...func(*Config)) error {
|
|||
return merge(dst, src, opts...)
|
||||
}
|
||||
|
||||
// MergeWithOverwrite will do the same as Merge except that non-empty dst attributes will be overridden by
|
||||
// MergeWithOverwrite will do the same as Merge except that non-empty dst attributes will be overriden by
|
||||
// non-empty src attribute values.
|
||||
// Deprecated: use Merge(…) with WithOverride
|
||||
func MergeWithOverwrite(dst, src interface{}, opts ...func(*Config)) error {
|
||||
|
|
@ -279,26 +218,11 @@ func WithOverride(config *Config) {
|
|||
config.Overwrite = true
|
||||
}
|
||||
|
||||
// WithOverwriteWithEmptyValue will make merge override non empty dst attributes with empty src attributes values.
|
||||
func WithOverwriteWithEmptyValue(config *Config) {
|
||||
config.overwriteWithEmptyValue = true
|
||||
}
|
||||
|
||||
// WithOverrideEmptySlice will make merge override empty dst slice with empty src slice.
|
||||
func WithOverrideEmptySlice(config *Config) {
|
||||
config.overwriteSliceWithEmptyValue = true
|
||||
}
|
||||
|
||||
// WithAppendSlice will make merge append slices instead of overwriting it.
|
||||
// WithAppendSlice will make merge append slices instead of overwriting it
|
||||
func WithAppendSlice(config *Config) {
|
||||
config.AppendSlice = true
|
||||
}
|
||||
|
||||
// WithTypeCheck will make merge check types while overwriting it (must be used with WithOverride).
|
||||
func WithTypeCheck(config *Config) {
|
||||
config.TypeCheck = true
|
||||
}
|
||||
|
||||
func merge(dst, src interface{}, opts ...func(*Config)) error {
|
||||
var (
|
||||
vDst, vSrc reflect.Value
|
||||
|
|
@ -314,25 +238,8 @@ func merge(dst, src interface{}, opts ...func(*Config)) error {
|
|||
if vDst, vSrc, err = resolveValues(dst, src); err != nil {
|
||||
return err
|
||||
}
|
||||
if !vDst.CanSet() {
|
||||
return fmt.Errorf("cannot set dst, needs reference")
|
||||
}
|
||||
if vDst.Type() != vSrc.Type() {
|
||||
return ErrDifferentArgumentsTypes
|
||||
}
|
||||
_, err = deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
|
||||
return err
|
||||
}
|
||||
|
||||
// IsReflectNil is the reflect value provided nil
|
||||
func isReflectNil(v reflect.Value) bool {
|
||||
k := v.Kind()
|
||||
switch k {
|
||||
case reflect.Interface, reflect.Slice, reflect.Chan, reflect.Func, reflect.Map, reflect.Ptr:
|
||||
// Both interface and slice are nil if first word is 0.
|
||||
// Both are always bigger than a word; assume flagIndir.
|
||||
return v.IsNil()
|
||||
default:
|
||||
return false
|
||||
}
|
||||
return deepMerge(vDst, vSrc, make(map[uintptr]*visit), 0, config)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
/test/*.dat*
|
||||
|
|
@ -1,202 +0,0 @@
|
|||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2018 InfluxData Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
# tdigest
|
||||
|
||||
This is an implementation of Ted Dunning's [t-digest](https://github.com/tdunning/t-digest/) in Go.
|
||||
|
||||
The implementation is based off [Derrick Burns' C++ implementation](https://github.com/derrickburns/tdigest).
|
||||
|
||||
## Example
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
|
||||
"github.com/influxdata/tdigest"
|
||||
)
|
||||
|
||||
func main() {
|
||||
td := tdigest.NewWithCompression(1000)
|
||||
for _, x := range []float64{1, 2, 3, 4, 5, 5, 4, 3, 2, 1} {
|
||||
td.Add(x, 1)
|
||||
}
|
||||
|
||||
// Compute Quantiles
|
||||
log.Println("50th", td.Quantile(0.5))
|
||||
log.Println("75th", td.Quantile(0.75))
|
||||
log.Println("90th", td.Quantile(0.9))
|
||||
log.Println("99th", td.Quantile(0.99))
|
||||
|
||||
// Compute CDFs
|
||||
log.Println("CDF(1) = ", td.CDF(1))
|
||||
log.Println("CDF(2) = ", td.CDF(2))
|
||||
log.Println("CDF(3) = ", td.CDF(3))
|
||||
log.Println("CDF(4) = ", td.CDF(4))
|
||||
log.Println("CDF(5) = ", td.CDF(5))
|
||||
}
|
||||
```
|
||||
|
||||
## TODO
|
||||
|
||||
Only the methods for a single TDigest have been implemented.
|
||||
The methods to merge two or more existing t-digests into a single t-digest have yet to be implemented.
|
||||
|
|
@ -1,60 +0,0 @@
|
|||
package tdigest
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// ErrWeightLessThanZero is used when the weight is not able to be processed.
|
||||
const ErrWeightLessThanZero = Error("centroid weight cannot be less than zero")
|
||||
|
||||
// Error is a domain error encountered while processing tdigests
|
||||
type Error string
|
||||
|
||||
func (e Error) Error() string {
|
||||
return string(e)
|
||||
}
|
||||
|
||||
// Centroid average position of all points in a shape
|
||||
type Centroid struct {
|
||||
Mean float64
|
||||
Weight float64
|
||||
}
|
||||
|
||||
func (c *Centroid) String() string {
|
||||
return fmt.Sprintf("{mean: %f weight: %f}", c.Mean, c.Weight)
|
||||
}
|
||||
|
||||
// Add averages the two centroids together and update this centroid
|
||||
func (c *Centroid) Add(r Centroid) error {
|
||||
if r.Weight < 0 {
|
||||
return ErrWeightLessThanZero
|
||||
}
|
||||
if c.Weight != 0 {
|
||||
c.Weight += r.Weight
|
||||
c.Mean += r.Weight * (r.Mean - c.Mean) / c.Weight
|
||||
} else {
|
||||
c.Weight = r.Weight
|
||||
c.Mean = r.Mean
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CentroidList is sorted by the Mean of the centroid, ascending.
|
||||
type CentroidList []Centroid
|
||||
|
||||
// Clear clears the list.
|
||||
func (l *CentroidList) Clear() {
|
||||
*l = (*l)[:0]
|
||||
}
|
||||
|
||||
func (l CentroidList) Len() int { return len(l) }
|
||||
func (l CentroidList) Less(i, j int) bool { return l[i].Mean < l[j].Mean }
|
||||
func (l CentroidList) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
|
||||
|
||||
// NewCentroidList creates a priority queue for the centroids
|
||||
func NewCentroidList(centroids []Centroid) CentroidList {
|
||||
l := CentroidList(centroids)
|
||||
sort.Sort(l)
|
||||
return l
|
||||
}
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
module github.com/influxdata/tdigest
|
||||
|
||||
require (
|
||||
github.com/google/go-cmp v0.2.0
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca
|
||||
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 // indirect
|
||||
)
|
||||
|
||||
go 1.13
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de h1:xSjD6HQTqT0H/k60N5yYBtnN1OEkVy7WIo/DYyxKRO0=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
|
||||
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 h1:4WsZyVtkthqrHTbDCJfiTs8IWNYE4uvsSDgaV6xpp+o=
|
||||
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
|
||||
|
|
@ -1,276 +0,0 @@
|
|||
package tdigest
|
||||
|
||||
import (
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// TDigest is a data structure for accurate on-line accumulation of
|
||||
// rank-based statistics such as quantiles and trimmed means.
|
||||
type TDigest struct {
|
||||
Compression float64
|
||||
|
||||
maxProcessed int
|
||||
maxUnprocessed int
|
||||
processed CentroidList
|
||||
unprocessed CentroidList
|
||||
cumulative []float64
|
||||
processedWeight float64
|
||||
unprocessedWeight float64
|
||||
min float64
|
||||
max float64
|
||||
}
|
||||
|
||||
// New initializes a new distribution with a default compression.
|
||||
func New() *TDigest {
|
||||
return NewWithCompression(1000)
|
||||
}
|
||||
|
||||
// NewWithCompression initializes a new distribution with custom compression.
|
||||
func NewWithCompression(c float64) *TDigest {
|
||||
t := &TDigest{
|
||||
Compression: c,
|
||||
}
|
||||
t.maxProcessed = processedSize(0, t.Compression)
|
||||
t.maxUnprocessed = unprocessedSize(0, t.Compression)
|
||||
t.processed = make(CentroidList, 0, t.maxProcessed)
|
||||
t.unprocessed = make(CentroidList, 0, t.maxUnprocessed+1)
|
||||
t.Reset()
|
||||
return t
|
||||
}
|
||||
|
||||
// Reset resets the distribution to its initial state.
|
||||
func (t *TDigest) Reset() {
|
||||
t.processed = t.processed[:0]
|
||||
t.unprocessed = t.unprocessed[:0]
|
||||
t.cumulative = t.cumulative[:0]
|
||||
t.processedWeight = 0
|
||||
t.unprocessedWeight = 0
|
||||
t.min = math.MaxFloat64
|
||||
t.max = -math.MaxFloat64
|
||||
}
|
||||
|
||||
// Add adds a value x with a weight w to the distribution.
|
||||
func (t *TDigest) Add(x, w float64) {
|
||||
if math.IsNaN(x) {
|
||||
return
|
||||
}
|
||||
t.AddCentroid(Centroid{Mean: x, Weight: w})
|
||||
}
|
||||
|
||||
// AddCentroidList can quickly add multiple centroids.
|
||||
func (t *TDigest) AddCentroidList(c CentroidList) {
|
||||
l := c.Len()
|
||||
for i := 0; i < l; i++ {
|
||||
diff := l - i
|
||||
room := t.maxUnprocessed - t.unprocessed.Len()
|
||||
mid := i + diff
|
||||
if room < diff {
|
||||
mid = i + room
|
||||
}
|
||||
for i < mid {
|
||||
t.AddCentroid(c[i])
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddCentroid adds a single centroid.
|
||||
func (t *TDigest) AddCentroid(c Centroid) {
|
||||
t.unprocessed = append(t.unprocessed, c)
|
||||
t.unprocessedWeight += c.Weight
|
||||
|
||||
if t.processed.Len() > t.maxProcessed ||
|
||||
t.unprocessed.Len() > t.maxUnprocessed {
|
||||
t.process()
|
||||
}
|
||||
}
|
||||
|
||||
func (t *TDigest) process() {
|
||||
if t.unprocessed.Len() > 0 ||
|
||||
t.processed.Len() > t.maxProcessed {
|
||||
|
||||
// Append all processed centroids to the unprocessed list and sort
|
||||
t.unprocessed = append(t.unprocessed, t.processed...)
|
||||
sort.Sort(&t.unprocessed)
|
||||
|
||||
// Reset processed list with first centroid
|
||||
t.processed.Clear()
|
||||
t.processed = append(t.processed, t.unprocessed[0])
|
||||
|
||||
t.processedWeight += t.unprocessedWeight
|
||||
t.unprocessedWeight = 0
|
||||
soFar := t.unprocessed[0].Weight
|
||||
limit := t.processedWeight * t.integratedQ(1.0)
|
||||
for _, centroid := range t.unprocessed[1:] {
|
||||
projected := soFar + centroid.Weight
|
||||
if projected <= limit {
|
||||
soFar = projected
|
||||
(&t.processed[t.processed.Len()-1]).Add(centroid)
|
||||
} else {
|
||||
k1 := t.integratedLocation(soFar / t.processedWeight)
|
||||
limit = t.processedWeight * t.integratedQ(k1+1.0)
|
||||
soFar += centroid.Weight
|
||||
t.processed = append(t.processed, centroid)
|
||||
}
|
||||
}
|
||||
t.min = math.Min(t.min, t.processed[0].Mean)
|
||||
t.max = math.Max(t.max, t.processed[t.processed.Len()-1].Mean)
|
||||
t.updateCumulative()
|
||||
t.unprocessed.Clear()
|
||||
}
|
||||
}
|
||||
|
||||
// Centroids returns a copy of processed centroids.
|
||||
// Useful when aggregating multiple t-digests.
|
||||
//
|
||||
// Pass in the CentroidList as the buffer to write into.
|
||||
func (t *TDigest) Centroids() CentroidList {
|
||||
t.process()
|
||||
cl := make([]Centroid, len(t.processed))
|
||||
copy(cl, t.processed)
|
||||
return cl
|
||||
}
|
||||
|
||||
func (t *TDigest) Count() float64 {
|
||||
t.process()
|
||||
count := 0.0
|
||||
for _, centroid := range t.processed {
|
||||
count += centroid.Weight
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (t *TDigest) updateCumulative() {
|
||||
if n := t.processed.Len() + 1; n <= cap(t.cumulative) {
|
||||
t.cumulative = t.cumulative[:n]
|
||||
} else {
|
||||
t.cumulative = make([]float64, n)
|
||||
}
|
||||
|
||||
prev := 0.0
|
||||
for i, centroid := range t.processed {
|
||||
cur := centroid.Weight
|
||||
t.cumulative[i] = prev + cur/2.0
|
||||
prev = prev + cur
|
||||
}
|
||||
t.cumulative[t.processed.Len()] = prev
|
||||
}
|
||||
|
||||
// Quantile returns the (approximate) quantile of
|
||||
// the distribution. Accepted values for q are between 0.0 and 1.0.
|
||||
// Returns NaN if Count is zero or bad inputs.
|
||||
func (t *TDigest) Quantile(q float64) float64 {
|
||||
t.process()
|
||||
if q < 0 || q > 1 || t.processed.Len() == 0 {
|
||||
return math.NaN()
|
||||
}
|
||||
if t.processed.Len() == 1 {
|
||||
return t.processed[0].Mean
|
||||
}
|
||||
index := q * t.processedWeight
|
||||
if index <= t.processed[0].Weight/2.0 {
|
||||
return t.min + 2.0*index/t.processed[0].Weight*(t.processed[0].Mean-t.min)
|
||||
}
|
||||
|
||||
lower := sort.Search(len(t.cumulative), func(i int) bool {
|
||||
return t.cumulative[i] >= index
|
||||
})
|
||||
|
||||
if lower+1 != len(t.cumulative) {
|
||||
z1 := index - t.cumulative[lower-1]
|
||||
z2 := t.cumulative[lower] - index
|
||||
return weightedAverage(t.processed[lower-1].Mean, z2, t.processed[lower].Mean, z1)
|
||||
}
|
||||
|
||||
z1 := index - t.processedWeight - t.processed[lower-1].Weight/2.0
|
||||
z2 := (t.processed[lower-1].Weight / 2.0) - z1
|
||||
return weightedAverage(t.processed[t.processed.Len()-1].Mean, z1, t.max, z2)
|
||||
}
|
||||
|
||||
// CDF returns the cumulative distribution function for a given value x.
|
||||
func (t *TDigest) CDF(x float64) float64 {
|
||||
t.process()
|
||||
switch t.processed.Len() {
|
||||
case 0:
|
||||
return 0.0
|
||||
case 1:
|
||||
width := t.max - t.min
|
||||
if x <= t.min {
|
||||
return 0.0
|
||||
}
|
||||
if x >= t.max {
|
||||
return 1.0
|
||||
}
|
||||
if (x - t.min) <= width {
|
||||
// min and max are too close together to do any viable interpolation
|
||||
return 0.5
|
||||
}
|
||||
return (x - t.min) / width
|
||||
}
|
||||
|
||||
if x <= t.min {
|
||||
return 0.0
|
||||
}
|
||||
if x >= t.max {
|
||||
return 1.0
|
||||
}
|
||||
m0 := t.processed[0].Mean
|
||||
// Left Tail
|
||||
if x <= m0 {
|
||||
if m0-t.min > 0 {
|
||||
return (x - t.min) / (m0 - t.min) * t.processed[0].Weight / t.processedWeight / 2.0
|
||||
}
|
||||
return 0.0
|
||||
}
|
||||
// Right Tail
|
||||
mn := t.processed[t.processed.Len()-1].Mean
|
||||
if x >= mn {
|
||||
if t.max-mn > 0.0 {
|
||||
return 1.0 - (t.max-x)/(t.max-mn)*t.processed[t.processed.Len()-1].Weight/t.processedWeight/2.0
|
||||
}
|
||||
return 1.0
|
||||
}
|
||||
|
||||
upper := sort.Search(t.processed.Len(), func(i int) bool {
|
||||
return t.processed[i].Mean > x
|
||||
})
|
||||
|
||||
z1 := x - t.processed[upper-1].Mean
|
||||
z2 := t.processed[upper].Mean - x
|
||||
return weightedAverage(t.cumulative[upper-1], z2, t.cumulative[upper], z1) / t.processedWeight
|
||||
}
|
||||
|
||||
func (t *TDigest) integratedQ(k float64) float64 {
|
||||
return (math.Sin(math.Min(k, t.Compression)*math.Pi/t.Compression-math.Pi/2.0) + 1.0) / 2.0
|
||||
}
|
||||
|
||||
func (t *TDigest) integratedLocation(q float64) float64 {
|
||||
return t.Compression * (math.Asin(2.0*q-1.0) + math.Pi/2.0) / math.Pi
|
||||
}
|
||||
|
||||
func weightedAverage(x1, w1, x2, w2 float64) float64 {
|
||||
if x1 <= x2 {
|
||||
return weightedAverageSorted(x1, w1, x2, w2)
|
||||
}
|
||||
return weightedAverageSorted(x2, w2, x1, w1)
|
||||
}
|
||||
|
||||
func weightedAverageSorted(x1, w1, x2, w2 float64) float64 {
|
||||
x := (x1*w1 + x2*w2) / (w1 + w2)
|
||||
return math.Max(x1, math.Min(x, x2))
|
||||
}
|
||||
|
||||
func processedSize(size int, compression float64) int {
|
||||
if size == 0 {
|
||||
return int(2 * math.Ceil(compression))
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func unprocessedSize(size int, compression float64) int {
|
||||
if size == 0 {
|
||||
return int(8 * math.Ceil(compression))
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
Docs: https://godoc.org/github.com/josharian/intern
|
||||
|
||||
See also [Go issue 5160](https://golang.org/issue/5160).
|
||||
|
||||
License: MIT
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
module github.com/josharian/intern
|
||||
|
||||
go 1.5
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
// Package intern interns strings.
|
||||
// Interning is best effort only.
|
||||
// Interned strings may be removed automatically
|
||||
// at any time without notification.
|
||||
// All functions may be called concurrently
|
||||
// with themselves and each other.
|
||||
package intern
|
||||
|
||||
import "sync"
|
||||
|
||||
var (
|
||||
pool sync.Pool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
return make(map[string]string)
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
// String returns s, interned.
|
||||
func String(s string) string {
|
||||
m := pool.Get().(map[string]string)
|
||||
c, ok := m[s]
|
||||
if ok {
|
||||
pool.Put(m)
|
||||
return c
|
||||
}
|
||||
m[s] = s
|
||||
pool.Put(m)
|
||||
return s
|
||||
}
|
||||
|
||||
// Bytes returns b converted to a string, interned.
|
||||
func Bytes(b []byte) string {
|
||||
m := pool.Get().(map[string]string)
|
||||
c, ok := m[string(b)]
|
||||
if ok {
|
||||
pool.Put(m)
|
||||
return c
|
||||
}
|
||||
s := string(b)
|
||||
m[s] = s
|
||||
pool.Put(m)
|
||||
return s
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2019 Josh Bleecher Snyder
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -4,7 +4,6 @@ package buffer
|
|||
|
||||
import (
|
||||
"io"
|
||||
"net"
|
||||
"sync"
|
||||
)
|
||||
|
||||
|
|
@ -53,14 +52,16 @@ func putBuf(buf []byte) {
|
|||
|
||||
// getBuf gets a chunk from reuse pool or creates a new one if reuse failed.
|
||||
func getBuf(size int) []byte {
|
||||
if size >= config.PooledSize {
|
||||
if size < config.PooledSize {
|
||||
return make([]byte, 0, size)
|
||||
}
|
||||
|
||||
if c := buffers[size]; c != nil {
|
||||
v := c.Get()
|
||||
if v != nil {
|
||||
return v.([]byte)
|
||||
}
|
||||
}
|
||||
}
|
||||
return make([]byte, 0, size)
|
||||
}
|
||||
|
||||
|
|
@ -77,12 +78,9 @@ type Buffer struct {
|
|||
// EnsureSpace makes sure that the current chunk contains at least s free bytes,
|
||||
// possibly creating a new chunk.
|
||||
func (b *Buffer) EnsureSpace(s int) {
|
||||
if cap(b.Buf)-len(b.Buf) < s {
|
||||
b.ensureSpaceSlow(s)
|
||||
if cap(b.Buf)-len(b.Buf) >= s {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Buffer) ensureSpaceSlow(s int) {
|
||||
l := len(b.Buf)
|
||||
if l > 0 {
|
||||
if cap(b.toPool) != cap(b.Buf) {
|
||||
|
|
@ -107,22 +105,18 @@ func (b *Buffer) ensureSpaceSlow(s int) {
|
|||
|
||||
// AppendByte appends a single byte to buffer.
|
||||
func (b *Buffer) AppendByte(data byte) {
|
||||
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
|
||||
b.EnsureSpace(1)
|
||||
}
|
||||
b.Buf = append(b.Buf, data)
|
||||
}
|
||||
|
||||
// AppendBytes appends a byte slice to buffer.
|
||||
func (b *Buffer) AppendBytes(data []byte) {
|
||||
if len(data) <= cap(b.Buf)-len(b.Buf) {
|
||||
b.Buf = append(b.Buf, data...) // fast path
|
||||
} else {
|
||||
b.appendBytesSlow(data)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Buffer) appendBytesSlow(data []byte) {
|
||||
for len(data) > 0 {
|
||||
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
|
||||
b.EnsureSpace(1)
|
||||
}
|
||||
|
||||
sz := cap(b.Buf) - len(b.Buf)
|
||||
if sz > len(data) {
|
||||
|
|
@ -134,18 +128,12 @@ func (b *Buffer) appendBytesSlow(data []byte) {
|
|||
}
|
||||
}
|
||||
|
||||
// AppendString appends a string to buffer.
|
||||
// AppendBytes appends a string to buffer.
|
||||
func (b *Buffer) AppendString(data string) {
|
||||
if len(data) <= cap(b.Buf)-len(b.Buf) {
|
||||
b.Buf = append(b.Buf, data...) // fast path
|
||||
} else {
|
||||
b.appendStringSlow(data)
|
||||
}
|
||||
}
|
||||
|
||||
func (b *Buffer) appendStringSlow(data string) {
|
||||
for len(data) > 0 {
|
||||
if cap(b.Buf) == len(b.Buf) { // EnsureSpace won't be inlined.
|
||||
b.EnsureSpace(1)
|
||||
}
|
||||
|
||||
sz := cap(b.Buf) - len(b.Buf)
|
||||
if sz > len(data) {
|
||||
|
|
@ -168,22 +156,26 @@ func (b *Buffer) Size() int {
|
|||
|
||||
// DumpTo outputs the contents of a buffer to a writer and resets the buffer.
|
||||
func (b *Buffer) DumpTo(w io.Writer) (written int, err error) {
|
||||
bufs := net.Buffers(b.bufs)
|
||||
if len(b.Buf) > 0 {
|
||||
bufs = append(bufs, b.Buf)
|
||||
}
|
||||
n, err := bufs.WriteTo(w)
|
||||
|
||||
var n int
|
||||
for _, buf := range b.bufs {
|
||||
if err == nil {
|
||||
n, err = w.Write(buf)
|
||||
written += n
|
||||
}
|
||||
putBuf(buf)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
n, err = w.Write(b.Buf)
|
||||
written += n
|
||||
}
|
||||
putBuf(b.toPool)
|
||||
|
||||
b.bufs = nil
|
||||
b.Buf = nil
|
||||
b.toPool = nil
|
||||
|
||||
return int(n), err
|
||||
return
|
||||
}
|
||||
|
||||
// BuildBytes creates a single byte slice with all the contents of the buffer. Data is
|
||||
|
|
@ -200,7 +192,7 @@ func (b *Buffer) BuildBytes(reuse ...[]byte) []byte {
|
|||
var ret []byte
|
||||
size := b.Size()
|
||||
|
||||
// If we got a buffer as argument and it is big enough, reuse it.
|
||||
// If we got a buffer as argument and it is big enought, reuse it.
|
||||
if len(reuse) == 1 && cap(reuse[0]) >= size {
|
||||
ret = reuse[0][:0]
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
package jlexer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
|
@ -15,8 +14,6 @@ import (
|
|||
"unicode"
|
||||
"unicode/utf16"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/josharian/intern"
|
||||
)
|
||||
|
||||
// tokenKind determines type of a token.
|
||||
|
|
@ -36,7 +33,6 @@ type token struct {
|
|||
kind tokenKind // Type of a token.
|
||||
|
||||
boolValue bool // Value if a boolean literal token.
|
||||
byteValueCloned bool // true if byteValue was allocated and does not refer to original json body
|
||||
byteValue []byte // Raw value of a token.
|
||||
delimValue byte
|
||||
}
|
||||
|
|
@ -244,65 +240,23 @@ func (r *Lexer) fetchNumber() {
|
|||
|
||||
// findStringLen tries to scan into the string literal for ending quote char to determine required size.
|
||||
// The size will be exact if no escapes are present and may be inexact if there are escaped chars.
|
||||
func findStringLen(data []byte) (isValid bool, length int) {
|
||||
for {
|
||||
idx := bytes.IndexByte(data, '"')
|
||||
if idx == -1 {
|
||||
return false, len(data)
|
||||
}
|
||||
if idx == 0 || (idx > 0 && data[idx-1] != '\\') {
|
||||
return true, length + idx
|
||||
}
|
||||
func findStringLen(data []byte) (isValid, hasEscapes bool, length int) {
|
||||
delta := 0
|
||||
|
||||
// count \\\\\\\ sequences. even number of slashes means quote is not really escaped
|
||||
cnt := 1
|
||||
for idx-cnt-1 >= 0 && data[idx-cnt-1] == '\\' {
|
||||
cnt++
|
||||
for i := 0; i < len(data); i++ {
|
||||
switch data[i] {
|
||||
case '\\':
|
||||
i++
|
||||
delta++
|
||||
if i < len(data) && data[i] == 'u' {
|
||||
delta++
|
||||
}
|
||||
if cnt%2 == 0 {
|
||||
return true, length + idx
|
||||
}
|
||||
|
||||
length += idx + 1
|
||||
data = data[idx+1:]
|
||||
case '"':
|
||||
return true, (delta > 0), (i - delta)
|
||||
}
|
||||
}
|
||||
|
||||
// unescapeStringToken performs unescaping of string token.
|
||||
// if no escaping is needed, original string is returned, otherwise - a new one allocated
|
||||
func (r *Lexer) unescapeStringToken() (err error) {
|
||||
data := r.token.byteValue
|
||||
var unescapedData []byte
|
||||
|
||||
for {
|
||||
i := bytes.IndexByte(data, '\\')
|
||||
if i == -1 {
|
||||
break
|
||||
}
|
||||
|
||||
escapedRune, escapedBytes, err := decodeEscape(data[i:])
|
||||
if err != nil {
|
||||
r.errParse(err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
if unescapedData == nil {
|
||||
unescapedData = make([]byte, 0, len(r.token.byteValue))
|
||||
}
|
||||
|
||||
var d [4]byte
|
||||
s := utf8.EncodeRune(d[:], escapedRune)
|
||||
unescapedData = append(unescapedData, data[:i]...)
|
||||
unescapedData = append(unescapedData, d[:s]...)
|
||||
|
||||
data = data[i+escapedBytes:]
|
||||
}
|
||||
|
||||
if unescapedData != nil {
|
||||
r.token.byteValue = append(unescapedData, data...)
|
||||
r.token.byteValueCloned = true
|
||||
}
|
||||
return
|
||||
return false, false, len(data)
|
||||
}
|
||||
|
||||
// getu4 decodes \uXXXX from the beginning of s, returning the hex value,
|
||||
|
|
@ -332,30 +286,36 @@ func getu4(s []byte) rune {
|
|||
return val
|
||||
}
|
||||
|
||||
// decodeEscape processes a single escape sequence and returns number of bytes processed.
|
||||
func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
|
||||
// processEscape processes a single escape sequence and returns number of bytes processed.
|
||||
func (r *Lexer) processEscape(data []byte) (int, error) {
|
||||
if len(data) < 2 {
|
||||
return 0, 0, errors.New("incorrect escape symbol \\ at the end of token")
|
||||
return 0, fmt.Errorf("syntax error at %v", string(data))
|
||||
}
|
||||
|
||||
c := data[1]
|
||||
switch c {
|
||||
case '"', '/', '\\':
|
||||
return rune(c), 2, nil
|
||||
r.token.byteValue = append(r.token.byteValue, c)
|
||||
return 2, nil
|
||||
case 'b':
|
||||
return '\b', 2, nil
|
||||
r.token.byteValue = append(r.token.byteValue, '\b')
|
||||
return 2, nil
|
||||
case 'f':
|
||||
return '\f', 2, nil
|
||||
r.token.byteValue = append(r.token.byteValue, '\f')
|
||||
return 2, nil
|
||||
case 'n':
|
||||
return '\n', 2, nil
|
||||
r.token.byteValue = append(r.token.byteValue, '\n')
|
||||
return 2, nil
|
||||
case 'r':
|
||||
return '\r', 2, nil
|
||||
r.token.byteValue = append(r.token.byteValue, '\r')
|
||||
return 2, nil
|
||||
case 't':
|
||||
return '\t', 2, nil
|
||||
r.token.byteValue = append(r.token.byteValue, '\t')
|
||||
return 2, nil
|
||||
case 'u':
|
||||
rr := getu4(data)
|
||||
if rr < 0 {
|
||||
return 0, 0, errors.New("incorrectly escaped \\uXXXX sequence")
|
||||
return 0, errors.New("syntax error")
|
||||
}
|
||||
|
||||
read := 6
|
||||
|
|
@ -368,10 +328,13 @@ func decodeEscape(data []byte) (decoded rune, bytesProcessed int, err error) {
|
|||
rr = unicode.ReplacementChar
|
||||
}
|
||||
}
|
||||
return rr, read, nil
|
||||
var d [4]byte
|
||||
s := utf8.EncodeRune(d[:], rr)
|
||||
r.token.byteValue = append(r.token.byteValue, d[:s]...)
|
||||
return read, nil
|
||||
}
|
||||
|
||||
return 0, 0, errors.New("incorrectly escaped bytes")
|
||||
return 0, errors.New("syntax error")
|
||||
}
|
||||
|
||||
// fetchString scans a string literal token.
|
||||
|
|
@ -379,14 +342,43 @@ func (r *Lexer) fetchString() {
|
|||
r.pos++
|
||||
data := r.Data[r.pos:]
|
||||
|
||||
isValid, length := findStringLen(data)
|
||||
isValid, hasEscapes, length := findStringLen(data)
|
||||
if !isValid {
|
||||
r.pos += length
|
||||
r.errParse("unterminated string literal")
|
||||
return
|
||||
}
|
||||
if !hasEscapes {
|
||||
r.token.byteValue = data[:length]
|
||||
r.pos += length + 1 // skip closing '"' as well
|
||||
r.pos += length + 1
|
||||
return
|
||||
}
|
||||
|
||||
r.token.byteValue = make([]byte, 0, length)
|
||||
p := 0
|
||||
for i := 0; i < len(data); {
|
||||
switch data[i] {
|
||||
case '"':
|
||||
r.pos += i + 1
|
||||
r.token.byteValue = append(r.token.byteValue, data[p:i]...)
|
||||
i++
|
||||
return
|
||||
|
||||
case '\\':
|
||||
r.token.byteValue = append(r.token.byteValue, data[p:i]...)
|
||||
off, err := r.processEscape(data[i:])
|
||||
if err != nil {
|
||||
r.errParse(err.Error())
|
||||
return
|
||||
}
|
||||
i += off
|
||||
p = i
|
||||
|
||||
default:
|
||||
i++
|
||||
}
|
||||
}
|
||||
r.errParse("unterminated string literal")
|
||||
}
|
||||
|
||||
// scanToken scans the next token if no token is currently available in the lexer.
|
||||
|
|
@ -401,7 +393,6 @@ func (r *Lexer) scanToken() {
|
|||
// consume resets the current token to allow scanning the next one.
|
||||
func (r *Lexer) consume() {
|
||||
r.token.kind = tokenUndef
|
||||
r.token.byteValueCloned = false
|
||||
r.token.delimValue = 0
|
||||
}
|
||||
|
||||
|
|
@ -529,7 +520,6 @@ func (r *Lexer) Skip() {
|
|||
func (r *Lexer) SkipRecursive() {
|
||||
r.scanToken()
|
||||
var start, end byte
|
||||
startPos := r.start
|
||||
|
||||
switch r.token.delimValue {
|
||||
case '{':
|
||||
|
|
@ -555,14 +545,6 @@ func (r *Lexer) SkipRecursive() {
|
|||
level--
|
||||
if level == 0 {
|
||||
r.pos += i + 1
|
||||
if !json.Valid(r.Data[startPos:r.pos]) {
|
||||
r.pos = len(r.Data)
|
||||
r.fatalError = &LexerError{
|
||||
Reason: "skipped array/object json value is invalid",
|
||||
Offset: r.pos,
|
||||
Data: string(r.Data[r.pos:]),
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
case c == '\\' && inQuotes:
|
||||
|
|
@ -620,7 +602,7 @@ func (r *Lexer) Consumed() {
|
|||
}
|
||||
}
|
||||
|
||||
func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) {
|
||||
func (r *Lexer) unsafeString() (string, []byte) {
|
||||
if r.token.kind == tokenUndef && r.Ok() {
|
||||
r.FetchToken()
|
||||
}
|
||||
|
|
@ -628,13 +610,6 @@ func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) {
|
|||
r.errInvalidToken("string")
|
||||
return "", nil
|
||||
}
|
||||
if !skipUnescape {
|
||||
if err := r.unescapeStringToken(); err != nil {
|
||||
r.errInvalidToken("string")
|
||||
return "", nil
|
||||
}
|
||||
}
|
||||
|
||||
bytes := r.token.byteValue
|
||||
ret := bytesToStr(r.token.byteValue)
|
||||
r.consume()
|
||||
|
|
@ -646,19 +621,13 @@ func (r *Lexer) unsafeString(skipUnescape bool) (string, []byte) {
|
|||
// Warning: returned string may point to the input buffer, so the string should not outlive
|
||||
// the input buffer. Intended pattern of usage is as an argument to a switch statement.
|
||||
func (r *Lexer) UnsafeString() string {
|
||||
ret, _ := r.unsafeString(false)
|
||||
ret, _ := r.unsafeString()
|
||||
return ret
|
||||
}
|
||||
|
||||
// UnsafeBytes returns the byte slice if the token is a string literal.
|
||||
func (r *Lexer) UnsafeBytes() []byte {
|
||||
_, ret := r.unsafeString(false)
|
||||
return ret
|
||||
}
|
||||
|
||||
// UnsafeFieldName returns current member name string token
|
||||
func (r *Lexer) UnsafeFieldName(skipUnescape bool) string {
|
||||
ret, _ := r.unsafeString(skipUnescape)
|
||||
_, ret := r.unsafeString()
|
||||
return ret
|
||||
}
|
||||
|
||||
|
|
@ -671,34 +640,7 @@ func (r *Lexer) String() string {
|
|||
r.errInvalidToken("string")
|
||||
return ""
|
||||
}
|
||||
if err := r.unescapeStringToken(); err != nil {
|
||||
r.errInvalidToken("string")
|
||||
return ""
|
||||
}
|
||||
var ret string
|
||||
if r.token.byteValueCloned {
|
||||
ret = bytesToStr(r.token.byteValue)
|
||||
} else {
|
||||
ret = string(r.token.byteValue)
|
||||
}
|
||||
r.consume()
|
||||
return ret
|
||||
}
|
||||
|
||||
// StringIntern reads a string literal, and performs string interning on it.
|
||||
func (r *Lexer) StringIntern() string {
|
||||
if r.token.kind == tokenUndef && r.Ok() {
|
||||
r.FetchToken()
|
||||
}
|
||||
if !r.Ok() || r.token.kind != tokenString {
|
||||
r.errInvalidToken("string")
|
||||
return ""
|
||||
}
|
||||
if err := r.unescapeStringToken(); err != nil {
|
||||
r.errInvalidToken("string")
|
||||
return ""
|
||||
}
|
||||
ret := intern.Bytes(r.token.byteValue)
|
||||
ret := string(r.token.byteValue)
|
||||
r.consume()
|
||||
return ret
|
||||
}
|
||||
|
|
@ -712,10 +654,6 @@ func (r *Lexer) Bytes() []byte {
|
|||
r.errInvalidToken("string")
|
||||
return nil
|
||||
}
|
||||
if err := r.unescapeStringToken(); err != nil {
|
||||
r.errInvalidToken("string")
|
||||
return nil
|
||||
}
|
||||
ret := make([]byte, base64.StdEncoding.DecodedLen(len(r.token.byteValue)))
|
||||
n, err := base64.StdEncoding.Decode(ret, r.token.byteValue)
|
||||
if err != nil {
|
||||
|
|
@ -901,7 +839,7 @@ func (r *Lexer) Int() int {
|
|||
}
|
||||
|
||||
func (r *Lexer) Uint8Str() uint8 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -918,7 +856,7 @@ func (r *Lexer) Uint8Str() uint8 {
|
|||
}
|
||||
|
||||
func (r *Lexer) Uint16Str() uint16 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -935,7 +873,7 @@ func (r *Lexer) Uint16Str() uint16 {
|
|||
}
|
||||
|
||||
func (r *Lexer) Uint32Str() uint32 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -952,7 +890,7 @@ func (r *Lexer) Uint32Str() uint32 {
|
|||
}
|
||||
|
||||
func (r *Lexer) Uint64Str() uint64 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -977,7 +915,7 @@ func (r *Lexer) UintptrStr() uintptr {
|
|||
}
|
||||
|
||||
func (r *Lexer) Int8Str() int8 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -994,7 +932,7 @@ func (r *Lexer) Int8Str() int8 {
|
|||
}
|
||||
|
||||
func (r *Lexer) Int16Str() int16 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -1011,7 +949,7 @@ func (r *Lexer) Int16Str() int16 {
|
|||
}
|
||||
|
||||
func (r *Lexer) Int32Str() int32 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -1028,7 +966,7 @@ func (r *Lexer) Int32Str() int32 {
|
|||
}
|
||||
|
||||
func (r *Lexer) Int64Str() int64 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -1066,7 +1004,7 @@ func (r *Lexer) Float32() float32 {
|
|||
}
|
||||
|
||||
func (r *Lexer) Float32Str() float32 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
@ -1099,7 +1037,7 @@ func (r *Lexer) Float64() float64 {
|
|||
}
|
||||
|
||||
func (r *Lexer) Float64Str() float64 {
|
||||
s, b := r.unsafeString(false)
|
||||
s, b := r.unsafeString()
|
||||
if !r.Ok() {
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -270,25 +270,16 @@ func (w *Writer) Bool(v bool) {
|
|||
|
||||
const chars = "0123456789abcdef"
|
||||
|
||||
func getTable(falseValues ...int) [128]bool {
|
||||
table := [128]bool{}
|
||||
|
||||
for i := 0; i < 128; i++ {
|
||||
table[i] = true
|
||||
func isNotEscapedSingleChar(c byte, escapeHTML bool) bool {
|
||||
// Note: might make sense to use a table if there are more chars to escape. With 4 chars
|
||||
// it benchmarks the same.
|
||||
if escapeHTML {
|
||||
return c != '<' && c != '>' && c != '&' && c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf
|
||||
} else {
|
||||
return c != '\\' && c != '"' && c >= 0x20 && c < utf8.RuneSelf
|
||||
}
|
||||
|
||||
for _, v := range falseValues {
|
||||
table[v] = false
|
||||
}
|
||||
|
||||
return table
|
||||
}
|
||||
|
||||
var (
|
||||
htmlEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '&', '<', '>', '\\')
|
||||
htmlNoEscapeTable = getTable(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, '"', '\\')
|
||||
)
|
||||
|
||||
func (w *Writer) String(s string) {
|
||||
w.Buffer.AppendByte('"')
|
||||
|
||||
|
|
@ -297,21 +288,15 @@ func (w *Writer) String(s string) {
|
|||
|
||||
p := 0 // last non-escape symbol
|
||||
|
||||
escapeTable := &htmlEscapeTable
|
||||
if w.NoEscapeHTML {
|
||||
escapeTable = &htmlNoEscapeTable
|
||||
}
|
||||
|
||||
for i := 0; i < len(s); {
|
||||
c := s[i]
|
||||
|
||||
if c < utf8.RuneSelf {
|
||||
if escapeTable[c] {
|
||||
if isNotEscapedSingleChar(c, !w.NoEscapeHTML) {
|
||||
// single-width character, no escaping is required
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
} else if c < utf8.RuneSelf {
|
||||
// single-with character, need to escape
|
||||
w.Buffer.AppendString(s[p:i])
|
||||
switch c {
|
||||
case '\t':
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
# This is the official list of go-diff authors for copyright purposes.
|
||||
# This file is distinct from the CONTRIBUTORS files.
|
||||
# See the latter for an explanation.
|
||||
|
||||
# Names should be added to this file as
|
||||
# Name or Organization <email address>
|
||||
# The email address is not required for organizations.
|
||||
|
||||
# Please keep the list sorted.
|
||||
|
||||
Danny Yoo <dannyyoo@google.com>
|
||||
James Kolb <jkolb@google.com>
|
||||
Jonathan Amsterdam <jba@google.com>
|
||||
Markus Zimmermann <markus.zimmermann@nethead.at> <markus.zimmermann@symflower.com> <zimmski@gmail.com>
|
||||
Matt Kovars <akaskik@gmail.com>
|
||||
Örjan Persson <orjan@spotify.com>
|
||||
Osman Masood <oamasood@gmail.com>
|
||||
Robert Carlsen <rwcarlsen@gmail.com>
|
||||
Rory Flynn <roryflynn@users.noreply.github.com>
|
||||
Sergi Mansilla <sergi.mansilla@gmail.com>
|
||||
Shatrugna Sadhu <ssadhu@apcera.com>
|
||||
Shawn Smith <shawnpsmith@gmail.com>
|
||||
Stas Maksimov <maksimov@gmail.com>
|
||||
Tor Arvid Lund <torarvid@gmail.com>
|
||||
Zac Bergquist <zbergquist99@gmail.com>
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
# This is the official list of people who can contribute
|
||||
# (and typically have contributed) code to the go-diff
|
||||
# repository.
|
||||
#
|
||||
# The AUTHORS file lists the copyright holders; this file
|
||||
# lists people. For example, ACME Inc. employees would be listed here
|
||||
# but not in AUTHORS, because ACME Inc. would hold the copyright.
|
||||
#
|
||||
# When adding J Random Contributor's name to this file,
|
||||
# either J's name or J's organization's name should be
|
||||
# added to the AUTHORS file.
|
||||
#
|
||||
# Names should be added to this file like so:
|
||||
# Name <email address>
|
||||
#
|
||||
# Please keep the list sorted.
|
||||
|
||||
Danny Yoo <dannyyoo@google.com>
|
||||
James Kolb <jkolb@google.com>
|
||||
Jonathan Amsterdam <jba@google.com>
|
||||
Markus Zimmermann <markus.zimmermann@nethead.at> <markus.zimmermann@symflower.com> <zimmski@gmail.com>
|
||||
Matt Kovars <akaskik@gmail.com>
|
||||
Örjan Persson <orjan@spotify.com>
|
||||
Osman Masood <oamasood@gmail.com>
|
||||
Robert Carlsen <rwcarlsen@gmail.com>
|
||||
Rory Flynn <roryflynn@users.noreply.github.com>
|
||||
Sergi Mansilla <sergi.mansilla@gmail.com>
|
||||
Shatrugna Sadhu <ssadhu@apcera.com>
|
||||
Shawn Smith <shawnpsmith@gmail.com>
|
||||
Stas Maksimov <maksimov@gmail.com>
|
||||
Tor Arvid Lund <torarvid@gmail.com>
|
||||
Zac Bergquist <zbergquist99@gmail.com>
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
Copyright (c) 2012-2016 The go-diff Authors. All rights reserved.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of this software and associated documentation files (the "Software"),
|
||||
to deal in the Software without restriction, including without limitation
|
||||
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
and/or sell copies of the Software, and to permit persons to whom the
|
||||
Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included
|
||||
in all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
||||
DEALINGS IN THE SOFTWARE.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -1,46 +0,0 @@
|
|||
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
|
||||
// https://github.com/sergi/go-diff
|
||||
// See the included LICENSE file for license details.
|
||||
//
|
||||
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
|
||||
// Original library is Copyright (c) 2006 Google Inc.
|
||||
// http://code.google.com/p/google-diff-match-patch/
|
||||
|
||||
// Package diffmatchpatch offers robust algorithms to perform the operations required for synchronizing plain text.
|
||||
package diffmatchpatch
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// DiffMatchPatch holds the configuration for diff-match-patch operations.
|
||||
type DiffMatchPatch struct {
|
||||
// Number of seconds to map a diff before giving up (0 for infinity).
|
||||
DiffTimeout time.Duration
|
||||
// Cost of an empty edit operation in terms of edit characters.
|
||||
DiffEditCost int
|
||||
// How far to search for a match (0 = exact location, 1000+ = broad match). A match this many characters away from the expected location will add 1.0 to the score (0.0 is a perfect match).
|
||||
MatchDistance int
|
||||
// When deleting a large block of text (over ~64 characters), how close do the contents have to be to match the expected contents. (0.0 = perfection, 1.0 = very loose). Note that MatchThreshold controls how closely the end points of a delete need to match.
|
||||
PatchDeleteThreshold float64
|
||||
// Chunk size for context length.
|
||||
PatchMargin int
|
||||
// The number of bits in an int.
|
||||
MatchMaxBits int
|
||||
// At what point is no match declared (0.0 = perfection, 1.0 = very loose).
|
||||
MatchThreshold float64
|
||||
}
|
||||
|
||||
// New creates a new DiffMatchPatch object with default parameters.
|
||||
func New() *DiffMatchPatch {
|
||||
// Defaults.
|
||||
return &DiffMatchPatch{
|
||||
DiffTimeout: time.Second,
|
||||
DiffEditCost: 4,
|
||||
MatchThreshold: 0.5,
|
||||
MatchDistance: 1000,
|
||||
PatchDeleteThreshold: 0.5,
|
||||
PatchMargin: 4,
|
||||
MatchMaxBits: 32,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,160 +0,0 @@
|
|||
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
|
||||
// https://github.com/sergi/go-diff
|
||||
// See the included LICENSE file for license details.
|
||||
//
|
||||
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
|
||||
// Original library is Copyright (c) 2006 Google Inc.
|
||||
// http://code.google.com/p/google-diff-match-patch/
|
||||
|
||||
package diffmatchpatch
|
||||
|
||||
import (
|
||||
"math"
|
||||
)
|
||||
|
||||
// MatchMain locates the best instance of 'pattern' in 'text' near 'loc'.
|
||||
// Returns -1 if no match found.
|
||||
func (dmp *DiffMatchPatch) MatchMain(text, pattern string, loc int) int {
|
||||
// Check for null inputs not needed since null can't be passed in C#.
|
||||
|
||||
loc = int(math.Max(0, math.Min(float64(loc), float64(len(text)))))
|
||||
if text == pattern {
|
||||
// Shortcut (potentially not guaranteed by the algorithm)
|
||||
return 0
|
||||
} else if len(text) == 0 {
|
||||
// Nothing to match.
|
||||
return -1
|
||||
} else if loc+len(pattern) <= len(text) && text[loc:loc+len(pattern)] == pattern {
|
||||
// Perfect match at the perfect spot! (Includes case of null pattern)
|
||||
return loc
|
||||
}
|
||||
// Do a fuzzy compare.
|
||||
return dmp.MatchBitap(text, pattern, loc)
|
||||
}
|
||||
|
||||
// MatchBitap locates the best instance of 'pattern' in 'text' near 'loc' using the Bitap algorithm.
|
||||
// Returns -1 if no match was found.
|
||||
func (dmp *DiffMatchPatch) MatchBitap(text, pattern string, loc int) int {
|
||||
// Initialise the alphabet.
|
||||
s := dmp.MatchAlphabet(pattern)
|
||||
|
||||
// Highest score beyond which we give up.
|
||||
scoreThreshold := dmp.MatchThreshold
|
||||
// Is there a nearby exact match? (speedup)
|
||||
bestLoc := indexOf(text, pattern, loc)
|
||||
if bestLoc != -1 {
|
||||
scoreThreshold = math.Min(dmp.matchBitapScore(0, bestLoc, loc,
|
||||
pattern), scoreThreshold)
|
||||
// What about in the other direction? (speedup)
|
||||
bestLoc = lastIndexOf(text, pattern, loc+len(pattern))
|
||||
if bestLoc != -1 {
|
||||
scoreThreshold = math.Min(dmp.matchBitapScore(0, bestLoc, loc,
|
||||
pattern), scoreThreshold)
|
||||
}
|
||||
}
|
||||
|
||||
// Initialise the bit arrays.
|
||||
matchmask := 1 << uint((len(pattern) - 1))
|
||||
bestLoc = -1
|
||||
|
||||
var binMin, binMid int
|
||||
binMax := len(pattern) + len(text)
|
||||
lastRd := []int{}
|
||||
for d := 0; d < len(pattern); d++ {
|
||||
// Scan for the best match; each iteration allows for one more error. Run a binary search to determine how far from 'loc' we can stray at this error level.
|
||||
binMin = 0
|
||||
binMid = binMax
|
||||
for binMin < binMid {
|
||||
if dmp.matchBitapScore(d, loc+binMid, loc, pattern) <= scoreThreshold {
|
||||
binMin = binMid
|
||||
} else {
|
||||
binMax = binMid
|
||||
}
|
||||
binMid = (binMax-binMin)/2 + binMin
|
||||
}
|
||||
// Use the result from this iteration as the maximum for the next.
|
||||
binMax = binMid
|
||||
start := int(math.Max(1, float64(loc-binMid+1)))
|
||||
finish := int(math.Min(float64(loc+binMid), float64(len(text))) + float64(len(pattern)))
|
||||
|
||||
rd := make([]int, finish+2)
|
||||
rd[finish+1] = (1 << uint(d)) - 1
|
||||
|
||||
for j := finish; j >= start; j-- {
|
||||
var charMatch int
|
||||
if len(text) <= j-1 {
|
||||
// Out of range.
|
||||
charMatch = 0
|
||||
} else if _, ok := s[text[j-1]]; !ok {
|
||||
charMatch = 0
|
||||
} else {
|
||||
charMatch = s[text[j-1]]
|
||||
}
|
||||
|
||||
if d == 0 {
|
||||
// First pass: exact match.
|
||||
rd[j] = ((rd[j+1] << 1) | 1) & charMatch
|
||||
} else {
|
||||
// Subsequent passes: fuzzy match.
|
||||
rd[j] = ((rd[j+1]<<1)|1)&charMatch | (((lastRd[j+1] | lastRd[j]) << 1) | 1) | lastRd[j+1]
|
||||
}
|
||||
if (rd[j] & matchmask) != 0 {
|
||||
score := dmp.matchBitapScore(d, j-1, loc, pattern)
|
||||
// This match will almost certainly be better than any existing match. But check anyway.
|
||||
if score <= scoreThreshold {
|
||||
// Told you so.
|
||||
scoreThreshold = score
|
||||
bestLoc = j - 1
|
||||
if bestLoc > loc {
|
||||
// When passing loc, don't exceed our current distance from loc.
|
||||
start = int(math.Max(1, float64(2*loc-bestLoc)))
|
||||
} else {
|
||||
// Already passed loc, downhill from here on in.
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if dmp.matchBitapScore(d+1, loc, loc, pattern) > scoreThreshold {
|
||||
// No hope for a (better) match at greater error levels.
|
||||
break
|
||||
}
|
||||
lastRd = rd
|
||||
}
|
||||
return bestLoc
|
||||
}
|
||||
|
||||
// matchBitapScore computes and returns the score for a match with e errors and x location.
|
||||
func (dmp *DiffMatchPatch) matchBitapScore(e, x, loc int, pattern string) float64 {
|
||||
accuracy := float64(e) / float64(len(pattern))
|
||||
proximity := math.Abs(float64(loc - x))
|
||||
if dmp.MatchDistance == 0 {
|
||||
// Dodge divide by zero error.
|
||||
if proximity == 0 {
|
||||
return accuracy
|
||||
}
|
||||
|
||||
return 1.0
|
||||
}
|
||||
return accuracy + (proximity / float64(dmp.MatchDistance))
|
||||
}
|
||||
|
||||
// MatchAlphabet initialises the alphabet for the Bitap algorithm.
|
||||
func (dmp *DiffMatchPatch) MatchAlphabet(pattern string) map[byte]int {
|
||||
s := map[byte]int{}
|
||||
charPattern := []byte(pattern)
|
||||
for _, c := range charPattern {
|
||||
_, ok := s[c]
|
||||
if !ok {
|
||||
s[c] = 0
|
||||
}
|
||||
}
|
||||
i := 0
|
||||
|
||||
for _, c := range charPattern {
|
||||
value := s[c] | int(uint(1)<<uint((len(pattern)-i-1)))
|
||||
s[c] = value
|
||||
i++
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
|
||||
// https://github.com/sergi/go-diff
|
||||
// See the included LICENSE file for license details.
|
||||
//
|
||||
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
|
||||
// Original library is Copyright (c) 2006 Google Inc.
|
||||
// http://code.google.com/p/google-diff-match-patch/
|
||||
|
||||
package diffmatchpatch
|
||||
|
||||
func min(x, y int) int {
|
||||
if x < y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
||||
func max(x, y int) int {
|
||||
if x > y {
|
||||
return x
|
||||
}
|
||||
return y
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
// Code generated by "stringer -type=Operation -trimprefix=Diff"; DO NOT EDIT.
|
||||
|
||||
package diffmatchpatch
|
||||
|
||||
import "fmt"
|
||||
|
||||
const _Operation_name = "DeleteEqualInsert"
|
||||
|
||||
var _Operation_index = [...]uint8{0, 6, 11, 17}
|
||||
|
||||
func (i Operation) String() string {
|
||||
i -= -1
|
||||
if i < 0 || i >= Operation(len(_Operation_index)-1) {
|
||||
return fmt.Sprintf("Operation(%d)", i+-1)
|
||||
}
|
||||
return _Operation_name[_Operation_index[i]:_Operation_index[i+1]]
|
||||
}
|
||||
|
|
@ -1,556 +0,0 @@
|
|||
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
|
||||
// https://github.com/sergi/go-diff
|
||||
// See the included LICENSE file for license details.
|
||||
//
|
||||
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
|
||||
// Original library is Copyright (c) 2006 Google Inc.
|
||||
// http://code.google.com/p/google-diff-match-patch/
|
||||
|
||||
package diffmatchpatch
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"math"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Patch represents one patch operation.
|
||||
type Patch struct {
|
||||
diffs []Diff
|
||||
Start1 int
|
||||
Start2 int
|
||||
Length1 int
|
||||
Length2 int
|
||||
}
|
||||
|
||||
// String emulates GNU diff's format.
|
||||
// Header: @@ -382,8 +481,9 @@
|
||||
// Indices are printed as 1-based, not 0-based.
|
||||
func (p *Patch) String() string {
|
||||
var coords1, coords2 string
|
||||
|
||||
if p.Length1 == 0 {
|
||||
coords1 = strconv.Itoa(p.Start1) + ",0"
|
||||
} else if p.Length1 == 1 {
|
||||
coords1 = strconv.Itoa(p.Start1 + 1)
|
||||
} else {
|
||||
coords1 = strconv.Itoa(p.Start1+1) + "," + strconv.Itoa(p.Length1)
|
||||
}
|
||||
|
||||
if p.Length2 == 0 {
|
||||
coords2 = strconv.Itoa(p.Start2) + ",0"
|
||||
} else if p.Length2 == 1 {
|
||||
coords2 = strconv.Itoa(p.Start2 + 1)
|
||||
} else {
|
||||
coords2 = strconv.Itoa(p.Start2+1) + "," + strconv.Itoa(p.Length2)
|
||||
}
|
||||
|
||||
var text bytes.Buffer
|
||||
_, _ = text.WriteString("@@ -" + coords1 + " +" + coords2 + " @@\n")
|
||||
|
||||
// Escape the body of the patch with %xx notation.
|
||||
for _, aDiff := range p.diffs {
|
||||
switch aDiff.Type {
|
||||
case DiffInsert:
|
||||
_, _ = text.WriteString("+")
|
||||
case DiffDelete:
|
||||
_, _ = text.WriteString("-")
|
||||
case DiffEqual:
|
||||
_, _ = text.WriteString(" ")
|
||||
}
|
||||
|
||||
_, _ = text.WriteString(strings.Replace(url.QueryEscape(aDiff.Text), "+", " ", -1))
|
||||
_, _ = text.WriteString("\n")
|
||||
}
|
||||
|
||||
return unescaper.Replace(text.String())
|
||||
}
|
||||
|
||||
// PatchAddContext increases the context until it is unique, but doesn't let the pattern expand beyond MatchMaxBits.
|
||||
func (dmp *DiffMatchPatch) PatchAddContext(patch Patch, text string) Patch {
|
||||
if len(text) == 0 {
|
||||
return patch
|
||||
}
|
||||
|
||||
pattern := text[patch.Start2 : patch.Start2+patch.Length1]
|
||||
padding := 0
|
||||
|
||||
// Look for the first and last matches of pattern in text. If two different matches are found, increase the pattern length.
|
||||
for strings.Index(text, pattern) != strings.LastIndex(text, pattern) &&
|
||||
len(pattern) < dmp.MatchMaxBits-2*dmp.PatchMargin {
|
||||
padding += dmp.PatchMargin
|
||||
maxStart := max(0, patch.Start2-padding)
|
||||
minEnd := min(len(text), patch.Start2+patch.Length1+padding)
|
||||
pattern = text[maxStart:minEnd]
|
||||
}
|
||||
// Add one chunk for good luck.
|
||||
padding += dmp.PatchMargin
|
||||
|
||||
// Add the prefix.
|
||||
prefix := text[max(0, patch.Start2-padding):patch.Start2]
|
||||
if len(prefix) != 0 {
|
||||
patch.diffs = append([]Diff{Diff{DiffEqual, prefix}}, patch.diffs...)
|
||||
}
|
||||
// Add the suffix.
|
||||
suffix := text[patch.Start2+patch.Length1 : min(len(text), patch.Start2+patch.Length1+padding)]
|
||||
if len(suffix) != 0 {
|
||||
patch.diffs = append(patch.diffs, Diff{DiffEqual, suffix})
|
||||
}
|
||||
|
||||
// Roll back the start points.
|
||||
patch.Start1 -= len(prefix)
|
||||
patch.Start2 -= len(prefix)
|
||||
// Extend the lengths.
|
||||
patch.Length1 += len(prefix) + len(suffix)
|
||||
patch.Length2 += len(prefix) + len(suffix)
|
||||
|
||||
return patch
|
||||
}
|
||||
|
||||
// PatchMake computes a list of patches.
|
||||
func (dmp *DiffMatchPatch) PatchMake(opt ...interface{}) []Patch {
|
||||
if len(opt) == 1 {
|
||||
diffs, _ := opt[0].([]Diff)
|
||||
text1 := dmp.DiffText1(diffs)
|
||||
return dmp.PatchMake(text1, diffs)
|
||||
} else if len(opt) == 2 {
|
||||
text1 := opt[0].(string)
|
||||
switch t := opt[1].(type) {
|
||||
case string:
|
||||
diffs := dmp.DiffMain(text1, t, true)
|
||||
if len(diffs) > 2 {
|
||||
diffs = dmp.DiffCleanupSemantic(diffs)
|
||||
diffs = dmp.DiffCleanupEfficiency(diffs)
|
||||
}
|
||||
return dmp.PatchMake(text1, diffs)
|
||||
case []Diff:
|
||||
return dmp.patchMake2(text1, t)
|
||||
}
|
||||
} else if len(opt) == 3 {
|
||||
return dmp.PatchMake(opt[0], opt[2])
|
||||
}
|
||||
return []Patch{}
|
||||
}
|
||||
|
||||
// patchMake2 computes a list of patches to turn text1 into text2.
|
||||
// text2 is not provided, diffs are the delta between text1 and text2.
|
||||
func (dmp *DiffMatchPatch) patchMake2(text1 string, diffs []Diff) []Patch {
|
||||
// Check for null inputs not needed since null can't be passed in C#.
|
||||
patches := []Patch{}
|
||||
if len(diffs) == 0 {
|
||||
return patches // Get rid of the null case.
|
||||
}
|
||||
|
||||
patch := Patch{}
|
||||
charCount1 := 0 // Number of characters into the text1 string.
|
||||
charCount2 := 0 // Number of characters into the text2 string.
|
||||
// Start with text1 (prepatchText) and apply the diffs until we arrive at text2 (postpatchText). We recreate the patches one by one to determine context info.
|
||||
prepatchText := text1
|
||||
postpatchText := text1
|
||||
|
||||
for i, aDiff := range diffs {
|
||||
if len(patch.diffs) == 0 && aDiff.Type != DiffEqual {
|
||||
// A new patch starts here.
|
||||
patch.Start1 = charCount1
|
||||
patch.Start2 = charCount2
|
||||
}
|
||||
|
||||
switch aDiff.Type {
|
||||
case DiffInsert:
|
||||
patch.diffs = append(patch.diffs, aDiff)
|
||||
patch.Length2 += len(aDiff.Text)
|
||||
postpatchText = postpatchText[:charCount2] +
|
||||
aDiff.Text + postpatchText[charCount2:]
|
||||
case DiffDelete:
|
||||
patch.Length1 += len(aDiff.Text)
|
||||
patch.diffs = append(patch.diffs, aDiff)
|
||||
postpatchText = postpatchText[:charCount2] + postpatchText[charCount2+len(aDiff.Text):]
|
||||
case DiffEqual:
|
||||
if len(aDiff.Text) <= 2*dmp.PatchMargin &&
|
||||
len(patch.diffs) != 0 && i != len(diffs)-1 {
|
||||
// Small equality inside a patch.
|
||||
patch.diffs = append(patch.diffs, aDiff)
|
||||
patch.Length1 += len(aDiff.Text)
|
||||
patch.Length2 += len(aDiff.Text)
|
||||
}
|
||||
if len(aDiff.Text) >= 2*dmp.PatchMargin {
|
||||
// Time for a new patch.
|
||||
if len(patch.diffs) != 0 {
|
||||
patch = dmp.PatchAddContext(patch, prepatchText)
|
||||
patches = append(patches, patch)
|
||||
patch = Patch{}
|
||||
// Unlike Unidiff, our patch lists have a rolling context. http://code.google.com/p/google-diff-match-patch/wiki/Unidiff Update prepatch text & pos to reflect the application of the just completed patch.
|
||||
prepatchText = postpatchText
|
||||
charCount1 = charCount2
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update the current character count.
|
||||
if aDiff.Type != DiffInsert {
|
||||
charCount1 += len(aDiff.Text)
|
||||
}
|
||||
if aDiff.Type != DiffDelete {
|
||||
charCount2 += len(aDiff.Text)
|
||||
}
|
||||
}
|
||||
|
||||
// Pick up the leftover patch if not empty.
|
||||
if len(patch.diffs) != 0 {
|
||||
patch = dmp.PatchAddContext(patch, prepatchText)
|
||||
patches = append(patches, patch)
|
||||
}
|
||||
|
||||
return patches
|
||||
}
|
||||
|
||||
// PatchDeepCopy returns an array that is identical to a given an array of patches.
|
||||
func (dmp *DiffMatchPatch) PatchDeepCopy(patches []Patch) []Patch {
|
||||
patchesCopy := []Patch{}
|
||||
for _, aPatch := range patches {
|
||||
patchCopy := Patch{}
|
||||
for _, aDiff := range aPatch.diffs {
|
||||
patchCopy.diffs = append(patchCopy.diffs, Diff{
|
||||
aDiff.Type,
|
||||
aDiff.Text,
|
||||
})
|
||||
}
|
||||
patchCopy.Start1 = aPatch.Start1
|
||||
patchCopy.Start2 = aPatch.Start2
|
||||
patchCopy.Length1 = aPatch.Length1
|
||||
patchCopy.Length2 = aPatch.Length2
|
||||
patchesCopy = append(patchesCopy, patchCopy)
|
||||
}
|
||||
return patchesCopy
|
||||
}
|
||||
|
||||
// PatchApply merges a set of patches onto the text. Returns a patched text, as well as an array of true/false values indicating which patches were applied.
|
||||
func (dmp *DiffMatchPatch) PatchApply(patches []Patch, text string) (string, []bool) {
|
||||
if len(patches) == 0 {
|
||||
return text, []bool{}
|
||||
}
|
||||
|
||||
// Deep copy the patches so that no changes are made to originals.
|
||||
patches = dmp.PatchDeepCopy(patches)
|
||||
|
||||
nullPadding := dmp.PatchAddPadding(patches)
|
||||
text = nullPadding + text + nullPadding
|
||||
patches = dmp.PatchSplitMax(patches)
|
||||
|
||||
x := 0
|
||||
// delta keeps track of the offset between the expected and actual location of the previous patch. If there are patches expected at positions 10 and 20, but the first patch was found at 12, delta is 2 and the second patch has an effective expected position of 22.
|
||||
delta := 0
|
||||
results := make([]bool, len(patches))
|
||||
for _, aPatch := range patches {
|
||||
expectedLoc := aPatch.Start2 + delta
|
||||
text1 := dmp.DiffText1(aPatch.diffs)
|
||||
var startLoc int
|
||||
endLoc := -1
|
||||
if len(text1) > dmp.MatchMaxBits {
|
||||
// PatchSplitMax will only provide an oversized pattern in the case of a monster delete.
|
||||
startLoc = dmp.MatchMain(text, text1[:dmp.MatchMaxBits], expectedLoc)
|
||||
if startLoc != -1 {
|
||||
endLoc = dmp.MatchMain(text,
|
||||
text1[len(text1)-dmp.MatchMaxBits:], expectedLoc+len(text1)-dmp.MatchMaxBits)
|
||||
if endLoc == -1 || startLoc >= endLoc {
|
||||
// Can't find valid trailing context. Drop this patch.
|
||||
startLoc = -1
|
||||
}
|
||||
}
|
||||
} else {
|
||||
startLoc = dmp.MatchMain(text, text1, expectedLoc)
|
||||
}
|
||||
if startLoc == -1 {
|
||||
// No match found. :(
|
||||
results[x] = false
|
||||
// Subtract the delta for this failed patch from subsequent patches.
|
||||
delta -= aPatch.Length2 - aPatch.Length1
|
||||
} else {
|
||||
// Found a match. :)
|
||||
results[x] = true
|
||||
delta = startLoc - expectedLoc
|
||||
var text2 string
|
||||
if endLoc == -1 {
|
||||
text2 = text[startLoc:int(math.Min(float64(startLoc+len(text1)), float64(len(text))))]
|
||||
} else {
|
||||
text2 = text[startLoc:int(math.Min(float64(endLoc+dmp.MatchMaxBits), float64(len(text))))]
|
||||
}
|
||||
if text1 == text2 {
|
||||
// Perfect match, just shove the Replacement text in.
|
||||
text = text[:startLoc] + dmp.DiffText2(aPatch.diffs) + text[startLoc+len(text1):]
|
||||
} else {
|
||||
// Imperfect match. Run a diff to get a framework of equivalent indices.
|
||||
diffs := dmp.DiffMain(text1, text2, false)
|
||||
if len(text1) > dmp.MatchMaxBits && float64(dmp.DiffLevenshtein(diffs))/float64(len(text1)) > dmp.PatchDeleteThreshold {
|
||||
// The end points match, but the content is unacceptably bad.
|
||||
results[x] = false
|
||||
} else {
|
||||
diffs = dmp.DiffCleanupSemanticLossless(diffs)
|
||||
index1 := 0
|
||||
for _, aDiff := range aPatch.diffs {
|
||||
if aDiff.Type != DiffEqual {
|
||||
index2 := dmp.DiffXIndex(diffs, index1)
|
||||
if aDiff.Type == DiffInsert {
|
||||
// Insertion
|
||||
text = text[:startLoc+index2] + aDiff.Text + text[startLoc+index2:]
|
||||
} else if aDiff.Type == DiffDelete {
|
||||
// Deletion
|
||||
startIndex := startLoc + index2
|
||||
text = text[:startIndex] +
|
||||
text[startIndex+dmp.DiffXIndex(diffs, index1+len(aDiff.Text))-index2:]
|
||||
}
|
||||
}
|
||||
if aDiff.Type != DiffDelete {
|
||||
index1 += len(aDiff.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
x++
|
||||
}
|
||||
// Strip the padding off.
|
||||
text = text[len(nullPadding) : len(nullPadding)+(len(text)-2*len(nullPadding))]
|
||||
return text, results
|
||||
}
|
||||
|
||||
// PatchAddPadding adds some padding on text start and end so that edges can match something.
|
||||
// Intended to be called only from within patchApply.
|
||||
func (dmp *DiffMatchPatch) PatchAddPadding(patches []Patch) string {
|
||||
paddingLength := dmp.PatchMargin
|
||||
nullPadding := ""
|
||||
for x := 1; x <= paddingLength; x++ {
|
||||
nullPadding += string(x)
|
||||
}
|
||||
|
||||
// Bump all the patches forward.
|
||||
for i := range patches {
|
||||
patches[i].Start1 += paddingLength
|
||||
patches[i].Start2 += paddingLength
|
||||
}
|
||||
|
||||
// Add some padding on start of first diff.
|
||||
if len(patches[0].diffs) == 0 || patches[0].diffs[0].Type != DiffEqual {
|
||||
// Add nullPadding equality.
|
||||
patches[0].diffs = append([]Diff{Diff{DiffEqual, nullPadding}}, patches[0].diffs...)
|
||||
patches[0].Start1 -= paddingLength // Should be 0.
|
||||
patches[0].Start2 -= paddingLength // Should be 0.
|
||||
patches[0].Length1 += paddingLength
|
||||
patches[0].Length2 += paddingLength
|
||||
} else if paddingLength > len(patches[0].diffs[0].Text) {
|
||||
// Grow first equality.
|
||||
extraLength := paddingLength - len(patches[0].diffs[0].Text)
|
||||
patches[0].diffs[0].Text = nullPadding[len(patches[0].diffs[0].Text):] + patches[0].diffs[0].Text
|
||||
patches[0].Start1 -= extraLength
|
||||
patches[0].Start2 -= extraLength
|
||||
patches[0].Length1 += extraLength
|
||||
patches[0].Length2 += extraLength
|
||||
}
|
||||
|
||||
// Add some padding on end of last diff.
|
||||
last := len(patches) - 1
|
||||
if len(patches[last].diffs) == 0 || patches[last].diffs[len(patches[last].diffs)-1].Type != DiffEqual {
|
||||
// Add nullPadding equality.
|
||||
patches[last].diffs = append(patches[last].diffs, Diff{DiffEqual, nullPadding})
|
||||
patches[last].Length1 += paddingLength
|
||||
patches[last].Length2 += paddingLength
|
||||
} else if paddingLength > len(patches[last].diffs[len(patches[last].diffs)-1].Text) {
|
||||
// Grow last equality.
|
||||
lastDiff := patches[last].diffs[len(patches[last].diffs)-1]
|
||||
extraLength := paddingLength - len(lastDiff.Text)
|
||||
patches[last].diffs[len(patches[last].diffs)-1].Text += nullPadding[:extraLength]
|
||||
patches[last].Length1 += extraLength
|
||||
patches[last].Length2 += extraLength
|
||||
}
|
||||
|
||||
return nullPadding
|
||||
}
|
||||
|
||||
// PatchSplitMax looks through the patches and breaks up any which are longer than the maximum limit of the match algorithm.
|
||||
// Intended to be called only from within patchApply.
|
||||
func (dmp *DiffMatchPatch) PatchSplitMax(patches []Patch) []Patch {
|
||||
patchSize := dmp.MatchMaxBits
|
||||
for x := 0; x < len(patches); x++ {
|
||||
if patches[x].Length1 <= patchSize {
|
||||
continue
|
||||
}
|
||||
bigpatch := patches[x]
|
||||
// Remove the big old patch.
|
||||
patches = append(patches[:x], patches[x+1:]...)
|
||||
x--
|
||||
|
||||
Start1 := bigpatch.Start1
|
||||
Start2 := bigpatch.Start2
|
||||
precontext := ""
|
||||
for len(bigpatch.diffs) != 0 {
|
||||
// Create one of several smaller patches.
|
||||
patch := Patch{}
|
||||
empty := true
|
||||
patch.Start1 = Start1 - len(precontext)
|
||||
patch.Start2 = Start2 - len(precontext)
|
||||
if len(precontext) != 0 {
|
||||
patch.Length1 = len(precontext)
|
||||
patch.Length2 = len(precontext)
|
||||
patch.diffs = append(patch.diffs, Diff{DiffEqual, precontext})
|
||||
}
|
||||
for len(bigpatch.diffs) != 0 && patch.Length1 < patchSize-dmp.PatchMargin {
|
||||
diffType := bigpatch.diffs[0].Type
|
||||
diffText := bigpatch.diffs[0].Text
|
||||
if diffType == DiffInsert {
|
||||
// Insertions are harmless.
|
||||
patch.Length2 += len(diffText)
|
||||
Start2 += len(diffText)
|
||||
patch.diffs = append(patch.diffs, bigpatch.diffs[0])
|
||||
bigpatch.diffs = bigpatch.diffs[1:]
|
||||
empty = false
|
||||
} else if diffType == DiffDelete && len(patch.diffs) == 1 && patch.diffs[0].Type == DiffEqual && len(diffText) > 2*patchSize {
|
||||
// This is a large deletion. Let it pass in one chunk.
|
||||
patch.Length1 += len(diffText)
|
||||
Start1 += len(diffText)
|
||||
empty = false
|
||||
patch.diffs = append(patch.diffs, Diff{diffType, diffText})
|
||||
bigpatch.diffs = bigpatch.diffs[1:]
|
||||
} else {
|
||||
// Deletion or equality. Only take as much as we can stomach.
|
||||
diffText = diffText[:min(len(diffText), patchSize-patch.Length1-dmp.PatchMargin)]
|
||||
|
||||
patch.Length1 += len(diffText)
|
||||
Start1 += len(diffText)
|
||||
if diffType == DiffEqual {
|
||||
patch.Length2 += len(diffText)
|
||||
Start2 += len(diffText)
|
||||
} else {
|
||||
empty = false
|
||||
}
|
||||
patch.diffs = append(patch.diffs, Diff{diffType, diffText})
|
||||
if diffText == bigpatch.diffs[0].Text {
|
||||
bigpatch.diffs = bigpatch.diffs[1:]
|
||||
} else {
|
||||
bigpatch.diffs[0].Text =
|
||||
bigpatch.diffs[0].Text[len(diffText):]
|
||||
}
|
||||
}
|
||||
}
|
||||
// Compute the head context for the next patch.
|
||||
precontext = dmp.DiffText2(patch.diffs)
|
||||
precontext = precontext[max(0, len(precontext)-dmp.PatchMargin):]
|
||||
|
||||
postcontext := ""
|
||||
// Append the end context for this patch.
|
||||
if len(dmp.DiffText1(bigpatch.diffs)) > dmp.PatchMargin {
|
||||
postcontext = dmp.DiffText1(bigpatch.diffs)[:dmp.PatchMargin]
|
||||
} else {
|
||||
postcontext = dmp.DiffText1(bigpatch.diffs)
|
||||
}
|
||||
|
||||
if len(postcontext) != 0 {
|
||||
patch.Length1 += len(postcontext)
|
||||
patch.Length2 += len(postcontext)
|
||||
if len(patch.diffs) != 0 && patch.diffs[len(patch.diffs)-1].Type == DiffEqual {
|
||||
patch.diffs[len(patch.diffs)-1].Text += postcontext
|
||||
} else {
|
||||
patch.diffs = append(patch.diffs, Diff{DiffEqual, postcontext})
|
||||
}
|
||||
}
|
||||
if !empty {
|
||||
x++
|
||||
patches = append(patches[:x], append([]Patch{patch}, patches[x:]...)...)
|
||||
}
|
||||
}
|
||||
}
|
||||
return patches
|
||||
}
|
||||
|
||||
// PatchToText takes a list of patches and returns a textual representation.
|
||||
func (dmp *DiffMatchPatch) PatchToText(patches []Patch) string {
|
||||
var text bytes.Buffer
|
||||
for _, aPatch := range patches {
|
||||
_, _ = text.WriteString(aPatch.String())
|
||||
}
|
||||
return text.String()
|
||||
}
|
||||
|
||||
// PatchFromText parses a textual representation of patches and returns a List of Patch objects.
|
||||
func (dmp *DiffMatchPatch) PatchFromText(textline string) ([]Patch, error) {
|
||||
patches := []Patch{}
|
||||
if len(textline) == 0 {
|
||||
return patches, nil
|
||||
}
|
||||
text := strings.Split(textline, "\n")
|
||||
textPointer := 0
|
||||
patchHeader := regexp.MustCompile("^@@ -(\\d+),?(\\d*) \\+(\\d+),?(\\d*) @@$")
|
||||
|
||||
var patch Patch
|
||||
var sign uint8
|
||||
var line string
|
||||
for textPointer < len(text) {
|
||||
|
||||
if !patchHeader.MatchString(text[textPointer]) {
|
||||
return patches, errors.New("Invalid patch string: " + text[textPointer])
|
||||
}
|
||||
|
||||
patch = Patch{}
|
||||
m := patchHeader.FindStringSubmatch(text[textPointer])
|
||||
|
||||
patch.Start1, _ = strconv.Atoi(m[1])
|
||||
if len(m[2]) == 0 {
|
||||
patch.Start1--
|
||||
patch.Length1 = 1
|
||||
} else if m[2] == "0" {
|
||||
patch.Length1 = 0
|
||||
} else {
|
||||
patch.Start1--
|
||||
patch.Length1, _ = strconv.Atoi(m[2])
|
||||
}
|
||||
|
||||
patch.Start2, _ = strconv.Atoi(m[3])
|
||||
|
||||
if len(m[4]) == 0 {
|
||||
patch.Start2--
|
||||
patch.Length2 = 1
|
||||
} else if m[4] == "0" {
|
||||
patch.Length2 = 0
|
||||
} else {
|
||||
patch.Start2--
|
||||
patch.Length2, _ = strconv.Atoi(m[4])
|
||||
}
|
||||
textPointer++
|
||||
|
||||
for textPointer < len(text) {
|
||||
if len(text[textPointer]) > 0 {
|
||||
sign = text[textPointer][0]
|
||||
} else {
|
||||
textPointer++
|
||||
continue
|
||||
}
|
||||
|
||||
line = text[textPointer][1:]
|
||||
line = strings.Replace(line, "+", "%2b", -1)
|
||||
line, _ = url.QueryUnescape(line)
|
||||
if sign == '-' {
|
||||
// Deletion.
|
||||
patch.diffs = append(patch.diffs, Diff{DiffDelete, line})
|
||||
} else if sign == '+' {
|
||||
// Insertion.
|
||||
patch.diffs = append(patch.diffs, Diff{DiffInsert, line})
|
||||
} else if sign == ' ' {
|
||||
// Minor equality.
|
||||
patch.diffs = append(patch.diffs, Diff{DiffEqual, line})
|
||||
} else if sign == '@' {
|
||||
// Start of next patch.
|
||||
break
|
||||
} else {
|
||||
// WTF?
|
||||
return patches, errors.New("Invalid patch mode '" + string(sign) + "' in: " + string(line))
|
||||
}
|
||||
textPointer++
|
||||
}
|
||||
|
||||
patches = append(patches, patch)
|
||||
}
|
||||
return patches, nil
|
||||
}
|
||||
|
|
@ -1,88 +0,0 @@
|
|||
// Copyright (c) 2012-2016 The go-diff authors. All rights reserved.
|
||||
// https://github.com/sergi/go-diff
|
||||
// See the included LICENSE file for license details.
|
||||
//
|
||||
// go-diff is a Go implementation of Google's Diff, Match, and Patch library
|
||||
// Original library is Copyright (c) 2006 Google Inc.
|
||||
// http://code.google.com/p/google-diff-match-patch/
|
||||
|
||||
package diffmatchpatch
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// unescaper unescapes selected chars for compatibility with JavaScript's encodeURI.
|
||||
// In speed critical applications this could be dropped since the receiving application will certainly decode these fine. Note that this function is case-sensitive. Thus "%3F" would not be unescaped. But this is ok because it is only called with the output of HttpUtility.UrlEncode which returns lowercase hex. Example: "%3f" -> "?", "%24" -> "$", etc.
|
||||
var unescaper = strings.NewReplacer(
|
||||
"%21", "!", "%7E", "~", "%27", "'",
|
||||
"%28", "(", "%29", ")", "%3B", ";",
|
||||
"%2F", "/", "%3F", "?", "%3A", ":",
|
||||
"%40", "@", "%26", "&", "%3D", "=",
|
||||
"%2B", "+", "%24", "$", "%2C", ",", "%23", "#", "%2A", "*")
|
||||
|
||||
// indexOf returns the first index of pattern in str, starting at str[i].
|
||||
func indexOf(str string, pattern string, i int) int {
|
||||
if i > len(str)-1 {
|
||||
return -1
|
||||
}
|
||||
if i <= 0 {
|
||||
return strings.Index(str, pattern)
|
||||
}
|
||||
ind := strings.Index(str[i:], pattern)
|
||||
if ind == -1 {
|
||||
return -1
|
||||
}
|
||||
return ind + i
|
||||
}
|
||||
|
||||
// lastIndexOf returns the last index of pattern in str, starting at str[i].
|
||||
func lastIndexOf(str string, pattern string, i int) int {
|
||||
if i < 0 {
|
||||
return -1
|
||||
}
|
||||
if i >= len(str) {
|
||||
return strings.LastIndex(str, pattern)
|
||||
}
|
||||
_, size := utf8.DecodeRuneInString(str[i:])
|
||||
return strings.LastIndex(str[:i+size], pattern)
|
||||
}
|
||||
|
||||
// runesIndexOf returns the index of pattern in target, starting at target[i].
|
||||
func runesIndexOf(target, pattern []rune, i int) int {
|
||||
if i > len(target)-1 {
|
||||
return -1
|
||||
}
|
||||
if i <= 0 {
|
||||
return runesIndex(target, pattern)
|
||||
}
|
||||
ind := runesIndex(target[i:], pattern)
|
||||
if ind == -1 {
|
||||
return -1
|
||||
}
|
||||
return ind + i
|
||||
}
|
||||
|
||||
func runesEqual(r1, r2 []rune) bool {
|
||||
if len(r1) != len(r2) {
|
||||
return false
|
||||
}
|
||||
for i, c := range r1 {
|
||||
if c != r2[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// runesIndex is the equivalent of strings.Index for rune slices.
|
||||
func runesIndex(r1, r2 []rune) int {
|
||||
last := len(r1) - len(r2)
|
||||
for i := 0; i <= last; i++ {
|
||||
if runesEqual(r1[i:i+len(r2)], r2) {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
eval/eval
|
||||
/target
|
||||
*.test
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
language: go
|
||||
|
||||
sudo: false
|
||||
|
||||
branches:
|
||||
except:
|
||||
- release
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- develop
|
||||
- travis
|
||||
|
||||
go:
|
||||
- 1.9
|
||||
- tip
|
||||
|
||||
matrix:
|
||||
allow_failures:
|
||||
- go: tip
|
||||
|
||||
before_install:
|
||||
- if [ -n "$GH_USER" ]; then git config --global github.user ${GH_USER}; fi;
|
||||
- if [ -n "$GH_TOKEN" ]; then git config --global github.token ${GH_TOKEN}; fi;
|
||||
- go get github.com/mattn/goveralls
|
||||
|
||||
before_script:
|
||||
- make deps
|
||||
|
||||
script:
|
||||
- make qa
|
||||
|
||||
after_failure:
|
||||
- cat ./target/test/report.xml
|
||||
|
||||
after_success:
|
||||
- if [ "$TRAVIS_GO_VERSION" = "1.9" ]; then $HOME/gopath/bin/goveralls -covermode=count -coverprofile=target/report/coverage.out -service=travis-ci; fi;
|
||||
|
|
@ -1,230 +0,0 @@
|
|||
Copyright (c) 2015, 2016 Damian Gryski <damian@gryski.com>
|
||||
Copyright (c) 2018 Tomás Senart <tsenart@gmail.com>
|
||||
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
|
||||
-------------------------------------------------------------------------------
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
|
@ -1,203 +0,0 @@
|
|||
# MAKEFILE
|
||||
#
|
||||
# @author Nicola Asuni <info@tecnick.com>
|
||||
# @link https://github.com/tsenart/go-tsz
|
||||
#
|
||||
# This file is intended to be executed in a Linux-compatible system.
|
||||
# It also assumes that the project has been cloned in the right path under GOPATH:
|
||||
# $GOPATH/src/github.com/tsenart/go-tsz
|
||||
#
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
# List special make targets that are not associated with files
|
||||
.PHONY: help all test format fmtcheck vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan qa deps clean nuke
|
||||
|
||||
# Use bash as shell (Note: Ubuntu now uses dash which doesn't support PIPESTATUS).
|
||||
SHELL=/bin/bash
|
||||
|
||||
# CVS path (path to the parent dir containing the project)
|
||||
CVSPATH=github.com/tsenart
|
||||
|
||||
# Project owner
|
||||
OWNER=tsenart
|
||||
|
||||
# Project vendor
|
||||
VENDOR=tsenart
|
||||
|
||||
# Project name
|
||||
PROJECT=go-tsz
|
||||
|
||||
# Project version
|
||||
VERSION=$(shell cat VERSION)
|
||||
|
||||
# Name of RPM or DEB package
|
||||
PKGNAME=${VENDOR}-${PROJECT}
|
||||
|
||||
# Current directory
|
||||
CURRENTDIR=$(shell pwd)
|
||||
|
||||
# GO lang path
|
||||
ifneq ($(GOPATH),)
|
||||
ifeq ($(findstring $(GOPATH),$(CURRENTDIR)),)
|
||||
# the defined GOPATH is not valid
|
||||
GOPATH=
|
||||
endif
|
||||
endif
|
||||
ifeq ($(GOPATH),)
|
||||
# extract the GOPATH
|
||||
GOPATH=$(firstword $(subst /src/, ,$(CURRENTDIR)))
|
||||
endif
|
||||
|
||||
# --- MAKE TARGETS ---
|
||||
|
||||
# Display general help about this command
|
||||
help:
|
||||
@echo ""
|
||||
@echo "$(PROJECT) Makefile."
|
||||
@echo "GOPATH=$(GOPATH)"
|
||||
@echo "The following commands are available:"
|
||||
@echo ""
|
||||
@echo " make qa : Run all the tests"
|
||||
@echo " make test : Run the unit tests"
|
||||
@echo ""
|
||||
@echo " make format : Format the source code"
|
||||
@echo " make fmtcheck : Check if the source code has been formatted"
|
||||
@echo " make vet : Check for suspicious constructs"
|
||||
@echo " make lint : Check for style errors"
|
||||
@echo " make coverage : Generate the coverage report"
|
||||
@echo " make cyclo : Generate the cyclomatic complexity report"
|
||||
@echo " make ineffassign : Detect ineffectual assignments"
|
||||
@echo " make misspell : Detect commonly misspelled words in source files"
|
||||
@echo " make structcheck : Find unused struct fields"
|
||||
@echo " make varcheck : Find unused global variables and constants"
|
||||
@echo " make errcheck : Check that error return values are used"
|
||||
@echo " make gosimple : Suggest code simplifications"
|
||||
@echo " make astscan : GO AST scanner"
|
||||
@echo ""
|
||||
@echo " make docs : Generate source code documentation"
|
||||
@echo ""
|
||||
@echo " make deps : Get the dependencies"
|
||||
@echo " make clean : Remove any build artifact"
|
||||
@echo " make nuke : Deletes any intermediate file"
|
||||
@echo ""
|
||||
|
||||
|
||||
# Alias for help target
|
||||
all: help
|
||||
|
||||
# Run the unit tests
|
||||
test:
|
||||
@mkdir -p target/test
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) \
|
||||
go test \
|
||||
-covermode=atomic \
|
||||
-bench=. \
|
||||
-race \
|
||||
-cpuprofile=target/report/cpu.out \
|
||||
-memprofile=target/report/mem.out \
|
||||
-mutexprofile=target/report/mutex.out \
|
||||
-coverprofile=target/report/coverage.out \
|
||||
-v . | \
|
||||
tee >(PATH=$(GOPATH)/bin:$(PATH) go-junit-report > target/test/report.xml); \
|
||||
test $${PIPESTATUS[0]} -eq 0
|
||||
|
||||
# Format the source code
|
||||
format:
|
||||
@find . -type f -name "*.go" -exec gofmt -s -w {} \;
|
||||
|
||||
# Check if the source code has been formatted
|
||||
fmtcheck:
|
||||
@mkdir -p target
|
||||
@find . -type f -name "*.go" -exec gofmt -s -d {} \; | tee target/format.diff
|
||||
@test ! -s target/format.diff || { echo "ERROR: the source code has not been formatted - please use 'make format' or 'gofmt'"; exit 1; }
|
||||
|
||||
# Check for syntax errors
|
||||
vet:
|
||||
GOPATH=$(GOPATH) go vet .
|
||||
|
||||
# Check for style errors
|
||||
lint:
|
||||
GOPATH=$(GOPATH) PATH=$(GOPATH)/bin:$(PATH) golint .
|
||||
|
||||
# Generate the coverage report
|
||||
coverage:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) \
|
||||
go tool cover -html=target/report/coverage.out -o target/report/coverage.html
|
||||
|
||||
# Report cyclomatic complexity
|
||||
cyclo:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) gocyclo -avg ./ | tee target/report/cyclo.txt ; test $${PIPESTATUS[0]} -eq 0
|
||||
|
||||
# Detect ineffectual assignments
|
||||
ineffassign:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) ineffassign ./ | tee target/report/ineffassign.txt ; test $${PIPESTATUS[0]} -eq 0
|
||||
|
||||
# Detect commonly misspelled words in source files
|
||||
misspell:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) misspell -error ./ | tee target/report/misspell.txt ; test $${PIPESTATUS[0]} -eq 0
|
||||
|
||||
# Find unused struct fields
|
||||
structcheck:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) structcheck -a ./ | tee target/report/structcheck.txt
|
||||
|
||||
# Find unused global variables and constants
|
||||
varcheck:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) varcheck -e ./ | tee target/report/varcheck.txt
|
||||
|
||||
# Check that error return values are used
|
||||
errcheck:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) errcheck ./ | tee target/report/errcheck.txt
|
||||
|
||||
# Suggest code simplifications
|
||||
gosimple:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) gosimple ./ | tee target/report/gosimple.txt
|
||||
|
||||
# AST scanner
|
||||
astscan:
|
||||
@mkdir -p target/report
|
||||
GOPATH=$(GOPATH) gas .//*.go | tee target/report/astscan.txt
|
||||
|
||||
# Generate source docs
|
||||
docs:
|
||||
@mkdir -p target/docs
|
||||
nohup sh -c 'GOPATH=$(GOPATH) godoc -http=127.0.0.1:6060' > target/godoc_server.log 2>&1 &
|
||||
wget --directory-prefix=target/docs/ --execute robots=off --retry-connrefused --recursive --no-parent --adjust-extension --page-requisites --convert-links http://127.0.0.1:6060/pkg/github.com/${VENDOR}/${PROJECT}/ ; kill -9 `lsof -ti :6060`
|
||||
@echo '<html><head><meta http-equiv="refresh" content="0;./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html"/></head><a href="./127.0.0.1:6060/pkg/'${CVSPATH}'/'${PROJECT}'/index.html">'${PKGNAME}' Documentation ...</a></html>' > target/docs/index.html
|
||||
|
||||
# Alias to run all quality-assurance checks
|
||||
qa: fmtcheck test vet lint coverage cyclo ineffassign misspell structcheck varcheck errcheck gosimple astscan
|
||||
|
||||
# --- INSTALL ---
|
||||
|
||||
# Get the dependencies
|
||||
deps:
|
||||
GOPATH=$(GOPATH) go get ./...
|
||||
GOPATH=$(GOPATH) go get github.com/golang/lint/golint
|
||||
GOPATH=$(GOPATH) go get github.com/jstemmer/go-junit-report
|
||||
GOPATH=$(GOPATH) go get github.com/axw/gocov/gocov
|
||||
GOPATH=$(GOPATH) go get github.com/fzipp/gocyclo
|
||||
GOPATH=$(GOPATH) go get github.com/gordonklaus/ineffassign
|
||||
GOPATH=$(GOPATH) go get github.com/client9/misspell/cmd/misspell
|
||||
GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/structcheck
|
||||
GOPATH=$(GOPATH) go get github.com/opennota/check/cmd/varcheck
|
||||
GOPATH=$(GOPATH) go get github.com/kisielk/errcheck
|
||||
GOPATH=$(GOPATH) go get honnef.co/go/tools/cmd/gosimple
|
||||
GOPATH=$(GOPATH) go get github.com/GoASTScanner/gas
|
||||
|
||||
# Remove any build artifact
|
||||
clean:
|
||||
GOPATH=$(GOPATH) go clean ./...
|
||||
|
||||
# Deletes any intermediate file
|
||||
nuke:
|
||||
rm -rf ./target
|
||||
GOPATH=$(GOPATH) go clean -i ./...
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
# go-tsz
|
||||
|
||||
* Package tsz implement time-series compression http://www.vldb.org/pvldb/vol8/p1816-teller.pdf in Go*
|
||||
|
||||
[](https://github.com/tsenart/go-tsz/tree/master)
|
||||
[](https://travis-ci.org/tsenart/go-tsz?branch=master)
|
||||
[](https://coveralls.io/github/tsenart/go-tsz?branch=master)
|
||||
[](https://goreportcard.com/report/github.com/tsenart/go-tsz)
|
||||
[](http://godoc.org/github.com/tsenart/go-tsz)
|
||||
|
||||
## Description
|
||||
|
||||
Package tsz is a fork of [github.com/dgryski/go-tsz](https://github.com/dgryski/go-tsz) that implements
|
||||
improvements over the [original Gorilla paper](http://www.vldb.org/pvldb/vol8/p1816-teller.pdf) developed by @burmann
|
||||
in his [Master Thesis](https://aaltodoc.aalto.fi/bitstream/handle/123456789/29099/master_Burman_Michael_2017.pdf?sequence=1)
|
||||
and released in https://github.com/burmanm/gorilla-tsc.
|
||||
|
||||
|
||||
### Differences from the original paper
|
||||
- Maximum number of leading zeros is stored with 6 bits to allow up to 63 leading zeros, which are necessary when storing long values.
|
||||
|
||||
- Timestamp delta-of-delta is stored by first turning it to a positive integer with ZigZag encoding and then reduced by one to fit in the necessary bits. In the decoding phase all the values are incremented by one to fetch the original value.
|
||||
|
||||
- The compressed blocks are created with a 27 bit delta header (unlike in the original paper, which uses a 14 bit delta header). This allows to use up to one day block size using millisecond precision.
|
||||
|
||||
## Getting started
|
||||
|
||||
This library is written in Go language, please refer to the guides in https://golang.org for getting started.
|
||||
|
||||
This project include a Makefile that allows you to test and build the project with simple commands.
|
||||
To see all available options:
|
||||
```bash
|
||||
make help
|
||||
```
|
||||
|
||||
## Running all tests
|
||||
|
||||
Before committing the code, please check if it passes all tests using
|
||||
```bash
|
||||
make qa
|
||||
```
|
||||
|
|
@ -1 +0,0 @@
|
|||
1.0.0
|
||||
|
|
@ -1,222 +0,0 @@
|
|||
package tsz
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
)
|
||||
|
||||
// bstream is a stream of bits
|
||||
type bstream struct {
|
||||
// the data stream
|
||||
stream []byte
|
||||
|
||||
// how many bits are valid in current byte
|
||||
count uint8
|
||||
}
|
||||
|
||||
func newBReader(b []byte) *bstream {
|
||||
return &bstream{stream: b, count: 8}
|
||||
}
|
||||
|
||||
func newBWriter(size int) *bstream {
|
||||
return &bstream{stream: make([]byte, 0, size), count: 0}
|
||||
}
|
||||
|
||||
func (b *bstream) clone() *bstream {
|
||||
d := make([]byte, len(b.stream))
|
||||
copy(d, b.stream)
|
||||
return &bstream{stream: d, count: b.count}
|
||||
}
|
||||
|
||||
func (b *bstream) bytes() []byte {
|
||||
return b.stream
|
||||
}
|
||||
|
||||
type bit bool
|
||||
|
||||
const (
|
||||
zero bit = false
|
||||
one bit = true
|
||||
)
|
||||
|
||||
func (b *bstream) writeBit(bit bit) {
|
||||
|
||||
if b.count == 0 {
|
||||
b.stream = append(b.stream, 0)
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
i := len(b.stream) - 1
|
||||
|
||||
if bit {
|
||||
b.stream[i] |= 1 << (b.count - 1)
|
||||
}
|
||||
|
||||
b.count--
|
||||
}
|
||||
|
||||
func (b *bstream) writeByte(byt byte) {
|
||||
|
||||
if b.count == 0 {
|
||||
b.stream = append(b.stream, 0)
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
i := len(b.stream) - 1
|
||||
|
||||
// fill up b.b with b.count bits from byt
|
||||
b.stream[i] |= byt >> (8 - b.count)
|
||||
|
||||
b.stream = append(b.stream, 0)
|
||||
i++
|
||||
b.stream[i] = byt << b.count
|
||||
}
|
||||
|
||||
func (b *bstream) writeBits(u uint64, nbits int) {
|
||||
u <<= (64 - uint(nbits))
|
||||
for nbits >= 8 {
|
||||
byt := byte(u >> 56)
|
||||
b.writeByte(byt)
|
||||
u <<= 8
|
||||
nbits -= 8
|
||||
}
|
||||
|
||||
for nbits > 0 {
|
||||
b.writeBit((u >> 63) == 1)
|
||||
u <<= 1
|
||||
nbits--
|
||||
}
|
||||
}
|
||||
|
||||
func (b *bstream) readBit() (bit, error) {
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return false, io.EOF
|
||||
}
|
||||
|
||||
if b.count == 0 {
|
||||
b.stream = b.stream[1:]
|
||||
// did we just run out of stuff to read?
|
||||
if len(b.stream) == 0 {
|
||||
return false, io.EOF
|
||||
}
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
b.count--
|
||||
d := b.stream[0] & 0x80
|
||||
b.stream[0] <<= 1
|
||||
return d != 0, nil
|
||||
}
|
||||
|
||||
func (b *bstream) readByte() (byte, error) {
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
if b.count == 0 {
|
||||
b.stream = b.stream[1:]
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
if b.count == 8 {
|
||||
b.count = 0
|
||||
return b.stream[0], nil
|
||||
}
|
||||
|
||||
byt := b.stream[0]
|
||||
b.stream = b.stream[1:]
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
|
||||
byt |= b.stream[0] >> b.count
|
||||
b.stream[0] <<= (8 - b.count)
|
||||
|
||||
return byt, nil
|
||||
}
|
||||
|
||||
func (b *bstream) readBits(nbits int) (uint64, error) {
|
||||
|
||||
var u uint64
|
||||
|
||||
for nbits >= 8 {
|
||||
byt, err := b.readByte()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
u = (u << 8) | uint64(byt)
|
||||
nbits -= 8
|
||||
}
|
||||
|
||||
if nbits == 0 {
|
||||
return u, nil
|
||||
}
|
||||
|
||||
if nbits > int(b.count) {
|
||||
u = (u << uint(b.count)) | uint64(b.stream[0]>>(8-b.count))
|
||||
nbits -= int(b.count)
|
||||
b.stream = b.stream[1:]
|
||||
|
||||
if len(b.stream) == 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
b.count = 8
|
||||
}
|
||||
|
||||
u = (u << uint(nbits)) | uint64(b.stream[0]>>(8-uint(nbits)))
|
||||
b.stream[0] <<= uint(nbits)
|
||||
b.count -= uint8(nbits)
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// Read until next unset bit is found or until nbits bits have been read.
|
||||
func (b *bstream) readUntilZero(nbits int) (uint64, error) {
|
||||
var u uint64
|
||||
for i := 0; i < nbits; i++ {
|
||||
u <<= 1
|
||||
bit, err := b.readBit()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if bit == zero {
|
||||
break
|
||||
}
|
||||
u |= 1
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface
|
||||
func (b *bstream) MarshalBinary() ([]byte, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
err := binary.Write(buf, binary.BigEndian, b.count)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = binary.Write(buf, binary.BigEndian, b.stream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
|
||||
func (b *bstream) UnmarshalBinary(bIn []byte) error {
|
||||
buf := bytes.NewReader(bIn)
|
||||
err := binary.Read(buf, binary.BigEndian, &b.count)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
b.stream = make([]byte, buf.Len())
|
||||
return binary.Read(buf, binary.BigEndian, &b.stream)
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
// +build gofuzz
|
||||
|
||||
package tsz
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/tsenart/go-tsz/testdata"
|
||||
)
|
||||
|
||||
func Fuzz(data []byte) int {
|
||||
|
||||
fuzzUnpack(data)
|
||||
|
||||
if len(data) < 9 {
|
||||
return 0
|
||||
}
|
||||
|
||||
t0 := uint32(1456236677)
|
||||
|
||||
v := float64(10000)
|
||||
|
||||
var vals []testdata.Point
|
||||
s := New(t0)
|
||||
t := t0
|
||||
for len(data) >= 10 {
|
||||
tdelta := uint32(binary.LittleEndian.Uint16(data))
|
||||
if t == t0 {
|
||||
tdelta &= (1 << 14) - 1
|
||||
}
|
||||
t += tdelta
|
||||
data = data[2:]
|
||||
v += float64(int16(binary.LittleEndian.Uint16(data))) + float64(binary.LittleEndian.Uint16(data[2:]))/float64(math.MaxUint16)
|
||||
data = data[8:]
|
||||
vals = append(vals, testdata.Point{V: v, T: t})
|
||||
s.Push(t, v)
|
||||
}
|
||||
|
||||
it := s.Iter()
|
||||
|
||||
var i int
|
||||
for it.Next() {
|
||||
gt, gv := it.Values()
|
||||
if gt != vals[i].T || (gv != vals[i].V || math.IsNaN(gv) && math.IsNaN(vals[i].V)) {
|
||||
panic(fmt.Sprintf("failure: gt=%v vals[i].T=%v gv=%v vals[i].V=%v", gt, vals[i].T, gv, vals[i].V))
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
if i != len(vals) {
|
||||
panic("extra data")
|
||||
}
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
func fuzzUnpack(data []byte) {
|
||||
|
||||
it, err := NewIterator(data)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for it.Next() {
|
||||
_, _ = it.Values()
|
||||
}
|
||||
}
|
||||
|
|
@ -1,264 +0,0 @@
|
|||
package testdata
|
||||
|
||||
type Point struct {
|
||||
T uint64
|
||||
V float64
|
||||
}
|
||||
|
||||
var Data = []Point{
|
||||
{1500400800000, 60000},
|
||||
{1500405481623, 69087},
|
||||
{1500405488693, 65640},
|
||||
{1500405495993, 58155},
|
||||
{1500405503743, 61025},
|
||||
{1500405511623, 91156},
|
||||
{1500405519803, 37516},
|
||||
{1500405528313, 93515},
|
||||
{1500405537233, 96226},
|
||||
{1500405546453, 23833},
|
||||
{1500405556103, 73186},
|
||||
{1500405566143, 96947},
|
||||
{1500405576163, 46927},
|
||||
{1500405586173, 77954},
|
||||
{1500405596183, 29302},
|
||||
{1500405606213, 6700},
|
||||
{1500405616163, 71971},
|
||||
{1500405625813, 8528},
|
||||
{1500405635763, 85321},
|
||||
{1500405645634, 83229},
|
||||
{1500405655633, 78298},
|
||||
{1500405665623, 87122},
|
||||
{1500405675623, 82055},
|
||||
{1500405685723, 75067},
|
||||
{1500405695663, 33680},
|
||||
{1500405705743, 17576},
|
||||
{1500405715813, 89701},
|
||||
{1500405725773, 21427},
|
||||
{1500405735883, 58255},
|
||||
{1500405745903, 3768},
|
||||
{1500405755863, 62086},
|
||||
{1500405765843, 66965},
|
||||
{1500405775773, 35801},
|
||||
{1500405785883, 72169},
|
||||
{1500405795843, 43089},
|
||||
{1500405805733, 31418},
|
||||
{1500405815853, 84781},
|
||||
{1500405825963, 36103},
|
||||
{1500405836004, 87431},
|
||||
{1500405845953, 7379},
|
||||
{1500405855913, 66919},
|
||||
{1500405865963, 30906},
|
||||
{1500405875953, 88630},
|
||||
{1500405885943, 27546},
|
||||
{1500405896033, 43813},
|
||||
{1500405906094, 2124},
|
||||
{1500405916063, 49399},
|
||||
{1500405926143, 94577},
|
||||
{1500405936123, 98459},
|
||||
{1500405946033, 49457},
|
||||
{1500405956023, 92838},
|
||||
{1500405966023, 15628},
|
||||
{1500405976043, 53916},
|
||||
{1500405986063, 90387},
|
||||
{1500405996123, 43176},
|
||||
{1500406006123, 18838},
|
||||
{1500406016174, 78847},
|
||||
{1500406026173, 39591},
|
||||
{1500406036004, 77070},
|
||||
{1500406045964, 56788},
|
||||
{1500406056043, 96706},
|
||||
{1500406066123, 20756},
|
||||
{1500406076113, 64433},
|
||||
{1500406086133, 45791},
|
||||
{1500406096123, 75028},
|
||||
{1500406106193, 55403},
|
||||
{1500406116213, 36991},
|
||||
{1500406126073, 92929},
|
||||
{1500406136103, 60416},
|
||||
{1500406146183, 55485},
|
||||
{1500406156383, 53525},
|
||||
{1500406166313, 96021},
|
||||
{1500406176414, 22705},
|
||||
{1500406186613, 89801},
|
||||
{1500406196543, 51975},
|
||||
{1500406206483, 86741},
|
||||
{1500406216483, 22440},
|
||||
{1500406226433, 51818},
|
||||
{1500406236403, 61965},
|
||||
{1500406246413, 19074},
|
||||
{1500406256494, 54521},
|
||||
{1500406266413, 59315},
|
||||
{1500406276303, 19171},
|
||||
{1500406286213, 98800},
|
||||
{1500406296183, 7086},
|
||||
{1500406306103, 60578},
|
||||
{1500406316073, 96828},
|
||||
{1500406326143, 83746},
|
||||
{1500406336153, 85481},
|
||||
{1500406346113, 22346},
|
||||
{1500406356133, 80976},
|
||||
{1500406366065, 43586},
|
||||
{1500406376074, 82500},
|
||||
{1500406386184, 13576},
|
||||
{1500406396113, 77871},
|
||||
{1500406406094, 60978},
|
||||
{1500406416203, 35264},
|
||||
{1500406426323, 79733},
|
||||
{1500406436343, 29140},
|
||||
{1500406446323, 7237},
|
||||
{1500406456344, 52866},
|
||||
{1500406466393, 88456},
|
||||
{1500406476493, 33533},
|
||||
{1500406486524, 96961},
|
||||
{1500406496453, 16389},
|
||||
{1500406506453, 31181},
|
||||
{1500406516433, 63282},
|
||||
{1500406526433, 92857},
|
||||
{1500406536413, 4582},
|
||||
{1500406546383, 46832},
|
||||
{1500406556473, 6335},
|
||||
{1500406566413, 44367},
|
||||
{1500406576513, 84640},
|
||||
{1500406586523, 36174},
|
||||
{1500406596553, 40075},
|
||||
{1500406606603, 80886},
|
||||
{1500406616623, 43784},
|
||||
{1500406626623, 25077},
|
||||
{1500406636723, 18617},
|
||||
{1500406646723, 72681},
|
||||
{1500406656723, 84811},
|
||||
{1500406666783, 90053},
|
||||
{1500406676685, 25708},
|
||||
{1500406686713, 57134},
|
||||
{1500406696673, 87193},
|
||||
{1500406706743, 66057},
|
||||
{1500406716724, 51404},
|
||||
{1500406726753, 90141},
|
||||
{1500406736813, 10434},
|
||||
{1500406746803, 29056},
|
||||
{1500406756833, 48160},
|
||||
{1500406766924, 96652},
|
||||
{1500406777113, 64141},
|
||||
{1500406787113, 22143},
|
||||
{1500406797093, 20561},
|
||||
{1500406807113, 66401},
|
||||
{1500406817283, 76802},
|
||||
{1500406827284, 37555},
|
||||
{1500406837323, 63169},
|
||||
{1500406847463, 45712},
|
||||
{1500406857513, 44751},
|
||||
{1500406867523, 98891},
|
||||
{1500406877523, 38122},
|
||||
{1500406887623, 46202},
|
||||
{1500406897703, 5875},
|
||||
{1500406907663, 17397},
|
||||
{1500406917603, 39994},
|
||||
{1500406927633, 82385},
|
||||
{1500406937623, 15598},
|
||||
{1500406947693, 36235},
|
||||
{1500406957703, 97536},
|
||||
{1500406967673, 28557},
|
||||
{1500406977723, 13985},
|
||||
{1500406987663, 64304},
|
||||
{1500406997573, 83693},
|
||||
{1500407007494, 6574},
|
||||
{1500407017493, 25134},
|
||||
{1500407027503, 50383},
|
||||
{1500407037523, 55922},
|
||||
{1500407047603, 73436},
|
||||
{1500407057473, 68235},
|
||||
{1500407067553, 1469},
|
||||
{1500407077463, 44315},
|
||||
{1500407087463, 95064},
|
||||
{1500407097443, 1997},
|
||||
{1500407107473, 17247},
|
||||
{1500407117453, 42454},
|
||||
{1500407127413, 73631},
|
||||
{1500407137363, 96890},
|
||||
{1500407147343, 43450},
|
||||
{1500407157363, 42042},
|
||||
{1500407167403, 83014},
|
||||
{1500407177473, 32051},
|
||||
{1500407187523, 69280},
|
||||
{1500407197495, 21425},
|
||||
{1500407207453, 93748},
|
||||
{1500407217413, 64151},
|
||||
{1500407227443, 38791},
|
||||
{1500407237463, 5248},
|
||||
{1500407247523, 92935},
|
||||
{1500407257513, 18516},
|
||||
{1500407267584, 98870},
|
||||
{1500407277573, 82244},
|
||||
{1500407287723, 65464},
|
||||
{1500407297723, 33801},
|
||||
{1500407307673, 18331},
|
||||
{1500407317613, 89744},
|
||||
{1500407327553, 98460},
|
||||
{1500407337503, 24709},
|
||||
{1500407347423, 8407},
|
||||
{1500407357383, 69451},
|
||||
{1500407367333, 51100},
|
||||
{1500407377373, 25309},
|
||||
{1500407387443, 16148},
|
||||
{1500407397453, 98974},
|
||||
{1500407407543, 80284},
|
||||
{1500407417583, 170},
|
||||
{1500407427453, 34706},
|
||||
{1500407437433, 39681},
|
||||
{1500407447603, 6140},
|
||||
{1500407457513, 64595},
|
||||
{1500407467564, 59862},
|
||||
{1500407477563, 53795},
|
||||
{1500407487593, 83493},
|
||||
{1500407497584, 90639},
|
||||
{1500407507623, 16777},
|
||||
{1500407517613, 11096},
|
||||
{1500407527673, 38512},
|
||||
{1500407537963, 52759},
|
||||
{1500407548023, 79567},
|
||||
{1500407558033, 48664},
|
||||
{1500407568113, 10710},
|
||||
{1500407578164, 25635},
|
||||
{1500407588213, 40985},
|
||||
{1500407598163, 94089},
|
||||
{1500407608163, 50056},
|
||||
{1500407618223, 15550},
|
||||
{1500407628143, 78823},
|
||||
{1500407638223, 9044},
|
||||
{1500407648173, 20782},
|
||||
{1500407658023, 86390},
|
||||
{1500407667903, 79444},
|
||||
{1500407677903, 84051},
|
||||
{1500407687923, 91554},
|
||||
{1500407697913, 58777},
|
||||
{1500407708003, 89474},
|
||||
{1500407718083, 94026},
|
||||
{1500407728034, 41613},
|
||||
{1500407738083, 64667},
|
||||
{1500407748034, 5160},
|
||||
{1500407758003, 45140},
|
||||
{1500407768033, 53704},
|
||||
{1500407778083, 68097},
|
||||
{1500407788043, 81137},
|
||||
{1500407798023, 59657},
|
||||
{1500407808033, 56572},
|
||||
{1500407817983, 1993},
|
||||
{1500407828063, 62608},
|
||||
{1500407838213, 76489},
|
||||
{1500407848203, 22147},
|
||||
{1500407858253, 92829},
|
||||
{1500407868073, 48499},
|
||||
{1500407878053, 89152},
|
||||
{1500407888073, 9191},
|
||||
{1500407898033, 49881},
|
||||
{1500407908113, 96020},
|
||||
{1500407918213, 90203},
|
||||
{1500407928234, 32217},
|
||||
{1500407938253, 94302},
|
||||
{1500407948293, 83111},
|
||||
{1500407958234, 75576},
|
||||
{1500407968073, 5973},
|
||||
{1500407978023, 5175},
|
||||
{1500407987923, 63350},
|
||||
{1500407997833, 44081},
|
||||
}
|
||||
|
|
@ -1,385 +0,0 @@
|
|||
// Package tsz implement time-series compression
|
||||
/*
|
||||
|
||||
http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
|
||||
|
||||
*/
|
||||
package tsz
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"math"
|
||||
"math/bits"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Series is the basic series primitive
|
||||
// you can concurrently put values, finish the stream, and create iterators
|
||||
type Series struct {
|
||||
sync.Mutex
|
||||
|
||||
T0 uint64
|
||||
t uint64
|
||||
val float64
|
||||
|
||||
bw bstream
|
||||
leading uint8
|
||||
trailing uint8
|
||||
finished bool
|
||||
|
||||
tDelta uint32
|
||||
}
|
||||
|
||||
// New series
|
||||
func New(t0 uint64) *Series {
|
||||
s := Series{
|
||||
T0: t0,
|
||||
leading: ^uint8(0),
|
||||
}
|
||||
|
||||
// block header
|
||||
s.bw.writeBits(uint64(t0), 64)
|
||||
|
||||
return &s
|
||||
|
||||
}
|
||||
|
||||
// Bytes value of the series stream
|
||||
func (s *Series) Bytes() []byte {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
return s.bw.bytes()
|
||||
}
|
||||
|
||||
func finish(w *bstream) {
|
||||
// write an end-of-stream record
|
||||
w.writeBits(0x0f, 4)
|
||||
w.writeBits(0xffffffff, 32)
|
||||
w.writeBit(zero)
|
||||
}
|
||||
|
||||
// Finish the series by writing an end-of-stream record
|
||||
func (s *Series) Finish() {
|
||||
s.Lock()
|
||||
if !s.finished {
|
||||
finish(&s.bw)
|
||||
s.finished = true
|
||||
}
|
||||
s.Unlock()
|
||||
}
|
||||
|
||||
// Push a timestamp and value to the series.
|
||||
// Values must be inserted in monotonically increasing time order.
|
||||
func (s *Series) Push(t uint64, v float64) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
if s.t == 0 {
|
||||
// first point
|
||||
s.t = t
|
||||
s.val = v
|
||||
s.tDelta = uint32(t - s.T0)
|
||||
s.bw.writeBits(uint64(s.tDelta), 27)
|
||||
s.bw.writeBits(math.Float64bits(v), 64)
|
||||
return
|
||||
}
|
||||
|
||||
// Difference to the original Facebook paper, we store the first delta as 27
|
||||
// bits to allow millisecond accuracy for a one day block.
|
||||
tDelta := uint32(t - s.t)
|
||||
d := int32(tDelta - s.tDelta)
|
||||
|
||||
if d == 0 {
|
||||
s.bw.writeBit(zero)
|
||||
} else {
|
||||
// Increase by one in the decompressing phase as we have one free bit
|
||||
switch dod := encodeZigZag32(d) - 1; 32 - bits.LeadingZeros32(dod) {
|
||||
case 1, 2, 3, 4, 5, 6, 7:
|
||||
s.bw.writeBits(uint64(dod|256), 9) // dod | 00000000000000000000000100000000
|
||||
case 8, 9:
|
||||
s.bw.writeBits(uint64(dod|3072), 12) // dod | 00000000000000000000110000000000
|
||||
case 10, 11, 12:
|
||||
s.bw.writeBits(uint64(dod|57344), 16) // dod | 00000000000000001110000000000000
|
||||
default:
|
||||
s.bw.writeBits(0x0f, 4) // '1111'
|
||||
s.bw.writeBits(uint64(dod), 32)
|
||||
}
|
||||
}
|
||||
|
||||
vDelta := math.Float64bits(s.val) ^ math.Float64bits(v)
|
||||
|
||||
if vDelta == 0 {
|
||||
s.bw.writeBit(zero)
|
||||
} else {
|
||||
leading := uint8(bits.LeadingZeros64(vDelta))
|
||||
trailing := uint8(bits.TrailingZeros64(vDelta))
|
||||
|
||||
s.bw.writeBit(one)
|
||||
|
||||
if leading >= s.leading && trailing >= s.trailing {
|
||||
s.bw.writeBit(zero)
|
||||
s.bw.writeBits(vDelta>>s.trailing, 64-int(s.leading)-int(s.trailing))
|
||||
} else {
|
||||
s.bw.writeBit(one)
|
||||
|
||||
// Different from version 1.x, use (significantBits - 1) in storage - avoids a branch
|
||||
sigbits := 64 - leading - trailing
|
||||
|
||||
// Different from original, bits 5 -> 6, avoids a branch, allows storing small longs
|
||||
s.bw.writeBits(uint64(leading), 6) // Number of leading zeros in the next 6 bits
|
||||
s.bw.writeBits(uint64(sigbits-1), 6) // Length of meaningful bits in the next 6 bits
|
||||
s.bw.writeBits(vDelta>>trailing, int(sigbits)) // Store the meaningful bits of XOR
|
||||
|
||||
s.leading, s.trailing = leading, trailing
|
||||
}
|
||||
}
|
||||
|
||||
s.tDelta = tDelta
|
||||
s.t = t
|
||||
s.val = v
|
||||
|
||||
}
|
||||
|
||||
// Iter lets you iterate over a series. It is not concurrency-safe.
|
||||
func (s *Series) Iter() *Iter {
|
||||
s.Lock()
|
||||
w := s.bw.clone()
|
||||
s.Unlock()
|
||||
|
||||
finish(w)
|
||||
iter, _ := bstreamIterator(w)
|
||||
return iter
|
||||
}
|
||||
|
||||
// Iter lets you iterate over a series. It is not concurrency-safe.
|
||||
type Iter struct {
|
||||
T0 uint64
|
||||
|
||||
t uint64
|
||||
val float64
|
||||
|
||||
br bstream
|
||||
leading uint8
|
||||
trailing uint8
|
||||
|
||||
finished bool
|
||||
|
||||
tDelta uint32
|
||||
err error
|
||||
}
|
||||
|
||||
func bstreamIterator(br *bstream) (*Iter, error) {
|
||||
|
||||
br.count = 8
|
||||
|
||||
t0, err := br.readBits(64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Iter{
|
||||
T0: uint64(t0),
|
||||
br: *br,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewIterator for the series
|
||||
func NewIterator(b []byte) (*Iter, error) {
|
||||
return bstreamIterator(newBReader(b))
|
||||
}
|
||||
|
||||
// Next iteration of the series iterator
|
||||
func (it *Iter) Next() bool {
|
||||
|
||||
if it.err != nil || it.finished {
|
||||
return false
|
||||
}
|
||||
|
||||
if it.t == 0 {
|
||||
// read first t and v
|
||||
tDelta, err := it.br.readBits(27)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
if tDelta == (1<<27)-1 {
|
||||
it.finished = true
|
||||
return false
|
||||
}
|
||||
|
||||
it.tDelta = uint32(tDelta)
|
||||
it.t = it.T0 + tDelta
|
||||
v, err := it.br.readBits(64)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
it.val = math.Float64frombits(v)
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// read delta-of-delta
|
||||
d, err := it.br.readUntilZero(4)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
if d != 0 {
|
||||
var sz uint
|
||||
switch d {
|
||||
case 0x02:
|
||||
sz = 7
|
||||
case 0x06:
|
||||
sz = 9
|
||||
case 0x0e:
|
||||
sz = 12
|
||||
case 0x0f:
|
||||
sz = 32
|
||||
}
|
||||
|
||||
bits, err := it.br.readBits(int(sz))
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
if sz == 32 && bits == 0xffffffff {
|
||||
it.finished = true
|
||||
return false
|
||||
}
|
||||
|
||||
dod := decodeZigZag32(uint32(int64(bits) + 1))
|
||||
it.tDelta += uint32(dod)
|
||||
}
|
||||
|
||||
it.t += uint64(it.tDelta)
|
||||
|
||||
val, err := it.br.readUntilZero(2)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
|
||||
switch val {
|
||||
case 3:
|
||||
bits, err := it.br.readBits(6)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.leading = uint8(bits)
|
||||
|
||||
bits, err = it.br.readBits(6)
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.trailing = 64 - (uint8(bits) + 1) - it.leading
|
||||
|
||||
fallthrough
|
||||
case 2:
|
||||
bits, err := it.br.readBits(int(64 - it.leading - it.trailing))
|
||||
if err != nil {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
it.val = math.Float64frombits(math.Float64bits(it.val) ^ (bits << it.trailing))
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// Values at the current iterator position
|
||||
func (it *Iter) Values() (uint64, float64) {
|
||||
return it.t, it.val
|
||||
}
|
||||
|
||||
// Err error at the current iterator position
|
||||
func (it *Iter) Err() error {
|
||||
return it.err
|
||||
}
|
||||
|
||||
type errMarshal struct {
|
||||
w io.Writer
|
||||
r io.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
func (em *errMarshal) write(t interface{}) {
|
||||
if em.err != nil {
|
||||
return
|
||||
}
|
||||
em.err = binary.Write(em.w, binary.BigEndian, t)
|
||||
}
|
||||
|
||||
func (em *errMarshal) read(t interface{}) {
|
||||
if em.err != nil {
|
||||
return
|
||||
}
|
||||
em.err = binary.Read(em.r, binary.BigEndian, t)
|
||||
}
|
||||
|
||||
// MarshalBinary implements the encoding.BinaryMarshaler interface
|
||||
func (s *Series) MarshalBinary() ([]byte, error) {
|
||||
buf := new(bytes.Buffer)
|
||||
em := &errMarshal{w: buf}
|
||||
em.write(s.T0)
|
||||
em.write(s.leading)
|
||||
em.write(s.t)
|
||||
em.write(s.tDelta)
|
||||
em.write(s.trailing)
|
||||
em.write(s.val)
|
||||
bStream, err := s.bw.MarshalBinary()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
em.write(bStream)
|
||||
if em.err != nil {
|
||||
return nil, em.err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
|
||||
func (s *Series) UnmarshalBinary(b []byte) error {
|
||||
buf := bytes.NewReader(b)
|
||||
em := &errMarshal{r: buf}
|
||||
em.read(&s.T0)
|
||||
em.read(&s.leading)
|
||||
em.read(&s.t)
|
||||
em.read(&s.tDelta)
|
||||
em.read(&s.trailing)
|
||||
em.read(&s.val)
|
||||
outBuf := make([]byte, buf.Len())
|
||||
em.read(outBuf)
|
||||
err := s.bw.UnmarshalBinary(outBuf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if em.err != nil {
|
||||
return em.err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Maps negative values to positive values while going back and
|
||||
// forth (0 = 0, -1 = 1, 1 = 2, -2 = 3, 2 = 4, -3 = 5, 3 = 6 ...)
|
||||
// Encodes signed integers into unsigned integers that can be efficiently
|
||||
// encoded with varint because negative values must be sign-extended to 64 bits to
|
||||
// be varint encoded, thus always taking 10 bytes on the wire.
|
||||
//
|
||||
// Read more: https://gist.github.com/mfuerstenau/ba870a29e16536fdbaba
|
||||
func encodeZigZag32(n int32) uint32 {
|
||||
// Note: The right-shift must be arithmetic which it is in Go.
|
||||
return uint32(n>>31) ^ (uint32(n) << 1)
|
||||
}
|
||||
|
||||
func decodeZigZag32(n uint32) int32 {
|
||||
return int32((n >> 1) ^ uint32((int32(n&1)<<31)>>31))
|
||||
}
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
*.exe
|
||||
|
||||
vegeta
|
||||
vegeta.test
|
||||
vegeta-*.tar.gz
|
||||
lib/lib.test
|
||||
dist
|
||||
vendor
|
||||
|
||||
*.gob
|
||||
*.lz
|
||||
|
||||
.DS_Store
|
||||
|
|
@ -1,35 +0,0 @@
|
|||
builds:
|
||||
- goos:
|
||||
- freebsd
|
||||
- linux
|
||||
- darwin
|
||||
- windows
|
||||
goarch:
|
||||
- amd64
|
||||
- arm64
|
||||
- arm
|
||||
goarm:
|
||||
- 7
|
||||
env:
|
||||
- CGO_ENABLED=0
|
||||
flags:
|
||||
- -a
|
||||
- -tags="netgo"
|
||||
ldflags:
|
||||
- -s -w -extldflags "-static" -X main.Version={{.Version}} -X main.Commit={{.Commit}} -X main.Date={{.Date}}"
|
||||
|
||||
release:
|
||||
name_template: "v{{.Version}}"
|
||||
|
||||
archive:
|
||||
name_template: "{{.ProjectName}}-{{.Version}}-{{.Os}}-{{.Arch}}"
|
||||
format: tar.gz
|
||||
format_overrides:
|
||||
- goos: windows
|
||||
format: zip
|
||||
|
||||
checksum:
|
||||
name_template: "{{ .ProjectName }}-checksums.txt"
|
||||
|
||||
sign:
|
||||
artifacts: checksum
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
language: go
|
||||
sudo: false
|
||||
go:
|
||||
- tip
|
||||
install:
|
||||
- go get -v golang.org/x/lint/golint
|
||||
- go get -d -t -v ./...
|
||||
- go build -v ./...
|
||||
script:
|
||||
- go vet ./...
|
||||
- $HOME/gopath/bin/golint -set_exit_status $(go list ./... | grep -v /vendor/)
|
||||
- go test -v ./...
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
2018-05-18: v7.0.0
|
||||
Include response body in hit results (#279)
|
||||
Added support for h2c requests (HTTP/2 without TLS) (#261)
|
||||
Prevent "null" Metrics.Errors JSON encoding (#277)
|
||||
Add option to override HTTP Proxy on Attacker (#234)
|
||||
|
||||
2017-03-19: v6.3.0
|
||||
Mark responses as success in no redirect following mode (#222)
|
||||
|
||||
2017-03-04: v6.2.0
|
||||
Allow any upper-case ASCII word to be an HTTP method (#217)
|
||||
Correctly compute Metrics.Rate with sub-second duration results (#208)
|
||||
|
||||
2016-08-26: v6.1.1
|
||||
Respect case sensitivity in target file header names (#195, #191)
|
||||
|
||||
2016-04-03: v6.1.0
|
||||
Add HTTP2 support
|
||||
|
||||
2015-11-27: v6.0.0
|
||||
Insecure attack flag (#160)
|
||||
Client certificates (#156)
|
||||
Infinite attacks (#155)
|
||||
Allow empty lines between targets (#147)
|
||||
|
||||
2015-09-19: v5.9.0
|
||||
Bounded memory streaming reporters (#136)
|
||||
|
||||
2015-09-04: v5.8.1
|
||||
Fix support for DELETE methods in targets
|
||||
|
||||
2015-08-11: v5.8.0
|
||||
Change reporters quantile estimation method to match R's 8th type.
|
||||
|
||||
2015-05-23: v5.7.1
|
||||
Revert end-to-end attack timeout change
|
||||
|
||||
2015-05-23: v5.7.0
|
||||
Allow case sensitve headers in attacks
|
||||
|
||||
2015-04-15: v5.6.3
|
||||
Expose connections flag in the attack command
|
||||
Add global cpu and heap profiling flags
|
||||
Measure actual attack rate and print it in relevant reporters
|
||||
Major performance improvements that allow much higher attack rates
|
||||
|
||||
2015-04-02: v5.6.2
|
||||
Update dygraph to latest version
|
||||
Improve plot reporter screenshot rendering by using html2canvas.js
|
||||
Improve plot reporter performance
|
||||
|
||||
2015-03-23: v5.6.1
|
||||
Allow spaces in hist reporter flag format
|
||||
|
||||
2015-03-12: v5.6.0
|
||||
Set default dumper to "json" in the dump command.
|
||||
Add --version to global vegeta command flags.
|
||||
Fix response body leak regression introduced in v5.5.3.
|
||||
|
||||
2015-03-11: v5.5.3
|
||||
Always read response bodies for each request.
|
||||
Homebrew install instructions.
|
||||
|
||||
2015-01-3: v5.5.2
|
||||
Refactor core request logic and simplify tests with a 4x speedup.
|
||||
|
||||
2015-01-2: v5.5.1
|
||||
Treat bad status codes as errors.
|
||||
|
||||
2014-11-21: v5.5.0
|
||||
Implement dump command with CSV and JSON record format.
|
||||
Optionally ignore redirects and treat them as successes.
|
||||
|
||||
2014-11-16: v5.4.0
|
||||
Add histogram reporter to the report command.
|
||||
|
||||
2014-11-16: v5.3.0
|
||||
Add support for extended targets dsl that supports per-target headers and body.
|
||||
Target file comments support has been removed.
|
||||
|
||||
2014-11-7: v5.2.0
|
||||
Don't treat 3xx status codes as errors.
|
||||
Add -keepalive flag to the attack command.
|
||||
|
||||
2014-11-3: v5.1.1
|
||||
Add FreeBSD and Windows releases.
|
||||
Fix non termination bug in the report command. #85
|
||||
|
|
@ -1,169 +0,0 @@
|
|||
# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
|
||||
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:c312c313bb8040fcfb5698f38b51ec279e4635c7619773e6f018c908586c64b1"
|
||||
name = "github.com/alecthomas/jsonschema"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "f2c93856175a7dd6abe88c5c3900b67ad054adcc"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:f31b92722c6eca05df9ba28e9cb38d94de0865f034fff164557f103d6d13b612"
|
||||
name = "github.com/bmizerany/perks"
|
||||
packages = ["quantile"]
|
||||
pruneopts = "UT"
|
||||
revision = "d9a9656a3a4b1c2864fdb44db2ef8619772d92aa"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:e8891cd354f2af940345f895d5698ab1ed439196d4f69eb44817fc91e80354c1"
|
||||
name = "github.com/c2h5oh/datasize"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "4eba002a5eaea69cf8d235a388fc6b65ae68d2dd"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:5c4cefdd3ba39aee4e4b6f9aad121a79fd2a989b9b54acbe4f530cc3a700cd75"
|
||||
name = "github.com/dgryski/go-gk"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "201884a44051d8544b65e6b1c05ae71c0c45b9e0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:bc4e2b6c1af97c03fe850864bf22574abc6ad0fcbf0e5dda0fd246d3d43f9ccb"
|
||||
name = "github.com/dgryski/go-lttb"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "318fcdf10a77c2df09d83777e41f5e18d236ec9d"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:2e3c336fc7fde5c984d2841455a658a6d626450b1754a854b3b32e7a8f49a07a"
|
||||
name = "github.com/google/go-cmp"
|
||||
packages = [
|
||||
"cmp",
|
||||
"cmp/internal/diff",
|
||||
"cmp/internal/function",
|
||||
"cmp/internal/value",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "3af367b6b30c263d47e8895973edcca9a49cf029"
|
||||
version = "v0.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:b07e7ba2bb99a70ce3318ea387a62666371c1ab4f9bfcae192a59e3f4b8ffa56"
|
||||
name = "github.com/influxdata/tdigest"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "a7d76c6f093a59b94a01c6c2b8429122d444a8cc"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:aa3d8d42865c42626b5c1add193692d045b3188b1479f0a0a88690d21fe20083"
|
||||
name = "github.com/mailru/easyjson"
|
||||
packages = [
|
||||
".",
|
||||
"buffer",
|
||||
"jlexer",
|
||||
"jwriter",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "60711f1a8329503b04e1c88535f419d0bb440bff"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:7ca2584fa7da0520cd2d1136a10194fe5a5b220bdb215074ab6f7b5ad91115f4"
|
||||
name = "github.com/shurcooL/httpfs"
|
||||
packages = ["vfsutil"]
|
||||
pruneopts = "UT"
|
||||
revision = "809beceb23714880abc4a382a00c05f89d13b1cc"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:b5db31f108c72b9de4e36ca3222a4e325beebfa492a9d3f62f313ffa9cdab3b7"
|
||||
name = "github.com/shurcooL/vfsgen"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "62bca832be04bd2bcaabd3b68a6b19a7ec044411"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:383e656d30a7bd2c663514940748c189307134c65ec8c7b2070ed01abb93e73d"
|
||||
name = "github.com/streadway/quantile"
|
||||
packages = ["."]
|
||||
pruneopts = "UT"
|
||||
revision = "b0c588724d25ae13f5afb3d90efec0edc636432b"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:be87c1e6f0a15e5b1c45036b9382cdd63057a8b65e10ff2fe2bda23886a99c6a"
|
||||
name = "github.com/tsenart/go-tsz"
|
||||
packages = [
|
||||
".",
|
||||
"testdata",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "cdeb9e1e981e85dafc95428f5da9cba59cbcc828"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:dca4094c3be476189bd5854752edbd1bd3d15e61d9570dd9133744bb1ecd599d"
|
||||
name = "golang.org/x/net"
|
||||
packages = [
|
||||
"http/httpguts",
|
||||
"http2",
|
||||
"http2/hpack",
|
||||
"idna",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "c39426892332e1bb5ec0a434a079bf82f5d30c54"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18"
|
||||
name = "golang.org/x/text"
|
||||
packages = [
|
||||
"collate",
|
||||
"collate/build",
|
||||
"internal/colltab",
|
||||
"internal/gen",
|
||||
"internal/tag",
|
||||
"internal/triegen",
|
||||
"internal/ucd",
|
||||
"language",
|
||||
"secure/bidirule",
|
||||
"transform",
|
||||
"unicode/bidi",
|
||||
"unicode/cldr",
|
||||
"unicode/norm",
|
||||
"unicode/rangetable",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
|
||||
version = "v0.3.0"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
input-imports = [
|
||||
"github.com/alecthomas/jsonschema",
|
||||
"github.com/bmizerany/perks/quantile",
|
||||
"github.com/c2h5oh/datasize",
|
||||
"github.com/dgryski/go-gk",
|
||||
"github.com/dgryski/go-lttb",
|
||||
"github.com/google/go-cmp/cmp",
|
||||
"github.com/influxdata/tdigest",
|
||||
"github.com/mailru/easyjson",
|
||||
"github.com/mailru/easyjson/jlexer",
|
||||
"github.com/mailru/easyjson/jwriter",
|
||||
"github.com/shurcooL/vfsgen",
|
||||
"github.com/streadway/quantile",
|
||||
"github.com/tsenart/go-tsz",
|
||||
"golang.org/x/net/http2",
|
||||
]
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
# Gopkg.toml example
|
||||
#
|
||||
# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md
|
||||
# for detailed Gopkg.toml documentation.
|
||||
#
|
||||
# required = ["github.com/user/thing/cmd/thing"]
|
||||
# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"]
|
||||
#
|
||||
# [[constraint]]
|
||||
# name = "github.com/user/project"
|
||||
# version = "1.0.0"
|
||||
#
|
||||
# [[constraint]]
|
||||
# name = "github.com/user/project2"
|
||||
# branch = "dev"
|
||||
# source = "github.com/myfork/project2"
|
||||
#
|
||||
# [[override]]
|
||||
# name = "github.com/x/y"
|
||||
# version = "2.4.0"
|
||||
#
|
||||
# [prune]
|
||||
# non-go = false
|
||||
# go-tests = true
|
||||
# unused-packages = true
|
||||
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/alecthomas/jsonschema"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/bmizerany/perks"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/dgryski/go-gk"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/dgryski/go-lttb"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/tsenart/go-tsz"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/influxdata/tdigest"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/streadway/quantile"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "golang.org/x/net"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/shurcooL/vfsgen"
|
||||
|
||||
[prune]
|
||||
go-tests = true
|
||||
unused-packages = true
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/c2h5oh/datasize"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/mailru/easyjson"
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2013-2016 Tomás Senart
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
this software and associated documentation files (the "Software"), to deal in
|
||||
the Software without restriction, including without limitation the rights to
|
||||
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
COMMIT=$(shell git rev-parse HEAD)
|
||||
VERSION=$(shell git describe --tags --exact-match --always)
|
||||
DATE=$(shell date +'%FT%TZ%z')
|
||||
|
||||
vegeta: vendor generate
|
||||
CGO_ENABLED=0 go build -v -a -tags=netgo \
|
||||
-ldflags '-s -w -extldflags "-static" -X main.Version=$(VERSION) -X main.Commit=$(COMMIT) -X main.Date=$(DATE)'
|
||||
|
||||
clean-vegeta:
|
||||
rm vegeta
|
||||
|
||||
generate: vendor
|
||||
go generate ./...
|
||||
|
||||
vendor:
|
||||
dep ensure -v
|
||||
|
||||
clean-vendor:
|
||||
rm -rf vendor
|
||||
|
||||
dist:
|
||||
goreleaser release --debug --skip-validate
|
||||
|
||||
clean-dist:
|
||||
rm -rf dist
|
||||
|
|
@ -1,803 +0,0 @@
|
|||
# Vegeta [](http://travis-ci.org/tsenart/vegeta) [](https://goreportcard.com/report/github.com/tsenart/vegeta) [](https://godoc.org/github.com/tsenart/vegeta) [](https://gitter.im/tsenart/vegeta?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [](#donate)
|
||||
|
||||
Vegeta is a versatile HTTP load testing tool built out of a need to drill
|
||||
HTTP services with a constant request rate.
|
||||
It can be used both as a command line utility and a library.
|
||||
|
||||
|
||||
|
||||
## Install
|
||||
|
||||
### Pre-compiled executables
|
||||
|
||||
Get them [here](http://github.com/tsenart/vegeta/releases).
|
||||
|
||||
### Homebrew on Mac OS X
|
||||
|
||||
You can install Vegeta using the [Homebrew](https://github.com/Homebrew/homebrew/) package manager on Mac OS X:
|
||||
|
||||
```shell
|
||||
$ brew update && brew install vegeta
|
||||
```
|
||||
|
||||
### Source
|
||||
|
||||
You need `go` installed and `GOBIN` in your `PATH`. Once that is done, run the
|
||||
command:
|
||||
|
||||
```shell
|
||||
$ go get -u github.com/tsenart/vegeta
|
||||
```
|
||||
|
||||
## Versioning
|
||||
|
||||
Both the library and the CLI are versioned with [SemVer v2.0.0](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
After [v8.0.0](https://github.com/tsenart/vegeta/tree/v8.0.0), the two components
|
||||
are versioned separately to better isolate breaking changes to each.
|
||||
|
||||
CLI releases are tagged with `cli/vMAJOR.MINOR.PATCH` and published on the [Github releases page](https://github.com/tsenart/vegeta/releases).
|
||||
As for the library, new versions are tagged with both `lib/vMAJOR.MINOR.PATCH` and `vMAJOR.MINOR.PATCH`.
|
||||
The latter tag is required for compatibility with `go mod`.
|
||||
|
||||
## Contributing
|
||||
|
||||
See [CONTRIBUTING.md](.github/CONTRIBUTING.md).
|
||||
|
||||
## Usage manual
|
||||
|
||||
```console
|
||||
Usage: vegeta [global flags] <command> [command flags]
|
||||
|
||||
global flags:
|
||||
-cpus int
|
||||
Number of CPUs to use (defaults to the number of CPUs you have)
|
||||
-profile string
|
||||
Enable profiling of [cpu, heap]
|
||||
-version
|
||||
Print version and exit
|
||||
|
||||
attack command:
|
||||
-body string
|
||||
Requests body file
|
||||
-cert string
|
||||
TLS client PEM encoded certificate file
|
||||
-connections int
|
||||
Max open idle connections per target host (default 10000)
|
||||
-duration duration
|
||||
Duration of the test [0 = forever]
|
||||
-format string
|
||||
Targets format [http, json] (default "http")
|
||||
-h2c
|
||||
Send HTTP/2 requests without TLS encryption
|
||||
-header value
|
||||
Request header
|
||||
-http2
|
||||
Send HTTP/2 requests when supported by the server (default true)
|
||||
-insecure
|
||||
Ignore invalid server TLS certificates
|
||||
-keepalive
|
||||
Use persistent connections (default true)
|
||||
-key string
|
||||
TLS client PEM encoded private key file
|
||||
-laddr value
|
||||
Local IP address (default 0.0.0.0)
|
||||
-lazy
|
||||
Read targets lazily
|
||||
-max-body value
|
||||
Maximum number of bytes to capture from response bodies. [-1 = no limit] (default -1)
|
||||
-max-workers uint
|
||||
Maximum number of workers (default 18446744073709551615)
|
||||
-name string
|
||||
Attack name
|
||||
-output string
|
||||
Output file (default "stdout")
|
||||
-rate value
|
||||
Number of requests per time unit [0 = infinity] (default 50/1s)
|
||||
-redirects int
|
||||
Number of redirects to follow. -1 will not follow but marks as success (default 10)
|
||||
-resolvers value
|
||||
List of addresses (ip:port) to use for DNS resolution. Disables use of local system DNS. (comma separated list)
|
||||
-root-certs value
|
||||
TLS root certificate files (comma separated list)
|
||||
-targets string
|
||||
Targets file (default "stdin")
|
||||
-timeout duration
|
||||
Requests timeout (default 30s)
|
||||
-unix-socket string
|
||||
Connect over a unix socket. This overrides the host address in target URLs
|
||||
-workers uint
|
||||
Initial number of workers (default 10)
|
||||
|
||||
encode command:
|
||||
-output string
|
||||
Output file (default "stdout")
|
||||
-to string
|
||||
Output encoding [csv, gob, json] (default "json")
|
||||
|
||||
plot command:
|
||||
-output string
|
||||
Output file (default "stdout")
|
||||
-threshold int
|
||||
Threshold of data points above which series are downsampled. (default 4000)
|
||||
-title string
|
||||
Title and header of the resulting HTML page (default "Vegeta Plot")
|
||||
|
||||
report command:
|
||||
-buckets string
|
||||
Histogram buckets, e.g.: "[0,1ms,10ms]"
|
||||
-every duration
|
||||
Report interval
|
||||
-output string
|
||||
Output file (default "stdout")
|
||||
-type string
|
||||
Report type to generate [text, json, hist[buckets], hdrplot] (default "text")
|
||||
|
||||
examples:
|
||||
echo "GET http://localhost/" | vegeta attack -duration=5s | tee results.bin | vegeta report
|
||||
vegeta report -type=json results.bin > metrics.json
|
||||
cat results.bin | vegeta plot > plot.html
|
||||
cat results.bin | vegeta report -type="hist[0,100ms,200ms,300ms]"
|
||||
```
|
||||
|
||||
#### `-cpus`
|
||||
|
||||
Specifies the number of CPUs to be used internally.
|
||||
It defaults to the amount of CPUs available in the system.
|
||||
|
||||
#### `-profile`
|
||||
|
||||
Specifies which profiler to enable during execution. Both _cpu_ and
|
||||
_heap_ profiles are supported. It defaults to none.
|
||||
|
||||
#### `-version`
|
||||
|
||||
Prints the version and exits.
|
||||
|
||||
### `attack` command
|
||||
|
||||
#### `-body`
|
||||
|
||||
Specifies the file whose content will be set as the body of every
|
||||
request unless overridden per attack target, see `-targets`.
|
||||
|
||||
#### `-cert`
|
||||
|
||||
Specifies the PEM encoded TLS client certificate file to be used with HTTPS requests.
|
||||
If `-key` isn't specified, it will be set to the value of this flag.
|
||||
|
||||
#### `-connections`
|
||||
|
||||
Specifies the maximum number of idle open connections per target host.
|
||||
|
||||
#### `-duration`
|
||||
|
||||
Specifies the amount of time to issue request to the targets.
|
||||
The internal concurrency structure's setup has this value as a variable.
|
||||
The actual run time of the test can be longer than specified due to the
|
||||
responses delay. Use 0 for an infinite attack.
|
||||
|
||||
#### `-format`
|
||||
|
||||
Specifies the targets format to decode.
|
||||
|
||||
##### `json` format
|
||||
|
||||
The JSON format makes integration with programs that produce targets dynamically easier.
|
||||
Each target is one JSON object in its own line. The method and url fields are required.
|
||||
If present, the body field must be base64 encoded. The generated [JSON Schema](lib/target.schema.json)
|
||||
defines the format in detail.
|
||||
|
||||
```bash
|
||||
jq -ncM '{method: "GET", url: "http://goku", body: "Punch!" | @base64, header: {"Content-Type": ["text/plain"]}}' |
|
||||
vegeta attack -format=json -rate=100 | vegeta encode
|
||||
```
|
||||
|
||||
##### `http` format
|
||||
|
||||
The http format almost resembles the plain-text HTTP message format defined in
|
||||
[RFC 2616](https://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html) but it
|
||||
doesn't support in-line HTTP bodies, only references to files that are loaded and used
|
||||
as request bodies (as exemplified below).
|
||||
|
||||
Although targets in this format can be produced by other programs, it was originally
|
||||
meant to be used by people writing targets by hand for simple use cases.
|
||||
|
||||
Here are a few examples of valid targets files in the http format:
|
||||
|
||||
###### Simple targets
|
||||
|
||||
```
|
||||
GET http://goku:9090/path/to/dragon?item=ball
|
||||
GET http://user:password@goku:9090/path/to
|
||||
HEAD http://goku:9090/path/to/success
|
||||
```
|
||||
|
||||
###### Targets with custom headers
|
||||
|
||||
```
|
||||
GET http://user:password@goku:9090/path/to
|
||||
X-Account-ID: 8675309
|
||||
|
||||
DELETE http://goku:9090/path/to/remove
|
||||
Confirmation-Token: 90215
|
||||
Authorization: Token DEADBEEF
|
||||
```
|
||||
|
||||
###### Targets with custom bodies
|
||||
|
||||
```
|
||||
POST http://goku:9090/things
|
||||
@/path/to/newthing.json
|
||||
|
||||
PATCH http://goku:9090/thing/71988591
|
||||
@/path/to/thing-71988591.json
|
||||
```
|
||||
|
||||
###### Targets with custom bodies and headers
|
||||
|
||||
```
|
||||
POST http://goku:9090/things
|
||||
X-Account-ID: 99
|
||||
@/path/to/newthing.json
|
||||
```
|
||||
|
||||
###### Add comments to the targets
|
||||
|
||||
Lines starting with `#` are ignored.
|
||||
|
||||
```
|
||||
# get a dragon ball
|
||||
GET http://goku:9090/path/to/dragon?item=ball
|
||||
```
|
||||
|
||||
#### `-h2c`
|
||||
|
||||
Specifies that HTTP2 requests are to be sent over TCP without TLS encryption.
|
||||
|
||||
#### `-header`
|
||||
|
||||
Specifies a request header to be used in all targets defined, see `-targets`.
|
||||
You can specify as many as needed by repeating the flag.
|
||||
|
||||
#### `-http2`
|
||||
|
||||
Specifies whether to enable HTTP/2 requests to servers which support it.
|
||||
|
||||
#### `-insecure`
|
||||
|
||||
Specifies whether to ignore invalid server TLS certificates.
|
||||
|
||||
#### `-keepalive`
|
||||
|
||||
Specifies whether to reuse TCP connections between HTTP requests.
|
||||
|
||||
#### `-key`
|
||||
|
||||
Specifies the PEM encoded TLS client certificate private key file to be
|
||||
used with HTTPS requests.
|
||||
|
||||
#### `-laddr`
|
||||
|
||||
Specifies the local IP address to be used.
|
||||
|
||||
#### `-lazy`
|
||||
|
||||
Specifies whether to read the input targets lazily instead of eagerly.
|
||||
This allows streaming targets into the attack command and reduces memory
|
||||
footprint.
|
||||
The trade-off is one of added latency in each hit against the targets.
|
||||
|
||||
#### `-max-body`
|
||||
|
||||
Specifies the maximum number of bytes to capture from the body of each
|
||||
response. Remaining unread bytes will be fully read but discarded.
|
||||
Set to -1 for no limit. It knows how to intepret values like these:
|
||||
|
||||
- `"10 MB"` -> `10MB`
|
||||
- `"10240 g"` -> `10TB`
|
||||
- `"2000"` -> `2000B`
|
||||
- `"1tB"` -> `1TB`
|
||||
- `"5 peta"` -> `5PB`
|
||||
- `"28 kilobytes"` -> `28KB`
|
||||
- `"1 gigabyte"` -> `1GB`
|
||||
|
||||
#### `-name`
|
||||
|
||||
Specifies the name of the attack to be recorded in responses.
|
||||
|
||||
#### `-output`
|
||||
|
||||
Specifies the output file to which the binary results will be written
|
||||
to. Made to be piped to the report command input. Defaults to stdout.
|
||||
|
||||
#### `-rate`
|
||||
|
||||
Specifies the request rate per time unit to issue against
|
||||
the targets. The actual request rate can vary slightly due to things like
|
||||
garbage collection, but overall it should stay very close to the specified.
|
||||
If no time unit is provided, 1s is used.
|
||||
|
||||
A `-rate` of `0` or `infinity` means vegeta will send requests as fast as possible.
|
||||
Use together with `-max-workers` to model a fixed set of concurrent users sending
|
||||
requests serially (i.e. waiting for a response before sending the next request).
|
||||
|
||||
Setting `-max-workers` to a very high number while setting `-rate=0` can result in
|
||||
vegeta consuming too many resources and crashing. Use with care.
|
||||
|
||||
#### `-redirects`
|
||||
|
||||
Specifies the max number of redirects followed on each request. The
|
||||
default is 10. When the value is -1, redirects are not followed but
|
||||
the response is marked as successful.
|
||||
|
||||
#### `-resolvers`
|
||||
|
||||
Specifies custom DNS resolver addresses to use for name resolution instead of
|
||||
the ones configured by the operating system. Works only on non Windows systems.
|
||||
|
||||
#### `-root-certs`
|
||||
|
||||
Specifies the trusted TLS root CAs certificate files as a comma separated
|
||||
list. If unspecified, the default system CAs certificates will be used.
|
||||
|
||||
#### `-targets`
|
||||
|
||||
Specifies the file from which to read targets, defaulting to stdin.
|
||||
See the [`-format`](#-format) section to learn about the different target formats.
|
||||
|
||||
#### `-timeout`
|
||||
|
||||
Specifies the timeout for each request. The default is 0 which disables
|
||||
timeouts.
|
||||
|
||||
#### `-workers`
|
||||
|
||||
Specifies the initial number of workers used in the attack. The actual
|
||||
number of workers will increase if necessary in order to sustain the
|
||||
requested rate, unless it'd go beyond `-max-workers`.
|
||||
|
||||
#### `-max-workers`
|
||||
|
||||
Specifies the maximum number of workers used in the attack. It can be used to
|
||||
control the concurrency level used by an attack.
|
||||
|
||||
### `report` command
|
||||
|
||||
```console
|
||||
Usage: vegeta report [options] [<file>...]
|
||||
|
||||
Outputs a report of attack results.
|
||||
|
||||
Arguments:
|
||||
<file> A file with vegeta attack results encoded with one of
|
||||
the supported encodings (gob | json | csv) [default: stdin]
|
||||
|
||||
Options:
|
||||
--type Which report type to generate (text | json | hist[buckets] | hdrplot).
|
||||
[default: text]
|
||||
|
||||
--buckets Histogram buckets, e.g.: '[0,1ms,10ms]'
|
||||
|
||||
--every Write the report to --output at every given interval (e.g 100ms)
|
||||
The default of 0 means the report will only be written after
|
||||
all results have been processed. [default: 0]
|
||||
|
||||
--output Output file [default: stdout]
|
||||
|
||||
Examples:
|
||||
echo "GET http://:80" | vegeta attack -rate=10/s > results.gob
|
||||
echo "GET http://:80" | vegeta attack -rate=100/s | vegeta encode > results.json
|
||||
vegeta report results.*
|
||||
```
|
||||
|
||||
#### `report -type=text`
|
||||
|
||||
```console
|
||||
Requests [total, rate, throughput] 1200, 120.00, 65.87
|
||||
Duration [total, attack, wait] 10.094965987s, 9.949883921s, 145.082066ms
|
||||
Latencies [mean, 50, 95, 99, max] 113.172398ms, 108.272568ms, 140.18235ms, 247.771566ms, 264.815246ms
|
||||
Bytes In [total, mean] 3714690, 3095.57
|
||||
Bytes Out [total, mean] 0, 0.00
|
||||
Success [ratio] 55.42%
|
||||
Status Codes [code:count] 0:535 200:665
|
||||
Error Set:
|
||||
Get http://localhost:6060: dial tcp 127.0.0.1:6060: connection refused
|
||||
Get http://localhost:6060: read tcp 127.0.0.1:6060: connection reset by peer
|
||||
Get http://localhost:6060: dial tcp 127.0.0.1:6060: connection reset by peer
|
||||
Get http://localhost:6060: write tcp 127.0.0.1:6060: broken pipe
|
||||
Get http://localhost:6060: net/http: transport closed before response was received
|
||||
Get http://localhost:6060: http: can't write HTTP request on broken connection
|
||||
```
|
||||
|
||||
The `Requests` row shows:
|
||||
|
||||
- The `total` number of issued requests.
|
||||
- The real request `rate` sustained during the `attack` period.
|
||||
- The `throughput` of successful requests over the `total` period.
|
||||
|
||||
The `Duration` row shows:
|
||||
|
||||
- The `attack` time taken issuing all requests (`total` - `wait`)
|
||||
- The `wait` time waiting for the response to the last issued request (`total` - `attack`)
|
||||
- The `total` time taken in the attack (`attack` + `wait`)
|
||||
|
||||
Latency is the amount of time taken for a response to a request to be read (including the `-max-body` bytes from the response body).
|
||||
|
||||
- `mean` is the [arithmetic mean / average](https://en.wikipedia.org/wiki/Arithmetic_mean) of the latencies of all requests in an attack.
|
||||
- `50`, `95`, `99` are the 50th, 95th an 99th [percentiles](https://en.wikipedia.org/wiki/Percentile), respectively, of the latencies of all requests in an attack. To understand more about why these are useful, I recommend [this article](https://bravenewgeek.com/everything-you-know-about-latency-is-wrong/) from @tylertreat.
|
||||
- `max` is the maximum latency of all requests in an attack.
|
||||
|
||||
The `Bytes In` and `Bytes Out` rows shows:
|
||||
|
||||
- The `total` number of bytes sent (out) or received (in) with the request or response bodies.
|
||||
- The `mean` number of bytes sent (out) or received (in) with the request or response bodies.
|
||||
|
||||
The `Success` ratio shows the percentage of requests whose responses didn't error and had status codes between **200** and **400** (non-inclusive).
|
||||
|
||||
The `Status Codes` row shows a histogram of status codes. `0` status codes mean a request failed to be sent.
|
||||
|
||||
The `Error Set` shows a unique set of errors returned by all issued requests. These include requests that got non-successful response status code.
|
||||
|
||||
#### `report -type=json`
|
||||
|
||||
All duration like fields are in nanoseconds.
|
||||
|
||||
```json
|
||||
{
|
||||
"latencies": {
|
||||
"total": 237119463,
|
||||
"mean": 2371194,
|
||||
"50th": 2854306,
|
||||
"95th": 3478629,
|
||||
"99th": 3530000,
|
||||
"max": 3660505
|
||||
},
|
||||
"buckets": {"0":9952,"1000000":40,"2000000":6,"3000000":0,"4000000":0,"5000000":2},
|
||||
"bytes_in": {
|
||||
"total": 606700,
|
||||
"mean": 6067
|
||||
},
|
||||
"bytes_out": {
|
||||
"total": 0,
|
||||
"mean": 0
|
||||
},
|
||||
"earliest": "2015-09-19T14:45:50.645818631+02:00",
|
||||
"latest": "2015-09-19T14:45:51.635818575+02:00",
|
||||
"end": "2015-09-19T14:45:51.639325797+02:00",
|
||||
"duration": 989999944,
|
||||
"wait": 3507222,
|
||||
"requests": 100,
|
||||
"rate": 101.01010672380401,
|
||||
"throughput": 101.00012489812,
|
||||
"success": 1,
|
||||
"status_codes": {
|
||||
"200": 100
|
||||
},
|
||||
"errors": []
|
||||
}
|
||||
```
|
||||
|
||||
In the `buckets` field, each key is a nanosecond value representing the lower bound of a bucket.
|
||||
The upper bound is implied by the next higher bucket.
|
||||
Upper bounds are non-inclusive.
|
||||
The highest bucket is the overflow bucket; it has no upper bound.
|
||||
The values are counts of how many requests fell into that particular bucket.
|
||||
If the `-buckets` parameter is not present, the `buckets` field is omitted.
|
||||
|
||||
#### `report -type=hist`
|
||||
|
||||
Computes and prints a text based histogram for the given buckets.
|
||||
Each bucket upper bound is non-inclusive.
|
||||
|
||||
```console
|
||||
cat results.bin | vegeta report -type='hist[0,2ms,4ms,6ms]'
|
||||
Bucket # % Histogram
|
||||
[0, 2ms] 6007 32.65% ########################
|
||||
[2ms, 4ms] 5505 29.92% ######################
|
||||
[4ms, 6ms] 2117 11.51% ########
|
||||
[6ms, +Inf] 4771 25.93% ###################
|
||||
```
|
||||
|
||||
#### `report -type=hdrplot`
|
||||
|
||||
Writes out results in a format plottable by https://hdrhistogram.github.io/HdrHistogram/plotFiles.html.
|
||||
|
||||
```
|
||||
Value(ms) Percentile TotalCount 1/(1-Percentile)
|
||||
0.076715 0.000000 0 1.000000
|
||||
0.439370 0.100000 200 1.111111
|
||||
0.480836 0.200000 400 1.250000
|
||||
0.495559 0.300000 599 1.428571
|
||||
0.505101 0.400000 799 1.666667
|
||||
0.513059 0.500000 999 2.000000
|
||||
0.516664 0.550000 1099 2.222222
|
||||
0.520455 0.600000 1199 2.500000
|
||||
0.525008 0.650000 1299 2.857143
|
||||
0.530174 0.700000 1399 3.333333
|
||||
0.534891 0.750000 1499 4.000000
|
||||
0.537572 0.775000 1548 4.444444
|
||||
0.540340 0.800000 1598 5.000000
|
||||
0.543763 0.825000 1648 5.714286
|
||||
0.547164 0.850000 1698 6.666667
|
||||
0.551432 0.875000 1748 8.000000
|
||||
0.553444 0.887500 1773 8.888889
|
||||
0.555774 0.900000 1798 10.000000
|
||||
0.558454 0.912500 1823 11.428571
|
||||
0.562123 0.925000 1848 13.333333
|
||||
0.565563 0.937500 1873 16.000000
|
||||
0.567831 0.943750 1886 17.777778
|
||||
0.570617 0.950000 1898 20.000000
|
||||
0.574522 0.956250 1911 22.857143
|
||||
0.579046 0.962500 1923 26.666667
|
||||
0.584426 0.968750 1936 32.000000
|
||||
0.586695 0.971875 1942 35.555556
|
||||
0.590451 0.975000 1948 40.000000
|
||||
0.597543 0.978125 1954 45.714286
|
||||
0.605637 0.981250 1961 53.333333
|
||||
0.613564 0.984375 1967 64.000000
|
||||
0.620393 0.985938 1970 71.113640
|
||||
0.629121 0.987500 1973 80.000000
|
||||
0.638060 0.989062 1976 91.424392
|
||||
0.648085 0.990625 1979 106.666667
|
||||
0.659689 0.992188 1982 128.008193
|
||||
0.665870 0.992969 1984 142.227279
|
||||
0.672985 0.993750 1986 160.000000
|
||||
0.680101 0.994531 1987 182.848784
|
||||
0.687810 0.995313 1989 213.356091
|
||||
0.695729 0.996094 1990 256.016385
|
||||
0.730641 0.996484 1991 284.414107
|
||||
0.785516 0.996875 1992 320.000000
|
||||
0.840392 0.997266 1993 365.764448
|
||||
1.009646 0.997656 1993 426.621160
|
||||
1.347020 0.998047 1994 512.032770
|
||||
1.515276 0.998242 1994 568.828214
|
||||
1.683532 0.998437 1995 639.795266
|
||||
1.887487 0.998633 1995 731.528895
|
||||
2.106249 0.998828 1996 853.242321
|
||||
2.325011 0.999023 1996 1023.541453
|
||||
2.434952 0.999121 1996 1137.656428
|
||||
2.544894 0.999219 1996 1280.409731
|
||||
2.589510 0.999316 1997 1461.988304
|
||||
2.605192 0.999414 1997 1706.484642
|
||||
2.620873 0.999512 1997 2049.180328
|
||||
2.628713 0.999561 1997 2277.904328
|
||||
2.636394 0.999609 1997 2557.544757
|
||||
2.644234 0.999658 1997 2923.976608
|
||||
2.652075 0.999707 1997 3412.969283
|
||||
2.658916 0.999756 1998 4098.360656
|
||||
2.658916 0.999780 1998 4545.454545
|
||||
2.658916 0.999805 1998 5128.205128
|
||||
2.658916 0.999829 1998 5847.953216
|
||||
2.658916 0.999854 1998 6849.315068
|
||||
2.658916 0.999878 1998 8196.721311
|
||||
2.658916 0.999890 1998 9090.909091
|
||||
2.658916 0.999902 1998 10204.081633
|
||||
2.658916 0.999915 1998 11764.705882
|
||||
2.658916 0.999927 1998 13698.630137
|
||||
2.658916 0.999939 1998 16393.442623
|
||||
2.658916 0.999945 1998 18181.818182
|
||||
2.658916 0.999951 1998 20408.163265
|
||||
2.658916 0.999957 1998 23255.813953
|
||||
2.658916 0.999963 1998 27027.027027
|
||||
2.658916 0.999969 1998 32258.064516
|
||||
2.658916 0.999973 1998 37037.037037
|
||||
2.658916 0.999976 1998 41666.666667
|
||||
2.658916 0.999979 1998 47619.047619
|
||||
2.658916 0.999982 1998 55555.555556
|
||||
2.658916 0.999985 1998 66666.666667
|
||||
2.658916 0.999986 1998 71428.571429
|
||||
2.658916 0.999988 1998 83333.333333
|
||||
2.658916 0.999989 1998 90909.090909
|
||||
2.658916 0.999991 1998 111111.111111
|
||||
2.658916 0.999992 1998 125000.000000
|
||||
2.658916 0.999993 1998 142857.142858
|
||||
2.658916 0.999994 1998 166666.666668
|
||||
2.658916 0.999995 1998 199999.999999
|
||||
2.658916 0.999996 1998 250000.000000
|
||||
2.658916 0.999997 1998 333333.333336
|
||||
2.658916 0.999998 1998 500000.000013
|
||||
2.658916 0.999999 1998 999999.999971
|
||||
2.658916 1.000000 1998 10000000.000000
|
||||
```
|
||||
|
||||
### `encode` command
|
||||
|
||||
```
|
||||
Usage: vegeta encode [options] [<file>...]
|
||||
|
||||
Encodes vegeta attack results from one encoding to another.
|
||||
The supported encodings are Gob (binary), CSV and JSON.
|
||||
Each input file may have a different encoding which is detected
|
||||
automatically.
|
||||
|
||||
The CSV encoder doesn't write a header. The columns written by it are:
|
||||
|
||||
1. Unix timestamp in nanoseconds since epoch
|
||||
2. HTTP status code
|
||||
3. Request latency in nanoseconds
|
||||
4. Bytes out
|
||||
5. Bytes in
|
||||
6. Error
|
||||
7. Base64 encoded response body
|
||||
8. Attack name
|
||||
9. Sequence number of request
|
||||
|
||||
Arguments:
|
||||
<file> A file with vegeta attack results encoded with one of
|
||||
the supported encodings (gob | json | csv) [default: stdin]
|
||||
|
||||
Options:
|
||||
--to Output encoding (gob | json | csv) [default: json]
|
||||
--output Output file [default: stdout]
|
||||
|
||||
Examples:
|
||||
echo "GET http://:80" | vegeta attack -rate=1/s > results.gob
|
||||
cat results.gob | vegeta encode | jq -c 'del(.body)' | vegeta encode -to gob
|
||||
```
|
||||
|
||||
### `plot` command
|
||||
|
||||
|
||||
|
||||
```
|
||||
Usage: vegeta plot [options] [<file>...]
|
||||
|
||||
Outputs an HTML time series plot of request latencies over time.
|
||||
The X axis represents elapsed time in seconds from the beginning
|
||||
of the earliest attack in all input files. The Y axis represents
|
||||
request latency in milliseconds.
|
||||
|
||||
Click and drag to select a region to zoom into. Double click to zoom out.
|
||||
Choose a different number on the bottom left corner input field
|
||||
to change the moving average window size (in data points).
|
||||
|
||||
Arguments:
|
||||
<file> A file output by running vegeta attack [default: stdin]
|
||||
|
||||
Options:
|
||||
--title Title and header of the resulting HTML page.
|
||||
[default: Vegeta Plot]
|
||||
--threshold Threshold of data points to downsample series to.
|
||||
Series with less than --threshold number of data
|
||||
points are not downsampled. [default: 4000]
|
||||
|
||||
Examples:
|
||||
echo "GET http://:80" | vegeta attack -name=50qps -rate=50 -duration=5s > results.50qps.bin
|
||||
cat results.50qps.bin | vegeta plot > plot.50qps.html
|
||||
echo "GET http://:80" | vegeta attack -name=100qps -rate=100 -duration=5s > results.100qps.bin
|
||||
vegeta plot results.50qps.bin results.100qps.bin > plot.html
|
||||
```
|
||||
|
||||
## Usage: Generated targets
|
||||
|
||||
Apart from accepting a static list of targets, Vegeta can be used together with another program that generates them in a streaming fashion. Here's an example of that using the `jq` utility that generates targets with an incrementing id in their body.
|
||||
|
||||
```console
|
||||
jq -ncM 'while(true; .+1) | {method: "POST", url: "http://:6060", body: {id: .} | @base64 }' | \
|
||||
vegeta attack -rate=50/s -lazy -format=json -duration=30s | \
|
||||
tee results.bin | \
|
||||
vegeta report
|
||||
```
|
||||
|
||||
## Usage: Distributed attacks
|
||||
|
||||
Whenever your load test can't be conducted due to Vegeta hitting machine limits
|
||||
such as open files, memory, CPU or network bandwidth, it's a good idea to use Vegeta in a distributed manner.
|
||||
|
||||
In a hypothetical scenario where the desired attack rate is 60k requests per second,
|
||||
let's assume we have 3 machines with `vegeta` installed.
|
||||
|
||||
Make sure open file descriptor and process limits are set to a high number for your user **on each machine**
|
||||
using the `ulimit` command.
|
||||
|
||||
We're ready to start the attack. All we need to do is to divide the intended rate by the number of machines,
|
||||
and use that number on each attack. Here we'll use [pdsh](https://code.google.com/p/pdsh/) for orchestration.
|
||||
|
||||
```shell
|
||||
$ PDSH_RCMD_TYPE=ssh pdsh -b -w '10.0.1.1,10.0.2.1,10.0.3.1' \
|
||||
'echo "GET http://target/" | vegeta attack -rate=20000 -duration=60s > result.bin'
|
||||
```
|
||||
|
||||
After the previous command finishes, we can gather the result files to use on our report.
|
||||
|
||||
```shell
|
||||
$ for machine in 10.0.1.1 10.0.2.1 10.0.3.1; do
|
||||
scp $machine:~/result.bin $machine.bin &
|
||||
done
|
||||
```
|
||||
|
||||
The `report` command accepts multiple result files.
|
||||
It'll read and sort them by timestamp before generating reports.
|
||||
|
||||
```console
|
||||
vegeta report *.bin
|
||||
```
|
||||
|
||||
## Usage: Real-time Analysis
|
||||
|
||||
If you are a happy user of iTerm, you can integrate vegeta with [jplot](https://github.com/rs/jplot) using [jaggr](https://github.com/rs/jaggr) to plot a vegeta report in real-time in the comfort of you terminal:
|
||||
|
||||
```
|
||||
echo 'GET http://localhost:8080' | \
|
||||
vegeta attack -rate 5000 -duration 10m | vegeta encode | \
|
||||
jaggr @count=rps \
|
||||
hist\[100,200,300,400,500\]:code \
|
||||
p25,p50,p95:latency \
|
||||
sum:bytes_in \
|
||||
sum:bytes_out | \
|
||||
jplot rps+code.hist.100+code.hist.200+code.hist.300+code.hist.400+code.hist.500 \
|
||||
latency.p95+latency.p50+latency.p25 \
|
||||
bytes_in.sum+bytes_out.sum
|
||||
```
|
||||
|
||||
|
||||
|
||||
## Usage (Library)
|
||||
|
||||
The library versioning follows [SemVer v2.0.0](https://semver.org/spec/v2.0.0.html).
|
||||
Since [lib/v9.0.0](https://github.com/tsenart/vegeta/tree/lib/v9.0.0), the library and cli
|
||||
are versioned separately to better isolate breaking changes to each component.
|
||||
|
||||
See [Versioning](#Versioning) for more details on git tag naming schemes and compatibility
|
||||
with `go mod`.
|
||||
|
||||
```go
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
vegeta "github.com/tsenart/vegeta/lib"
|
||||
)
|
||||
|
||||
func main() {
|
||||
rate := vegeta.Rate{Freq: 100, Per: time.Second}
|
||||
duration := 4 * time.Second
|
||||
targeter := vegeta.NewStaticTargeter(vegeta.Target{
|
||||
Method: "GET",
|
||||
URL: "http://localhost:9100/",
|
||||
})
|
||||
attacker := vegeta.NewAttacker()
|
||||
|
||||
var metrics vegeta.Metrics
|
||||
for res := range attacker.Attack(targeter, rate, duration, "Big Bang!") {
|
||||
metrics.Add(res)
|
||||
}
|
||||
metrics.Close()
|
||||
|
||||
fmt.Printf("99th percentile: %s\n", metrics.Latencies.P99)
|
||||
}
|
||||
```
|
||||
|
||||
#### Limitations
|
||||
|
||||
There will be an upper bound of the supported `rate` which varies on the
|
||||
machine being used.
|
||||
You could be CPU bound (unlikely), memory bound (more likely) or
|
||||
have system resource limits being reached which ought to be tuned for
|
||||
the process execution. The important limits for us are file descriptors
|
||||
and processes. On a UNIX system you can get and set the current
|
||||
soft-limit values for a user.
|
||||
|
||||
```shell
|
||||
$ ulimit -n # file descriptors
|
||||
2560
|
||||
$ ulimit -u # processes / threads
|
||||
709
|
||||
```
|
||||
|
||||
Just pass a new number as the argument to change it.
|
||||
|
||||
## License
|
||||
|
||||
See [LICENSE](LICENSE).
|
||||
|
||||
## Donate
|
||||
|
||||
If you use and love Vegeta, please consider sending some Satoshi to
|
||||
`1MDmKC51ve7Upxt75KoNM6x1qdXHFK6iW2`. In case you want to be mentioned as a
|
||||
sponsor, let me know!
|
||||
|
||||
[](#donate)
|
||||
|
|
@ -1,242 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/tsenart/vegeta/internal/resolver"
|
||||
vegeta "github.com/tsenart/vegeta/lib"
|
||||
)
|
||||
|
||||
func attackCmd() command {
|
||||
fs := flag.NewFlagSet("vegeta attack", flag.ExitOnError)
|
||||
opts := &attackOpts{
|
||||
headers: headers{http.Header{}},
|
||||
laddr: localAddr{&vegeta.DefaultLocalAddr},
|
||||
rate: vegeta.Rate{Freq: 50, Per: time.Second},
|
||||
maxBody: vegeta.DefaultMaxBody,
|
||||
}
|
||||
fs.StringVar(&opts.name, "name", "", "Attack name")
|
||||
fs.StringVar(&opts.targetsf, "targets", "stdin", "Targets file")
|
||||
fs.StringVar(&opts.format, "format", vegeta.HTTPTargetFormat,
|
||||
fmt.Sprintf("Targets format [%s]", strings.Join(vegeta.TargetFormats, ", ")))
|
||||
fs.StringVar(&opts.outputf, "output", "stdout", "Output file")
|
||||
fs.StringVar(&opts.bodyf, "body", "", "Requests body file")
|
||||
fs.StringVar(&opts.certf, "cert", "", "TLS client PEM encoded certificate file")
|
||||
fs.StringVar(&opts.keyf, "key", "", "TLS client PEM encoded private key file")
|
||||
fs.Var(&opts.rootCerts, "root-certs", "TLS root certificate files (comma separated list)")
|
||||
fs.BoolVar(&opts.http2, "http2", true, "Send HTTP/2 requests when supported by the server")
|
||||
fs.BoolVar(&opts.h2c, "h2c", false, "Send HTTP/2 requests without TLS encryption")
|
||||
fs.BoolVar(&opts.insecure, "insecure", false, "Ignore invalid server TLS certificates")
|
||||
fs.BoolVar(&opts.lazy, "lazy", false, "Read targets lazily")
|
||||
fs.DurationVar(&opts.duration, "duration", 0, "Duration of the test [0 = forever]")
|
||||
fs.DurationVar(&opts.timeout, "timeout", vegeta.DefaultTimeout, "Requests timeout")
|
||||
fs.Uint64Var(&opts.workers, "workers", vegeta.DefaultWorkers, "Initial number of workers")
|
||||
fs.Uint64Var(&opts.maxWorkers, "max-workers", vegeta.DefaultMaxWorkers, "Maximum number of workers")
|
||||
fs.IntVar(&opts.connections, "connections", vegeta.DefaultConnections, "Max open idle connections per target host")
|
||||
fs.IntVar(&opts.redirects, "redirects", vegeta.DefaultRedirects, "Number of redirects to follow. -1 will not follow but marks as success")
|
||||
fs.Var(&maxBodyFlag{&opts.maxBody}, "max-body", "Maximum number of bytes to capture from response bodies. [-1 = no limit]")
|
||||
fs.Var(&rateFlag{&opts.rate}, "rate", "Number of requests per time unit [0 = infinity]")
|
||||
fs.Var(&opts.headers, "header", "Request header")
|
||||
fs.Var(&opts.laddr, "laddr", "Local IP address")
|
||||
fs.BoolVar(&opts.keepalive, "keepalive", true, "Use persistent connections")
|
||||
fs.StringVar(&opts.unixSocket, "unix-socket", "", "Connect over a unix socket. This overrides the host address in target URLs")
|
||||
systemSpecificFlags(fs, opts)
|
||||
|
||||
return command{fs, func(args []string) error {
|
||||
fs.Parse(args)
|
||||
return attack(opts)
|
||||
}}
|
||||
}
|
||||
|
||||
var (
|
||||
errZeroRate = errors.New("rate frequency and time unit must be bigger than zero")
|
||||
errBadCert = errors.New("bad certificate")
|
||||
)
|
||||
|
||||
// attackOpts aggregates the attack function command options
|
||||
type attackOpts struct {
|
||||
name string
|
||||
targetsf string
|
||||
format string
|
||||
outputf string
|
||||
bodyf string
|
||||
certf string
|
||||
keyf string
|
||||
rootCerts csl
|
||||
http2 bool
|
||||
h2c bool
|
||||
insecure bool
|
||||
lazy bool
|
||||
duration time.Duration
|
||||
timeout time.Duration
|
||||
rate vegeta.Rate
|
||||
workers uint64
|
||||
maxWorkers uint64
|
||||
connections int
|
||||
redirects int
|
||||
maxBody int64
|
||||
headers headers
|
||||
laddr localAddr
|
||||
keepalive bool
|
||||
resolvers csl
|
||||
unixSocket string
|
||||
}
|
||||
|
||||
// attack validates the attack arguments, sets up the
|
||||
// required resources, launches the attack and writes the results
|
||||
func attack(opts *attackOpts) (err error) {
|
||||
if opts.maxWorkers == vegeta.DefaultMaxWorkers && opts.rate.Freq == 0 {
|
||||
return fmt.Errorf("-rate=0 requires setting -max-workers")
|
||||
}
|
||||
|
||||
if len(opts.resolvers) > 0 {
|
||||
res, err := resolver.NewResolver(opts.resolvers)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
net.DefaultResolver = res
|
||||
}
|
||||
|
||||
files := map[string]io.Reader{}
|
||||
for _, filename := range []string{opts.targetsf, opts.bodyf} {
|
||||
if filename == "" {
|
||||
continue
|
||||
}
|
||||
f, err := file(filename, false)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening %s: %s", filename, err)
|
||||
}
|
||||
defer f.Close()
|
||||
files[filename] = f
|
||||
}
|
||||
|
||||
var body []byte
|
||||
if bodyf, ok := files[opts.bodyf]; ok {
|
||||
if body, err = ioutil.ReadAll(bodyf); err != nil {
|
||||
return fmt.Errorf("error reading %s: %s", opts.bodyf, err)
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
tr vegeta.Targeter
|
||||
src = files[opts.targetsf]
|
||||
hdr = opts.headers.Header
|
||||
)
|
||||
|
||||
switch opts.format {
|
||||
case vegeta.JSONTargetFormat:
|
||||
tr = vegeta.NewJSONTargeter(src, body, hdr)
|
||||
case vegeta.HTTPTargetFormat:
|
||||
tr = vegeta.NewHTTPTargeter(src, body, hdr)
|
||||
default:
|
||||
return fmt.Errorf("format %q isn't one of [%s]",
|
||||
opts.format, strings.Join(vegeta.TargetFormats, ", "))
|
||||
}
|
||||
|
||||
if !opts.lazy {
|
||||
targets, err := vegeta.ReadAllTargets(tr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
tr = vegeta.NewStaticTargeter(targets...)
|
||||
}
|
||||
|
||||
out, err := file(opts.outputf, true)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening %s: %s", opts.outputf, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
tlsc, err := tlsConfig(opts.insecure, opts.certf, opts.keyf, opts.rootCerts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
atk := vegeta.NewAttacker(
|
||||
vegeta.Redirects(opts.redirects),
|
||||
vegeta.Timeout(opts.timeout),
|
||||
vegeta.LocalAddr(*opts.laddr.IPAddr),
|
||||
vegeta.TLSConfig(tlsc),
|
||||
vegeta.Workers(opts.workers),
|
||||
vegeta.MaxWorkers(opts.maxWorkers),
|
||||
vegeta.KeepAlive(opts.keepalive),
|
||||
vegeta.Connections(opts.connections),
|
||||
vegeta.HTTP2(opts.http2),
|
||||
vegeta.H2C(opts.h2c),
|
||||
vegeta.MaxBody(opts.maxBody),
|
||||
vegeta.UnixSocket(opts.unixSocket),
|
||||
)
|
||||
|
||||
res := atk.Attack(tr, opts.rate, opts.duration, opts.name)
|
||||
enc := vegeta.NewEncoder(out)
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, os.Interrupt)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-sig:
|
||||
atk.Stop()
|
||||
return nil
|
||||
case r, ok := <-res:
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if err = enc.Encode(r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// tlsConfig builds a *tls.Config from the given options.
|
||||
func tlsConfig(insecure bool, certf, keyf string, rootCerts []string) (*tls.Config, error) {
|
||||
var err error
|
||||
files := map[string][]byte{}
|
||||
filenames := append([]string{certf, keyf}, rootCerts...)
|
||||
for _, f := range filenames {
|
||||
if f != "" {
|
||||
if files[f], err = ioutil.ReadFile(f); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
c := tls.Config{InsecureSkipVerify: insecure}
|
||||
if cert, ok := files[certf]; ok {
|
||||
key, ok := files[keyf]
|
||||
if !ok {
|
||||
key = cert
|
||||
}
|
||||
|
||||
certificate, err := tls.X509KeyPair(cert, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.Certificates = append(c.Certificates, certificate)
|
||||
c.BuildNameToCertificate()
|
||||
}
|
||||
|
||||
if len(rootCerts) > 0 {
|
||||
c.RootCAs = x509.NewCertPool()
|
||||
for _, f := range rootCerts {
|
||||
if !c.RootCAs.AppendCertsFromPEM(files[f]) {
|
||||
return nil, errBadCert
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &c, nil
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
// +build !windows
|
||||
|
||||
package main
|
||||
|
||||
import "flag"
|
||||
|
||||
func systemSpecificFlags(fs *flag.FlagSet, opts *attackOpts) {
|
||||
fs.Var(&opts.resolvers, "resolvers", "List of addresses (ip:port) to use for DNS resolution. Disables use of local system DNS. (comma separated list)")
|
||||
}
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
package main
|
||||
|
||||
import "flag"
|
||||
|
||||
func systemSpecificFlags(fs *flag.FlagSet, opts *attackOpts) {}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
)
|
||||
|
||||
func dumpCmd() command {
|
||||
return command{fn: func([]string) error {
|
||||
return fmt.Errorf("vegeta dump has been deprecated and succeeded by the vegeta encode command")
|
||||
}}
|
||||
}
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
|
||||
vegeta "github.com/tsenart/vegeta/lib"
|
||||
)
|
||||
|
||||
const (
|
||||
encodingCSV = "csv"
|
||||
encodingGob = "gob"
|
||||
encodingJSON = "json"
|
||||
)
|
||||
|
||||
const encodeUsage = `Usage: vegeta encode [options] [<file>...]
|
||||
|
||||
Encodes vegeta attack results from one encoding to another.
|
||||
The supported encodings are Gob (binary), CSV and JSON.
|
||||
Each input file may have a different encoding which is detected
|
||||
automatically.
|
||||
|
||||
The CSV encoder doesn't write a header. The columns written by it are:
|
||||
|
||||
1. Unix timestamp in nanoseconds since epoch
|
||||
2. HTTP status code
|
||||
3. Request latency in nanoseconds
|
||||
4. Bytes out
|
||||
5. Bytes in
|
||||
6. Error
|
||||
7. Base64 encoded response body
|
||||
8. Attack name
|
||||
9. Sequence number of request
|
||||
|
||||
Arguments:
|
||||
<file> A file with vegeta attack results encoded with one of
|
||||
the supported encodings (gob | json | csv) [default: stdin]
|
||||
|
||||
Options:
|
||||
--to Output encoding (gob | json | csv) [default: json]
|
||||
--output Output file [default: stdout]
|
||||
|
||||
Examples:
|
||||
echo "GET http://:80" | vegeta attack -rate=1/s > results.gob
|
||||
cat results.gob | vegeta encode | jq -c 'del(.body)' | vegeta encode -to gob
|
||||
`
|
||||
|
||||
func encodeCmd() command {
|
||||
encs := "[" + strings.Join([]string{encodingCSV, encodingGob, encodingJSON}, ", ") + "]"
|
||||
fs := flag.NewFlagSet("vegeta encode", flag.ExitOnError)
|
||||
to := fs.String("to", encodingJSON, "Output encoding "+encs)
|
||||
output := fs.String("output", "stdout", "Output file")
|
||||
|
||||
fs.Usage = func() {
|
||||
fmt.Fprintln(os.Stderr, encodeUsage)
|
||||
}
|
||||
|
||||
return command{fs, func(args []string) error {
|
||||
fs.Parse(args)
|
||||
files := fs.Args()
|
||||
if len(files) == 0 {
|
||||
files = append(files, "stdin")
|
||||
}
|
||||
return encode(files, *to, *output)
|
||||
}}
|
||||
}
|
||||
|
||||
func encode(files []string, to, output string) error {
|
||||
dec, mc, err := decoder(files)
|
||||
defer mc.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
out, err := file(output, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
var enc vegeta.Encoder
|
||||
switch to {
|
||||
case encodingCSV:
|
||||
enc = vegeta.NewCSVEncoder(out)
|
||||
case encodingGob:
|
||||
enc = vegeta.NewEncoder(out)
|
||||
case encodingJSON:
|
||||
enc = vegeta.NewJSONEncoder(out)
|
||||
default:
|
||||
return fmt.Errorf("encode: unknown encoding %q", to)
|
||||
}
|
||||
|
||||
sigch := make(chan os.Signal, 1)
|
||||
signal.Notify(sigch, os.Interrupt)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-sigch:
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
var r vegeta.Result
|
||||
if err = dec.Decode(&r); err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
return err
|
||||
} else if err = enc.Encode(&r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue