// Copyright 2017 CoreOS, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package dbtester import ( "sort" "time" "github.com/coreos/etcd/pkg/report" ) // CumulativeKeyNumToAvgLatency wraps the cumulative number of keys // and according latency data. So the higher 'CumulativeKeyNum' is, // the later the data points are in the time series. type CumulativeKeyNumToAvgLatency struct { CumulativeKeyNum int64 MinLatency time.Duration AvgLatency time.Duration MaxLatency time.Duration } // CumulativeKeyNumToAvgLatencySlice is a slice of CumulativeKeyNumToAvgLatency to sort by CumulativeKeyNum. type CumulativeKeyNumToAvgLatencySlice []CumulativeKeyNumToAvgLatency func (t CumulativeKeyNumToAvgLatencySlice) Swap(i, j int) { t[i], t[j] = t[j], t[i] } func (t CumulativeKeyNumToAvgLatencySlice) Len() int { return len(t) } func (t CumulativeKeyNumToAvgLatencySlice) Less(i, j int) bool { return t[i].CumulativeKeyNum < t[j].CumulativeKeyNum } // FindRangesLatency sorts all data points by its timestamp. // And then aggregate by the cumulative throughput, // in order to map the number of keys to the average latency. // // type DataPoint struct { // Timestamp int64 // MinLatency time.Duration // AvgLatency time.Duration // MaxLatency time.Duration // ThroughPut int64 // } // // If unit is 1000 and the average throughput per second is 30,000 // and its average latency is 10ms, it will have 30 data points with // latency 10ms. func FindRangesLatency(data report.TimeSeries, unit int64, totalRequests int64) CumulativeKeyNumToAvgLatencySlice { // need to sort by timestamps because we want the 'cumulative' // trends as we write more keys, 'report.TimeSeries' already implements // sort interface, so just sort.Sort(data) sort.Sort(data) cumulKeyN := int64(0) maxKey := int64(0) rm := make(map[int64]CumulativeKeyNumToAvgLatency) // this data is aggregated by second // and we want to map number of keys to latency // so the range is the key // and the value is the cumulative throughput for _, ts := range data { cumulKeyN += ts.ThroughPut if cumulKeyN < unit { // not enough data points yet continue } // cumulKeyN >= unit for cumulKeyN > maxKey { maxKey += unit rm[maxKey] = CumulativeKeyNumToAvgLatency{ MinLatency: ts.MinLatency, AvgLatency: ts.AvgLatency, MaxLatency: ts.MaxLatency, } } } // fill-in empty rows for i := maxKey; i < totalRequests; i += unit { if _, ok := rm[i]; !ok { rm[i] = CumulativeKeyNumToAvgLatency{} } } if _, ok := rm[totalRequests]; !ok { rm[totalRequests] = CumulativeKeyNumToAvgLatency{} } kss := []CumulativeKeyNumToAvgLatency{} delete(rm, 0) // drop data at beginning for k, v := range rm { // make sure to use 'k' as CumulativeKeyNum kss = append(kss, CumulativeKeyNumToAvgLatency{ CumulativeKeyNum: k, MinLatency: v.MinLatency, AvgLatency: v.AvgLatency, MaxLatency: v.MaxLatency, }) } // sort by cumulative throughput (number of keys) in ascending order sort.Sort(CumulativeKeyNumToAvgLatencySlice(kss)) return kss } // CumulativeKeyNumAndOtherData wraps the cumulative number of keys // and according memory data. So the higher 'CumulativeKeyNum' is, // the later the data points are in the time series. type CumulativeKeyNumAndOtherData struct { UnixSecond int64 Throughput int64 CumulativeKeyNum int64 MinMemoryMB float64 AvgMemoryMB float64 MaxMemoryMB float64 AvgReadBytesDelta float64 AvgWriteBytesDelta float64 } // CumulativeKeyNumAndOtherDataSlice is a slice of CumulativeKeyNumAndOtherData to sort by CumulativeKeyNum. type CumulativeKeyNumAndOtherDataSlice []CumulativeKeyNumAndOtherData func (t CumulativeKeyNumAndOtherDataSlice) Swap(i, j int) { t[i], t[j] = t[j], t[i] } func (t CumulativeKeyNumAndOtherDataSlice) Len() int { return len(t) } func (t CumulativeKeyNumAndOtherDataSlice) Less(i, j int) bool { return t[i].CumulativeKeyNum < t[j].CumulativeKeyNum } // CumulativeKeyNumAndOtherDataByUnixSecond is a slice of CumulativeKeyNumAndOtherData to sort by UnixSecond. type CumulativeKeyNumAndOtherDataByUnixSecond []CumulativeKeyNumAndOtherData func (t CumulativeKeyNumAndOtherDataByUnixSecond) Swap(i, j int) { t[i], t[j] = t[j], t[i] } func (t CumulativeKeyNumAndOtherDataByUnixSecond) Len() int { return len(t) } func (t CumulativeKeyNumAndOtherDataByUnixSecond) Less(i, j int) bool { return t[i].UnixSecond < t[j].UnixSecond } // FindRangesData sorts all data points by its timestamp. // And then aggregate by the cumulative throughput, // in order to map the number of keys to the average memory usage. func FindRangesData(data []CumulativeKeyNumAndOtherData, unit int64, totalRequests int64) CumulativeKeyNumAndOtherDataSlice { // need to sort by timestamps because we want the 'cumulative' // trends as we write more keys, 'report.TimeSeries' already implements // sort interface, so just sort.Sort(data) sort.Sort(CumulativeKeyNumAndOtherDataByUnixSecond(data)) cumulKeyN := int64(0) maxKey := int64(0) rm := make(map[int64]CumulativeKeyNumAndOtherData) // this data is aggregated by second // and we want to map number of keys to memory usage // so the range is the key // and the value is the cumulative throughput for _, ts := range data { cumulKeyN += ts.Throughput if cumulKeyN < unit { // not enough data points yet continue } // cumulKeyN >= unit for cumulKeyN > maxKey { maxKey += unit rm[maxKey] = ts } } // fill-in empty rows for i := maxKey; i < int64(totalRequests); i += unit { if _, ok := rm[i]; !ok { rm[i] = CumulativeKeyNumAndOtherData{} } } if _, ok := rm[int64(totalRequests)]; !ok { rm[int64(totalRequests)] = CumulativeKeyNumAndOtherData{} } kss := []CumulativeKeyNumAndOtherData{} delete(rm, 0) // drop data at beginning for k, v := range rm { // make sure to use 'k' as keyNum kss = append(kss, CumulativeKeyNumAndOtherData{ CumulativeKeyNum: k, MinMemoryMB: v.MinMemoryMB, AvgMemoryMB: v.AvgMemoryMB, MaxMemoryMB: v.MaxMemoryMB, AvgReadBytesDelta: v.AvgReadBytesDelta, AvgWriteBytesDelta: v.AvgWriteBytesDelta, }) } // sort by cumulative throughput (number of keys) in ascending order sort.Sort(CumulativeKeyNumAndOtherDataSlice(kss)) return kss }