import { check, fail, sleep } from 'k6'; import http from 'k6/http' import { Trend } from 'k6/metrics'; import { getCookies, login } from "../rancher/rancher_utils.js"; import { vu as metaVU } from 'k6/execution' import * as crdUtil from "./crd_utils.js"; import * as k6Util from "../generic/k6_utils.js"; const vus = __ENV.K6_VUS || 1 const crdCount = __ENV.CRD_COUNT || 500 const iterations = __ENV.PER_VU_ITERATIONS || 30 const baseUrl = __ENV.BASE_URL const username = __ENV.USERNAME const password = __ENV.PASSWORD const namePrefix = "crontabs-test-" export const epDataRecv = new Trend('endpoint_data_recv'); export const headerDataRecv = new Trend('header_data_recv'); export const timePolled = new Trend('time_polled', true); export const options = { scenarios: { load: { executor: 'shared-iterations', exec: 'loadCRDs', vus: vus, iterations: iterations, maxDuration: '24h', } }, thresholds: { http_req_failed: ['rate<=0.01'], // http errors should be less than 1% http_req_duration: ['p(99)<=500'], // 95% of requests should be below 500ms checks: ['rate>0.99'], // the rate of successful checks should be higher than 99% header_data_recv: ['p(95) < 1024'], [`endpoint_data_recv{url:'/v1/apiextensions.k8s.io.customresourcedefinitions'}`]: ['min > 2048'], // bytes in this case [`endpoint_data_recv{url:'/v1/apiextensions.k8s.io.customresourcedefinitions/'}`]: ['max < 512'], // bytes in this case [`time_polled{url:'/v1/apiextensions.k8s.io.customresourcedefinitions'}`]: ['p(99) < 5000', 'avg < 2500'], }, setupTimeout: '30m', teardownTimeout: '30m' } let newSchema = { "name": "v3", "schema": { "openAPIV3Schema": { "properties": { "spec": { "properties": { "cronSpec": { "type": "string" }, "image": { "type": "string" }, "newField": { "type": "string" }, "replicas": { "type": "integer" } }, "type": "object" } }, "type": "object" } }, "served": true, "storage": true, } function cleanup(cookies) { let { res, crdArray } = crdUtil.getCRDsMatchingName(baseUrl, cookies, namePrefix); console.log("Got CRD Status in delete: ", res.status); let deleteAllFailed = false; crdArray.forEach(r => { let delRes = crdUtil.deleteCRD(baseUrl, cookies, r["id"]); if (delRes.status !== 200 && delRes.status !== 204) deleteAllFailed = true; sleep(0.5); }) return deleteAllFailed; } // Test functions, in order of execution export function setup() { // log in if (login(baseUrl, {}, username, password).status !== 200) { fail(`could not login into cluster`); } const cookies = getCookies(baseUrl); let deleteAllFailed = cleanup(cookies, namePrefix); if (deleteAllFailed) fail("Failed to delete all existing crontab CRDs during setup!"); let { _, crdArray } = crdUtil.getCRDsMatchingName(baseUrl, cookies, namePrefix); // return data that remains constant throughout the test return { cookies: cookies, crdArray: checkAndBuildCRDArray(cookies, crdArray) }; } export function checkAndBuildCRDArray(cookies, crdArray) { let retries = 3; let attempts = 0; while (crdArray.length != crdCount && attempts < retries) { console.log("Creating needed CRDs"); // delete leftovers, if any so that we create exactly crdCount if (crdArray.length == crdCount) { console.log("Finished setting up expected CRD count"); break; } if (crdArray.length > 0) { let deleteAllFailed = cleanup(cookies,); if (deleteAllFailed && attempts == (retries - 1)) fail("Failed to delete all existing crontab CRDs during setup!"); } for (let i = 0; i < crdCount; i++) { let crdSuffix = `${i}`; let res = crdUtil.createCRD(baseUrl, cookies, crdSuffix); k6Util.trackResponseSizePerURL(res, crdUtil.crdsTag, headerDataRecv, epDataRecv); sleep(0.25); } sleep(0.15); let { res, crdArray: crds } = crdUtil.getCRDsMatchingName(baseUrl, cookies, namePrefix); console.log("RES Status: ", res.status); console.log("NUM CRDS: ", crds.length); if (Array.isArray(crds) && crds.length) crdArray = crds; if (res.status != 200 && attempts == (retries - 1)) fail("Failed to retrieve expected CRDs during setup"); attempts += 1; } if (crdArray.length != crdCount) fail(`Failed to create expected # of CRDs (${crdCount}), got (${crdArray.length})`); sleep(300); return crdArray; } export function loadCRDs(data) { const timeWas = new Date(); while (new Date() - timeWas < crdUtil.backgroundRefreshMs) { let crds = data.crdArray; crds.forEach(c => { if (c.spec.versions.length != 2) { fail("CRD DOES NOT HAVE EXPECTED # OF VERSIONS (2)"); } let res = crdUtil.getCRD(baseUrl, data.cookies, c.id); let modifyCRD = JSON.parse(res.body); k6Util.trackResponseSizePerURL(res, crdUtil.crdTag, headerDataRecv, epDataRecv); modifyCRD.spec.versions[1].storage = false; modifyCRD.spec.versions[2] = newSchema; if (modifyCRD.spec.versions.length != 3) { fail("CRD DOES NOT HAVE EXPECTED # OF VERSIONS (3)"); } res = crdUtil.updateCRD(baseUrl, data.cookies, modifyCRD); k6Util.trackResponseSizePerURL(res, crdUtil.putCRDTag, headerDataRecv, epDataRecv); }) let { res, timeSpent } = crdUtil.verifyCRDs(baseUrl, data.cookies, namePrefix, crdCount, crdUtil.crdRefreshDelayMs * 5); console.log("VERIFY STATUS: ", res.status); timePolled.add(timeSpent, crdUtil.crdsTag); let numUpdated = JSON.parse(res.body)["data"].filter(r => r["metadata"]["name"].startsWith(namePrefix)).filter(r => r.spec.versions.length == 3).length; check((numUpdated / crdCount), { 'Total % of CRDs reflecting the newly added version >= 99%': (v) => v >= 0.99, }); } const verifyCRDsTime = new Date(); let verifyCRDsTimeSpent = null; let crdArray = []; while (new Date() - verifyCRDsTime < (crdUtil.crdRefreshDelayMs * 5)) { crdArray = crdUtil.getCRDsMatchingNameVersions(baseUrl, data.cookies, namePrefix, 3); if (crdArray.length == crdCount) { verifyCRDsTimeSpent = new Date() - timeWas; console.log("VERIFIED CRDs AFTER: ", verifyCRDsTimeSpent); timePolled.add(verifyCRDsTimeSpent, crdUtil.crdsTag); } } if (crdArray.length != crdCount) fail(`Did not get the expected count (${crdCount}) of updated CRDs, got (${crdArray.length}) instead`) } export function teardown(data) { cleanup(data.cookies) }