feat(go-feature-flag-web): Add support for data collection (#1101)

Signed-off-by: Thomas Poignant <thomas.poignant@gofeatureflag.org>
This commit is contained in:
Thomas Poignant 2024-12-03 21:32:11 +01:00 committed by GitHub
parent 775a7c88d2
commit 34fcecd78b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 532 additions and 48 deletions

View File

@ -0,0 +1,143 @@
import fetchMock from 'fetch-mock-jest';
import { GoffApiController } from './goff-api';
import { GoFeatureFlagWebProviderOptions } from '../model';
describe('Collect Data API', () => {
beforeEach(() => {
fetchMock.mockClear();
fetchMock.mockReset();
jest.resetAllMocks();
});
it('should call the API to collect data with apiKey', async () => {
fetchMock.post('https://gofeatureflag.org/v1/data/collector', 200);
const options: GoFeatureFlagWebProviderOptions = {
endpoint: 'https://gofeatureflag.org',
apiTimeout: 1000,
apiKey: '123456',
};
const goff = new GoffApiController(options);
await goff.collectData(
[
{
key: 'flagKey',
contextKind: 'user',
creationDate: 1733138237486,
default: false,
kind: 'feature',
userKey: 'toto',
value: true,
variation: 'varA',
},
],
{ provider: 'open-feature-js-sdk' },
);
expect(fetchMock.lastUrl()).toBe('https://gofeatureflag.org/v1/data/collector');
expect(fetchMock.lastOptions()?.headers).toEqual({
'Content-Type': 'application/json',
Accept: 'application/json',
Authorization: `Bearer ${options.apiKey}`,
});
expect(fetchMock.lastOptions()?.body).toEqual(
JSON.stringify({
events: [
{
key: 'flagKey',
contextKind: 'user',
creationDate: 1733138237486,
default: false,
kind: 'feature',
userKey: 'toto',
value: true,
variation: 'varA',
},
],
meta: { provider: 'open-feature-js-sdk' },
}),
);
});
it('should call the API to collect data', async () => {
fetchMock.post('https://gofeatureflag.org/v1/data/collector', 200);
const options: GoFeatureFlagWebProviderOptions = {
endpoint: 'https://gofeatureflag.org',
apiTimeout: 1000,
};
const goff = new GoffApiController(options);
await goff.collectData(
[
{
key: 'flagKey',
contextKind: 'user',
creationDate: 1733138237486,
default: false,
kind: 'feature',
userKey: 'toto',
value: true,
variation: 'varA',
},
],
{ provider: 'open-feature-js-sdk' },
);
expect(fetchMock.lastUrl()).toBe('https://gofeatureflag.org/v1/data/collector');
expect(fetchMock.lastOptions()?.headers).toEqual({
'Content-Type': 'application/json',
Accept: 'application/json',
});
expect(fetchMock.lastOptions()?.body).toEqual(
JSON.stringify({
events: [
{
key: 'flagKey',
contextKind: 'user',
creationDate: 1733138237486,
default: false,
kind: 'feature',
userKey: 'toto',
value: true,
variation: 'varA',
},
],
meta: { provider: 'open-feature-js-sdk' },
}),
);
});
it('should not call the API to collect data if no event provided', async () => {
fetchMock.post('https://gofeatureflag.org/v1/data/collector', 200);
const options: GoFeatureFlagWebProviderOptions = {
endpoint: 'https://gofeatureflag.org',
apiTimeout: 1000,
apiKey: '123456',
};
const goff = new GoffApiController(options);
await goff.collectData([], { provider: 'open-feature-js-sdk' });
expect(fetchMock).toHaveBeenCalledTimes(0);
});
it('should throw an error if API call fails', async () => {
fetchMock.post('https://gofeatureflag.org/v1/data/collector', 500);
const options: GoFeatureFlagWebProviderOptions = {
endpoint: 'https://gofeatureflag.org',
apiTimeout: 1000,
};
const goff = new GoffApiController(options);
await expect(
goff.collectData(
[
{
key: 'flagKey',
contextKind: 'user',
creationDate: 1733138237486,
default: false,
kind: 'feature',
userKey: 'toto',
value: true,
variation: 'varA',
},
],
{ provider: 'open-feature-js-sdk' },
),
).rejects.toThrow('impossible to send the data to the collector');
});
});

View File

@ -0,0 +1,54 @@
import { DataCollectorRequest, FeatureEvent, GoFeatureFlagWebProviderOptions } from '../model';
import { CollectorError } from '../errors/collector-error';
export class GoffApiController {
// endpoint of your go-feature-flag relay proxy instance
private readonly endpoint: string;
// timeout in millisecond before we consider the request as a failure
private readonly timeout: number;
private options: GoFeatureFlagWebProviderOptions;
constructor(options: GoFeatureFlagWebProviderOptions) {
this.endpoint = options.endpoint;
this.timeout = options.apiTimeout ?? 0;
this.options = options;
}
async collectData(events: FeatureEvent<any>[], dataCollectorMetadata: Record<string, string>) {
if (events?.length === 0) {
return;
}
const request: DataCollectorRequest<boolean> = { events: events, meta: dataCollectorMetadata };
const endpointURL = new URL(this.endpoint);
endpointURL.pathname = 'v1/data/collector';
try {
const headers: HeadersInit = {
'Content-Type': 'application/json',
Accept: 'application/json',
};
if (this.options.apiKey) {
headers['Authorization'] = `Bearer ${this.options.apiKey}`;
}
const controller = new AbortController();
const id = setTimeout(() => controller.abort(), this.timeout ?? 10000);
const response = await fetch(endpointURL.toString(), {
method: 'POST',
headers: headers,
body: JSON.stringify(request),
signal: controller.signal,
});
clearTimeout(id);
if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}
} catch (e) {
throw new CollectorError(`impossible to send the data to the collector: ${e}`);
}
}
}

View File

@ -0,0 +1,91 @@
import { EvaluationDetails, FlagValue, Hook, HookContext, Logger } from '@openfeature/server-sdk';
import { FeatureEvent, GoFeatureFlagWebProviderOptions } from './model';
import { copy } from 'copy-anything';
import { CollectorError } from './errors/collector-error';
import { GoffApiController } from './controller/goff-api';
const defaultTargetingKey = 'undefined-targetingKey';
type Timer = ReturnType<typeof setInterval>;
export class GoFeatureFlagDataCollectorHook implements Hook {
// bgSchedulerId contains the id of the setInterval that is running.
private bgScheduler?: Timer;
// dataCollectorBuffer contains all the FeatureEvents that we need to send to the relay-proxy for data collection.
private dataCollectorBuffer?: FeatureEvent<any>[];
// dataFlushInterval interval time (in millisecond) we use to call the relay proxy to collect data.
private readonly dataFlushInterval: number;
// dataCollectorMetadata are the metadata used when calling the data collector endpoint
private readonly dataCollectorMetadata: Record<string, string> = {
provider: 'open-feature-js-sdk',
};
private readonly goffApiController: GoffApiController;
// logger is the Open Feature logger to use
private logger?: Logger;
constructor(options: GoFeatureFlagWebProviderOptions, logger?: Logger) {
this.dataFlushInterval = options.dataFlushInterval || 1000 * 60;
this.logger = logger;
this.goffApiController = new GoffApiController(options);
}
init() {
this.bgScheduler = setInterval(async () => await this.callGoffDataCollection(), this.dataFlushInterval);
this.dataCollectorBuffer = [];
}
async close() {
clearInterval(this.bgScheduler);
// We call the data collector with what is still in the buffer.
await this.callGoffDataCollection();
}
/**
* callGoffDataCollection is a function called periodically to send the usage of the flag to the
* central service in charge of collecting the data.
*/
async callGoffDataCollection() {
const dataToSend = copy(this.dataCollectorBuffer) || [];
this.dataCollectorBuffer = [];
try {
await this.goffApiController.collectData(dataToSend, this.dataCollectorMetadata);
} catch (e) {
if (!(e instanceof CollectorError)) {
throw e;
}
this.logger?.error(e);
// if we have an issue calling the collector, we put the data back in the buffer
this.dataCollectorBuffer = [...this.dataCollectorBuffer, ...dataToSend];
return;
}
}
after(hookContext: HookContext, evaluationDetails: EvaluationDetails<FlagValue>) {
const event = {
contextKind: hookContext.context['anonymous'] ? 'anonymousUser' : 'user',
kind: 'feature',
creationDate: Math.round(Date.now() / 1000),
default: false,
key: hookContext.flagKey,
value: evaluationDetails.value,
variation: evaluationDetails.variant || 'SdkDefault',
userKey: hookContext.context.targetingKey || defaultTargetingKey,
source: 'PROVIDER_CACHE',
};
this.dataCollectorBuffer?.push(event);
}
error(hookContext: HookContext) {
const event = {
contextKind: hookContext.context['anonymous'] ? 'anonymousUser' : 'user',
kind: 'feature',
creationDate: Math.round(Date.now() / 1000),
default: true,
key: hookContext.flagKey,
value: hookContext.defaultValue,
variation: 'SdkDefault',
userKey: hookContext.context.targetingKey || defaultTargetingKey,
source: 'PROVIDER_CACHE',
};
this.dataCollectorBuffer?.push(event);
}
}

View File

@ -0,0 +1,10 @@
import { GoFeatureFlagError } from './goff-error';
/**
* An error occurred while calling the GOFF event collector.
*/
export class CollectorError extends GoFeatureFlagError {
constructor(message?: string, originalError?: Error) {
super(`${message}: ${originalError}`);
}
}

View File

@ -0,0 +1 @@
export class GoFeatureFlagError extends Error {}

View File

@ -1,12 +1,12 @@
import { GoFeatureFlagWebProvider } from './go-feature-flag-web-provider';
import {
ErrorCode,
EvaluationContext,
EvaluationDetails,
JsonValue,
OpenFeature,
ProviderEvents,
StandardResolutionReasons,
ErrorCode,
EvaluationDetails,
JsonValue,
} from '@openfeature/web-sdk';
import WS from 'jest-websocket-mock';
import TestLogger from './test-logger';
@ -17,6 +17,7 @@ describe('GoFeatureFlagWebProvider', () => {
let websocketMockServer: WS;
const endpoint = 'http://localhost:1031/';
const allFlagsEndpoint = `${endpoint}v1/allflags`;
const dataCollectorEndpoint = `${endpoint}v1/data/collector`;
const websocketEndpoint = 'ws://localhost:1031/ws/v1/flag/change';
const defaultAllFlagResponse = {
flags: {
@ -95,6 +96,7 @@ describe('GoFeatureFlagWebProvider', () => {
await jest.resetAllMocks();
websocketMockServer = new WS(websocketEndpoint, { jsonProtocol: true });
fetchMock.post(allFlagsEndpoint, defaultAllFlagResponse);
fetchMock.post(dataCollectorEndpoint, 200);
defaultProvider = new GoFeatureFlagWebProvider(
{
endpoint: endpoint,
@ -128,6 +130,7 @@ describe('GoFeatureFlagWebProvider', () => {
endpoint: endpoint,
apiTimeout: 1000,
maxRetries: 1,
disableDataCollection: true,
},
logger,
);
@ -451,4 +454,156 @@ describe('GoFeatureFlagWebProvider', () => {
expect(staleHandler).toBeCalled();
});
});
describe('data collector testing', () => {
it('should call the data collector when closing Open Feature', async () => {
const clientName = expect.getState().currentTestName ?? 'test-provider';
await OpenFeature.setContext(defaultContext);
const p = new GoFeatureFlagWebProvider(
{
endpoint: endpoint,
apiTimeout: 1000,
maxRetries: 1,
dataFlushInterval: 10000,
apiKey: 'toto',
},
logger,
);
OpenFeature.setProvider(clientName, p);
const client = OpenFeature.getClient(clientName);
await websocketMockServer.connected;
await new Promise((resolve) => setTimeout(resolve, 5));
client.getBooleanDetails('bool_flag', false);
client.getBooleanDetails('bool_flag', false);
await OpenFeature.close();
expect(fetchMock.calls(dataCollectorEndpoint).length).toBe(1);
expect(fetchMock.lastOptions(dataCollectorEndpoint)?.headers).toEqual({
'Content-Type': 'application/json',
Accept: 'application/json',
Authorization: 'Bearer toto',
});
});
it('should call the data collector when waiting more than the dataFlushInterval', async () => {
const clientName = expect.getState().currentTestName ?? 'test-provider';
await OpenFeature.setContext(defaultContext);
const p = new GoFeatureFlagWebProvider(
{
endpoint: endpoint,
apiTimeout: 1000,
maxRetries: 1,
dataFlushInterval: 200,
},
logger,
);
OpenFeature.setProvider(clientName, p);
const client = OpenFeature.getClient(clientName);
await websocketMockServer.connected;
await new Promise((resolve) => setTimeout(resolve, 5));
client.getBooleanDetails('bool_flag', false);
client.getBooleanDetails('bool_flag', false);
await new Promise((resolve) => setTimeout(resolve, 300));
expect(fetchMock.calls(dataCollectorEndpoint).length).toBe(1);
expect(fetchMock.lastOptions(dataCollectorEndpoint)?.headers).toEqual({
'Content-Type': 'application/json',
Accept: 'application/json',
});
await OpenFeature.close();
});
it('should call the data collector multiple time while waiting dataFlushInterval time', async () => {
const clientName = expect.getState().currentTestName ?? 'test-provider';
await OpenFeature.setContext(defaultContext);
const p = new GoFeatureFlagWebProvider(
{
endpoint: endpoint,
apiTimeout: 1000,
maxRetries: 1,
dataFlushInterval: 200,
},
logger,
);
OpenFeature.setProvider(clientName, p);
const client = OpenFeature.getClient(clientName);
await websocketMockServer.connected;
await new Promise((resolve) => setTimeout(resolve, 5));
client.getBooleanDetails('bool_flag', false);
client.getBooleanDetails('bool_flag', false);
await new Promise((resolve) => setTimeout(resolve, 250));
client.getBooleanDetails('bool_flag', false);
client.getBooleanDetails('bool_flag', false);
await new Promise((resolve) => setTimeout(resolve, 300));
expect(fetchMock.calls(dataCollectorEndpoint).length).toBe(2);
await OpenFeature.close();
});
it('should not call the data collector before the dataFlushInterval', async () => {
const clientName = expect.getState().currentTestName ?? 'test-provider';
await OpenFeature.setContext(defaultContext);
const p = new GoFeatureFlagWebProvider(
{
endpoint: endpoint,
apiTimeout: 1000,
maxRetries: 1,
dataFlushInterval: 200,
},
logger,
);
OpenFeature.setProvider(clientName, p);
const client = OpenFeature.getClient(clientName);
await websocketMockServer.connected;
await new Promise((resolve) => setTimeout(resolve, 5));
client.getBooleanDetails('bool_flag', false);
client.getBooleanDetails('bool_flag', false);
await new Promise((resolve) => setTimeout(resolve, 100));
expect(fetchMock.calls(dataCollectorEndpoint).length).toBe(0);
await OpenFeature.close();
});
it('should have a log when data collector is not available', async () => {
const clientName = expect.getState().currentTestName ?? 'test-provider';
fetchMock.post(dataCollectorEndpoint, 500, { overwriteRoutes: true });
OpenFeature.setContext(defaultContext);
const p = new GoFeatureFlagWebProvider(
{
endpoint: endpoint,
apiTimeout: 1000,
maxRetries: 1,
dataFlushInterval: 200,
},
logger,
);
OpenFeature.setProvider(clientName, p);
const client = OpenFeature.getClient(clientName);
await websocketMockServer.connected;
await new Promise((resolve) => setTimeout(resolve, 5));
client.getBooleanDetails('bool_flag', false);
client.getBooleanDetails('bool_flag', false);
await new Promise((resolve) => setTimeout(resolve, 250));
fetchMock.post(dataCollectorEndpoint, 500, { overwriteRoutes: true });
client.getBooleanDetails('bool_flag', false);
client.getBooleanDetails('bool_flag', false);
fetchMock.post(dataCollectorEndpoint, 200, { overwriteRoutes: true });
await new Promise((resolve) => setTimeout(resolve, 250));
const lastBody = fetchMock.lastOptions(dataCollectorEndpoint)?.body;
const parsedBody = JSON.parse(lastBody as never);
expect(parsedBody['events'].length).toBe(4);
await OpenFeature.close();
});
});
});

View File

@ -2,12 +2,12 @@ import {
EvaluationContext,
FlagNotFoundError,
FlagValue,
Hook,
Logger,
OpenFeature,
OpenFeatureEventEmitter,
Provider,
ProviderEvents,
ProviderStatus,
ResolutionDetails,
StandardResolutionReasons,
TypeMismatchError,
@ -20,16 +20,17 @@ import {
GOFeatureFlagWebsocketResponse,
} from './model';
import { transformContext } from './context-transformer';
import { FetchError } from './fetch-error';
import { FetchError } from './errors/fetch-error';
import { GoFeatureFlagDataCollectorHook } from './data-collector-hook';
export class GoFeatureFlagWebProvider implements Provider {
private readonly _websocketPath = 'ws/v1/flag/change';
metadata = {
name: GoFeatureFlagWebProvider.name,
};
events = new OpenFeatureEventEmitter();
// hooks is the list of hooks that are used by the provider
hooks?: Hook[];
private readonly _websocketPath = 'ws/v1/flag/change';
// logger is the Open Feature logger to use
private _logger?: Logger;
// endpoint of your go-feature-flag relay proxy instance
@ -38,19 +39,19 @@ export class GoFeatureFlagWebProvider implements Provider {
private readonly _apiTimeout: number;
// apiKey is the key used to identify your request in GO Feature Flag
private readonly _apiKey: string | undefined;
// initial delay in millisecond to wait before retrying to connect
private readonly _retryInitialDelay;
// multiplier of _retryInitialDelay after each failure
private readonly _retryDelayMultiplier;
// maximum number of retries
private readonly _maxRetries;
// status of the provider
private _status: ProviderStatus = ProviderStatus.NOT_READY;
// _websocket is the reference to the websocket connection
private _websocket?: WebSocket;
// _flags is the in memory representation of all the flags.
private _flags: { [key: string]: ResolutionDetails<FlagValue> } = {};
private readonly _dataCollectorHook: GoFeatureFlagDataCollectorHook;
// disableDataCollection set to true if you don't want to collect the usage of flags retrieved in the cache.
private readonly _disableDataCollection: boolean;
constructor(options: GoFeatureFlagWebProviderOptions, logger?: Logger) {
this._logger = logger;
@ -60,23 +61,23 @@ export class GoFeatureFlagWebProvider implements Provider {
this._retryDelayMultiplier = options.retryDelayMultiplier || 2;
this._maxRetries = options.maxRetries || 10;
this._apiKey = options.apiKey;
}
get status(): ProviderStatus {
return this._status;
this._disableDataCollection = options.disableDataCollection || false;
this._dataCollectorHook = new GoFeatureFlagDataCollectorHook(options, logger);
}
async initialize(context: EvaluationContext): Promise<void> {
if (!this._disableDataCollection && this._dataCollectorHook) {
this.hooks = [this._dataCollectorHook];
this._dataCollectorHook.init();
}
return Promise.all([this.fetchAll(context), this.connectWebsocket()])
.then(() => {
this._status = ProviderStatus.READY;
this._logger?.debug(`${GoFeatureFlagWebProvider.name}: go-feature-flag provider initialized`);
})
.catch((error) => {
this._logger?.error(
`${GoFeatureFlagWebProvider.name}: initialization failed, provider is on error, we will try to reconnect: ${error}`,
);
this._status = ProviderStatus.ERROR;
this.handleFetchErrors(error);
// The initialization of the provider is in a failing state, we unblock the initialize method,
@ -144,6 +145,37 @@ export class GoFeatureFlagWebProvider implements Provider {
});
}
async onClose(): Promise<void> {
if (!this._disableDataCollection && this._dataCollectorHook) {
await this._dataCollectorHook?.close();
}
this._websocket?.close(1000, 'Closing GO Feature Flag provider');
return Promise.resolve();
}
async onContextChange(_: EvaluationContext, newContext: EvaluationContext): Promise<void> {
this._logger?.debug(`${GoFeatureFlagWebProvider.name}: new context provided: ${newContext}`);
this.events.emit(ProviderEvents.Stale, { message: 'context has changed' });
await this.retryFetchAll(newContext);
this.events.emit(ProviderEvents.Ready, { message: '' });
}
resolveNumberEvaluation(flagKey: string): ResolutionDetails<number> {
return this.evaluate(flagKey, 'number');
}
resolveObjectEvaluation<T extends FlagValue>(flagKey: string): ResolutionDetails<T> {
return this.evaluate(flagKey, 'object');
}
resolveStringEvaluation(flagKey: string): ResolutionDetails<string> {
return this.evaluate(flagKey, 'string');
}
resolveBooleanEvaluation(flagKey: string): ResolutionDetails<boolean> {
return this.evaluate(flagKey, 'boolean');
}
/**
* extract flag names from the websocket answer
*/
@ -186,34 +218,6 @@ export class GoFeatureFlagWebProvider implements Provider {
});
}
onClose(): Promise<void> {
this._websocket?.close(1000, 'Closing GO Feature Flag provider');
return Promise.resolve();
}
async onContextChange(_: EvaluationContext, newContext: EvaluationContext): Promise<void> {
this._logger?.debug(`${GoFeatureFlagWebProvider.name}: new context provided: ${newContext}`);
this.events.emit(ProviderEvents.Stale, { message: 'context has changed' });
await this.retryFetchAll(newContext);
this.events.emit(ProviderEvents.Ready, { message: '' });
}
resolveNumberEvaluation(flagKey: string): ResolutionDetails<number> {
return this.evaluate(flagKey, 'number');
}
resolveObjectEvaluation<T extends FlagValue>(flagKey: string): ResolutionDetails<T> {
return this.evaluate(flagKey, 'object');
}
resolveStringEvaluation(flagKey: string): ResolutionDetails<string> {
return this.evaluate(flagKey, 'string');
}
resolveBooleanEvaluation(flagKey: string): ResolutionDetails<boolean> {
return this.evaluate(flagKey, 'boolean');
}
private evaluate<T extends FlagValue>(flagKey: string, type: string): ResolutionDetails<T> {
const resolved = this._flags[flagKey];
if (!resolved) {
@ -240,10 +244,8 @@ export class GoFeatureFlagWebProvider implements Provider {
attempt++;
try {
await this.fetchAll(ctx, flagsChanged);
this._status = ProviderStatus.READY;
return;
} catch (err) {
this._status = ProviderStatus.ERROR;
this.handleFetchErrors(err);
await new Promise((resolve) => setTimeout(resolve, delay));
delay *= this._retryDelayMultiplier;

View File

@ -1,5 +1,5 @@
/* eslint-disable @typescript-eslint/no-explicit-any */
import { FlagValue, ErrorCode, EvaluationContextValue } from '@openfeature/web-sdk';
import { ErrorCode, EvaluationContextValue, FlagValue } from '@openfeature/web-sdk';
/**
* GoFeatureFlagEvaluationContext is the representation of a user for GO Feature Flag
@ -30,6 +30,7 @@ export interface GoFeatureFlagWebProviderOptions {
endpoint: string;
// timeout is the time in millisecond we wait for an answer from the server.
// Default: 10000 ms
apiTimeout?: number;
// apiKey (optional) If the relay proxy is configured to authenticate the requests, you should provide
@ -50,6 +51,15 @@ export interface GoFeatureFlagWebProviderOptions {
// maximum number of retries before considering GO Feature Flag is unreachable
// Default: 10
maxRetries?: number;
// dataFlushInterval (optional) interval time (in millisecond) we use to call the relay proxy to collect data.
// The parameter is used only if the cache is enabled, otherwise the collection of the data is done directly
// when calling the evaluation API.
// default: 1 minute
dataFlushInterval?: number;
// disableDataCollection set to true if you don't want to collect the usage of flags retrieved in the cache.
disableDataCollection?: boolean;
}
/**
@ -84,3 +94,21 @@ export interface GOFeatureFlagWebsocketResponse {
added?: { [key: string]: any };
updated?: { [key: string]: any };
}
export interface DataCollectorRequest<T> {
events: FeatureEvent<T>[];
meta: Record<string, string>;
}
export interface FeatureEvent<T> {
contextKind: string;
creationDate: number;
default: boolean;
key: string;
kind: string;
userKey: string;
value: T;
variation: string;
version?: string;
source?: string;
}