landscapeapp/tools/apiClients.js

const { env } = require('process');
const { stringify, parse } = require('query-string');
const axios = require('axios');
const OAuth1 = require('oauth-1.0a');
const crypto = require('crypto');
const _ = require('lodash');

['GITHUB_KEY', 'TWITTER_KEYS'].forEach((key) => {
  if (!env[key]) {
    console.info(`${key} not provided`);
  }
});

let requests = {};

const maxAttempts = 5
const delay = 30000

const getOauth1Header = config => {
  const { method = 'GET', url = {}, params } = config
  const data = parse(stringify(params))
  const request = { method, url, data }
  const { consumer_key, consumer_secret, access_token_key, access_token_secret } = config.oauth

  const oauth = OAuth1({
    consumer: {
      key: consumer_key,
      secret: consumer_secret,
    },
    signature_method: 'HMAC-SHA1',
    hash_function(base_string, key) {
      return crypto
        .createHmac('sha1', key)
        .update(base_string)
        .digest('base64')
    },
  })

  const authorization = oauth.authorize(request, {
    key: access_token_key,
    secret: access_token_secret,
  });

  return oauth.toHeader(authorization);
}

const requestWithRetry = async ({ attempts = maxAttempts, resolveWithFullResponse, retryStatuses, delayFn, applyKey, keys, ...rest }) => {
  if (!applyKey) {
    keys = [true];
    applyKey = () => true;
  }

  let lastEx = null;
  for (var key of keys) {
    applyKey(rest, key);
    try {
      axios.interceptors.request.use(function (config) {
        const authHeader = config.oauth ? getOauth1Header(config) : {}
        return { ...config, headers: { ...config.headers, ...authHeader } }
      })

      const response = await axios(rest);
      return resolveWithFullResponse ? response : response.data
    } catch (ex) {

      const { response = {}, ...error } = ex
      const { status } = response

      const isGithubIssue = (response?.data?.message || '').indexOf('is too large to list') !== -1;

      const message = [
        `Attempt #${maxAttempts - attempts + 1}`,
        `(Status Code: ${status || error.code})`,
        `(URI: ${rest.url})`
      ].join(' ')
      if (key === keys[keys.length - 1]) {
        console.info(message);
      } else {
        console.info(`Failed to use key #${keys.indexOf(key)} of ${keys.length}`);
      }
      const rateLimited = retryStatuses.includes(status)
      const dnsError = error.code === 'ENOTFOUND' && error.syscall === 'getaddrinfo'
      if (attempts <= 0 || (!rateLimited && !dnsError) || isGithubIssue) {
        throw ex;
      }
      lastEx = ex;
    }
  }
  await new Promise(r => setTimeout(r, delayFn ? delayFn(lastEx) : delay))
  return await requestWithRetry({ attempts: attempts - 1, retryStatuses, delayFn, ...rest });
}

// We only want to retry a request when rate limited. By default the status code is 429.
const ApiClient = ({ baseURL, applyKey, keys, defaultOptions = {}, defaultParams = {}, retryStatuses = [429], delayFn = null }) => {
  return {
    request: async ({ path = null, url = null, method = 'GET', params = {}, resolveWithFullResponse = false, ...rest }) => {
      const queryParams = { ...defaultParams, ...params };

      if (path) {
        url = `${baseURL}${path[0] === '/' ? '' : '/' }${path}`;
      }

      const key = `${method} ${url}?${stringify(queryParams)}`;

      if (!requests[key]) {
        requests[key] = requestWithRetry({
          applyKey: applyKey,
          keys: keys,
          method: method,
          url: url,
          params: queryParams,
          ...defaultOptions,
          ...rest,
          retryStatuses,
          delayFn,
          resolveWithFullResponse
        })
      }

      return await requests[key];
    }
  }
};

module.exports.CrunchbaseClient = ApiClient({
  baseURL: 'https://api.crunchbase.com/api/v4',
  defaultParams: { user_key: env.CRUNCHBASE_KEY_4 },
  defaultOptions: { followRedirect: true, maxRedirects: 5, timeout: 10 * 1000 }
});

module.exports.GithubClient = ApiClient({
  baseURL: 'https://api.github.com',
  retryStatuses: [401, 403], // Github returns 403 when rate limiting.
  delayFn: error => {
    const rateLimitRemaining = parseInt(_.get(error, ['response', 'headers', 'x-ratelimit-remaining'], 1))
    const rateLimitReset = parseInt(_.get(error, ['response', 'headers', 'x-ratelimit-reset'], 1)) * 1000
    if (rateLimitRemaining > 0) {
      return 30000
    } else {
      const delay = Math.max(Math.ceil((new Date(rateLimitReset)) - (new Date())), 60000)
      console.log(`Hourly rate limit exceeded on Github, delaying for ${Math.round(delay / 1000 / 60)} minutes`)
      return delay
    }
  },
  defaultOptions: {
    followRedirect: true,
    timeout: 10 * 1000,
    headers: {
      'User-agent': 'CNCF'
    },
  },
  keys: (env.GITHUB_KEY || '').split(',').map( (x) => x.trim()),
  applyKey: (options, key) => options.headers.Authorization = `token ${key}`
});

const [consumerKey, consumerSecret, accessTokenKey, accessTokenSecret] = (env.TWITTER_KEYS || '').split(',');

module.exports.TwitterClient = ApiClient({
  baseURL: 'https://api.twitter.com/1.1',
  defaultOptions: {
    oauth: {
      consumer_key: consumerKey,
      consumer_secret: consumerSecret,
      access_token_key: accessTokenKey,
      access_token_secret: accessTokenSecret
    }
  }
});

module.exports.YahooFinanceClient = ApiClient({
  baseURL: 'https://query2.finance.yahoo.com',
});