uniq items

* Re-write `mobile.twitter.com` to `twitter.com`

Signed-off-by: John Mertic <jmertic@linuxfoundation.org>

* Fix typo

---------

Signed-off-by: John Mertic <jmertic@linuxfoundation.org>

.nvmrc

@@ -1 +1 @@
-v18.1
+v18.3

.yarnrc.yml

@@ -1,3 +1,3 @@
 enableProgressBars: false
 
-yarnPath: .yarn/releases/yarn-3.2.0.cjs
+yarnPath: .yarn/releases/yarn-3.2.1.cjs

_headers

@@ -2,15 +2,6 @@
   X-Robots-Tag: all
   Access-Control-Allow-Origin: *
 
-/*.js
-  Cache-Control: public, max-age=31536000, immutable
-
-/*.css
-  Cache-Control: public, max-age=31536000, immutable
-
-/*.woff2
-  Cache-Control: public, max-age=31536000, immutable
-
 /*.svg
   Content-Type: image/svg+xml; charset=utf-8
 

landscapes.yml

@@ -1,19 +1,30 @@
-ip: 145.40.64.211
+ip: 147.75.72.237
 landscapes:
   # name: how we name a landscape project, used on a build server for logs and settings
   # repo: a github repo for a specific landscape
   # netlify: full | skip - do we build it on a netlify build or not
   # hook: - id for a build hook, so it will be triggered after a default branch build
+  - landscape:
+    name: lfedge
+    repo: State-of-the-Edge/lfedge-landscape
+    hook: 5c80e31894c5c7758edb31e4
+  - landscape:
+    name: lfenergy
+    repo: lf-energy/lfenergy-landscape
+    hook: 606487bb9da603110d4b8139
+  - landscape:
+    name: lf
+    repo: jmertic/lf-landscape
+    hook: 606487123224e20fb8f3896e
+  - landscape:
+    name: dlt
+    repo: dltlandscape/dlt-landscape
+    hook: demo
   - landscape:
     name: aswf-landscape
     repo: AcademySoftwareFoundation/aswf-landscape
     hook: 608aa68eb6e5723d5d8a7e00
     required: true
-  - landscape:
-    name: cncf
-    repo: cncf/landscape
-    hook: 5c1bd968fdd72a78a54bdcd1
-    required: true
   - landscape:
     name: cdf
     repo: cdfoundation/cdf-landscape
@@ -32,23 +43,11 @@ landscapes:
     repo: graphql/graphql-landscape
     hook: 5d5c7ccf64ecb5bd3d2592f7
     required: true
-  - landscape:
-    name: lf
-    repo: jmertic/lf-landscape
-    hook: 606487123224e20fb8f3896e
   - landscape:
     name: lfai
     repo: lfai/landscape
     hook: 60648e5b74c76017210a2f53
     required: true
-  - landscape:
-    name: lfedge
-    repo: State-of-the-Edge/lfedge-landscape
-    hook: 5c80e31894c5c7758edb31e4
-  - landscape:
-    name: lfenergy
-    repo: lf-energy/lfenergy-landscape
-    hook: 606487bb9da603110d4b8139
   - landscape:
     name: lfph
     repo: lfph/lfph-landscape
@@ -62,10 +61,6 @@ landscapes:
     name: openssf
     repo: ossf/ossf-landscape
     hook: 613768338a16cb9182b21c3d
-  - landscape:
-    name: ospo
-    repo: todogroup/ospolandscape
-    hook: 6033b43a2572da242aee6a92
   - landscape:
     name: presto
     repo: prestodb/presto-landscape
@@ -82,3 +77,7 @@ landscapes:
     name: lfn
     repo: lfnetworking/member_landscape
     hook: 60788f5fa3cbd68181e3c209
+  - landscape:
+    name: riscv
+    repo: riscv-admin/riscv-landscape
+    hook: demo

netlify.md

@@ -9,3 +9,5 @@ server via rsync.
 
 Most chances is that we will switch to a different build tool soon, so this is
 an experimental approach to speedup netlify builds.
+
+

netlify/landscape.js

@@ -1,7 +1,6 @@
 // We will execute this script from a landscape build,
 // "prepublish": "cp yarn.lock _yarn.lock",
 // "postpublish": "rm _yarn.lock || true"
-const LANDSCAPEAPP = process.env.LANDSCAPEAPP || "@latest"
 const remote = `root@${process.env.BUILD_SERVER}`;
 const dockerImage = 'netlify/build:focal';
 const dockerHome = '/opt/buildhome';
@@ -31,28 +30,20 @@ const debug = function() {
   }
 }
 
-const runLocal = function(command, options = {}) {
+const runLocal = function(command, showProgress) {
-  const { assignFn, showOutputFn } = options;
 
   // report the output once every 5 seconds
   let lastOutput = { s: '', time: new Date().getTime() };
   let displayIfRequired = function(text) {
+    if (showProgress) {
+      console.info(text);
+    }
     lastOutput.s = lastOutput.s + text;
-    if (showOutputFn && showOutputFn()) {
-      if (lastOutput.done || new Date().getTime() > lastOutput.time + 5 * 1000) {
-        console.info(lastOutput.s);
-        lastOutput.s = "";
-        lastOutput.time = new Date().getTime();
-      }
-    }
   }
 
   return new Promise(function(resolve) {
     var spawn = require('child_process').spawn;
     var child = spawn('bash', ['-lc',`set -e \n${command}`]);
-    if (assignFn) {
-      assignFn(child);
-    }
     let output = [];
     child.stdout.on('data', function(data) {
       const text = maskSecrets(data.toString('utf-8'));
@@ -93,7 +84,7 @@ ${(process.env.BUILDBOT_KEY || '').replace(/\s/g,'\n')}
 require('fs').writeFileSync('/tmp/buildbot', key);
 require('fs').chmodSync('/tmp/buildbot', 0o600);
 
-const runRemote = async function(command, options) {
+const runRemote = async function(command, count = 3) {
   const bashCommand = `
     nocheck=" -o StrictHostKeyChecking=no "
     ssh -i /tmp/buildbot $nocheck ${remote} << 'EOSSH'
@@ -101,7 +92,12 @@ const runRemote = async function(command, options) {
     ${command}
 EOSSH
 `
-  return await runLocal(bashCommand, options);
+  const result = await runLocal(bashCommand, true);
+  if (result.exitCode === 255 && count > 0) {
+    console.info(`Attempts to retry more: ${count}`);
+    return await runRemote(command, count - 1);
+  }
+  return result;
 };
 
 const runRemoteWithoutErrors = async function(command) {
@@ -114,16 +110,8 @@ const runRemoteWithoutErrors = async function(command) {
 
 const makeRemoteBuildWithCache = async function() {
   await runLocalWithoutErrors(`
-    echo extracting
+    rm -rf packageRemote || true
-    mkdir tmpRemote
+    git clone -b deploy --single-branch https://github.com/cncf/landscapeapp packageRemote
-    cd tmpRemote
-    rm -rf package || true
-    # npm pack github:cncf/landscapeapp#vanilla
-    npm pack interactive-landscape${LANDSCAPEAPP}
-    tar xzf interactive*.tgz
-    cd ..
-    mv tmpRemote/package packageRemote
-    cp packageRemote/_yarn.lock packageRemote/yarn.lock
   `);
 
   //how to get a hash based on our files
@@ -156,56 +144,6 @@ const makeRemoteBuildWithCache = async function() {
   const hash = getHash();
   const tmpHash = require('crypto').createHash('sha256').update(getTmpFile()).digest('hex');
   // lets guarantee npm install for this folder first
-  {
-    const buildCommand = [
-      "(ls . ~/.nvm/nvm.sh || (curl -s -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash >/dev/null))",
-      ". ~/.nvm/nvm.sh",
-      `nvm install ${nvmrc}`,
-      `echo 0`,
-      `nvm use ${nvmrc}`,
-      `echo 1`,
-      `npm install -g agentkeepalive --save`,
-      `npm install -g npm --no-progress`,
-      `npm install -g yarn@latest`,
-      `cd /opt/repo/packageRemote`,
-      `yarn >/dev/null`
-    ].join(' && ');
-    const npmInstallCommand = `
-      mkdir -p /root/builds/node_cache
-      ls -l /root/builds/node_cache/${hash}/yarnLocal/unplugged 2>/dev/null || (
-          mkdir -p /root/builds/node_cache/${tmpHash}/{yarnLocal,nvm,yarnGlobal}
-          cp -r /root/builds/${folder}/packageRemote/.yarn/* /root/builds/node_cache/${tmpHash}/yarnLocal
-          chmod -R 777 /root/builds/node_cache/${tmpHash}
-          docker run --shm-size 1G --rm -t \
-            -v /root/builds/node_cache/${tmpHash}/yarnLocal:/opt/repo/packageRemote/.yarn \
-            -v /root/builds/node_cache/${tmpHash}/nvm:${dockerHome}/.nvm \
-            -v /root/builds/node_cache/${tmpHash}/yarnGlobal:${dockerHome}/.yarn \
-            -v /root/builds/${folder}:/opt/repo \
-            ${dockerImage} /bin/bash -lc "${buildCommand}"
-
-          ln -s /root/builds/node_cache/${tmpHash} /root/builds/node_cache/${hash} || (
-            rm -rf /root/builds/node_cache/${tmpHash}
-          )
-          echo "packages for ${hash} had been installed"
-      )
-      chmod -R 777 /root/builds/node_cache/${hash}
-    `;
-    debug(npmInstallCommand);
-    console.info(`Remote with cache: Installing npm packages if required`);
-    const output = await runRemote(npmInstallCommand);
-    console.info(`Remote with cache: Output from npm install: exit code: ${output.exitCode}`);
-    if (output.exitCode !== 0) {
-      console.info(output.text);
-      throw new Error('Remote with cahce: npm install failed');
-    }
-
-    const lines = output.text.split('\n');
-    const index = lines.indexOf(lines.filter( (line) => line.match(/added \d+ packages in/))[0]);
-    const filteredLines = lines.slice(index !== -1 ? index : 0).join('\n');
-    console.info(filteredLines || 'Reusing an existing folder for node');
-
-  }
-
   // do not pass REVIEW_ID because on failure we will run it locally and report
   // from there
   const vars = [
@@ -221,9 +159,14 @@ const makeRemoteBuildWithCache = async function() {
   const outputFolder = 'landscape' + getTmpFile();
   const buildCommand = [
     `cd /opt/repo/packageRemote`,
+    "(ls . ~/.nvm/nvm.sh || (curl -s -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash >/dev/null))",
     `. ~/.nvm/nvm.sh`,
+    `cat .nvmrc`,
     `nvm install ${nvmrc}`,
     `nvm use ${nvmrc}`,
+    `npm install -g agentkeepalive --save`,
+    `npm install -g npm@9 --no-progress`,
+    `npm install -g yarn@latest`,
     `yarn`,
     `git config --global --add safe.directory /opt/repo`,
     `export NODE_OPTIONS="--unhandled-rejections=strict"`,
@@ -235,16 +178,12 @@ const makeRemoteBuildWithCache = async function() {
       mkdir -p /root/builds/${outputFolder}
       chmod -R 777 /root/builds/${outputFolder}
       chmod -R 777 /root/builds/${folder}
-      chmod -R 777 /root/builds/node_cache/${hash}
 
       docker run --shm-size 1G --rm -t \
         ${vars.map( (v) => ` -e ${v}="${process.env[v]}" `).join(' ')} \
         -e NVM_NO_PROGRESS=1 \
         -e NETLIFY=1 \
         -e PARALLEL=TRUE \
-        -v /root/builds/node_cache/${hash}/yarnLocal:/opt/repo/packageRemote/.yarn \
-        -v /root/builds/node_cache/${hash}/nvm:${dockerHome}/.nvm \
-        -v /root/builds/node_cache/${hash}/yarnGlobal:${dockerHome}/.yarn \
         -v /root/builds/${folder}:/opt/repo \
         -v /root/builds/${outputFolder}:/dist \
         ${dockerImage} /bin/bash -lc "${buildCommand}"
@@ -276,7 +215,7 @@ const makeRemoteBuildWithCache = async function() {
       rsync -az --chmod=a+r -p -e "ssh -i /tmp/buildbot  -o StrictHostKeyChecking=no " ${remote}:/root/builds/${outputFolder}/dist/* distRemote
     `
   ));
-  await runRemoteWithoutErrors(
+  await runRemote(
     `
       rm -rf /root/builds/${folder}
       rm -rf /root/builds/${outputFolder}
@@ -304,7 +243,6 @@ async function main() {
   await runLocal('rm package*.json');
 
   const cleanPromise = runRemoteWithoutErrors(`
-    find builds/node_cache -maxdepth 1 -mtime +1 -exec rm -rf {} +;
     find builds/ -maxdepth 1 -not -path "builds/node_cache" -mtime +1 -exec rm -rf {} +;
   `).catch(function() {
     console.info('Failed to clean up a builds folder');

netlify/landscapeapp.js

@@ -1,3 +1,14 @@
+if (process.env.KEY3) {
+  require('fs').mkdirSync(process.env.HOME + '/.ssh', { recursive: true});
+  require('fs').writeFileSync(process.env.HOME + '/.ssh/bot3',
+    "-----BEGIN RSA PRIVATE KEY-----\n" +
+    process.env.KEY3.replace(/\s/g,"\n") +
+    "\n-----END RSA PRIVATE KEY-----\n\n"
+  );
+  require('fs').chmodSync(process.env.HOME + '/.ssh/bot3', 0o600);
+  console.info('Made a bot3 file');
+}
+
 const path = require('path')
 const { readdirSync, writeFileSync } = require('fs')
 const generateIndex = require('./generateIndex')
@@ -20,6 +31,7 @@ const landscapesInfo = yaml.load(require('fs').readFileSync('landscapes.yml'));
 const dockerImage = 'netlify/build:focal';
 const dockerHome = '/opt/buildhome';
 
+
 async function main() {
   const nvmrc = require('fs').readFileSync('.nvmrc', 'utf-8').trim();
   const secrets = [
@@ -53,7 +65,7 @@ ${process.env.BUILDBOT_KEY.replace(/\s/g,'\n')}
 
   // now our goal is to run this on a remote server. Step 1 - xcopy the repo
   const folder = new Date().getTime();
-  const remote = 'root@147.75.76.177';
+  const remote = 'root@147.75.199.15';
 
   const runRemote = async function(command) {
     const bashCommand = `
@@ -312,24 +324,18 @@ EOSSH
   await runLocalWithoutErrors('cp -r dist netlify');
 
   if (process.env.BRANCH === 'master') {
+    console.info(await runLocal('git remote -v'));
     await runLocalWithoutErrors(`
       git config --global user.email "info@cncf.io"
       git config --global user.name "CNCF-bot"
       git remote rm github 2>/dev/null || true
-      git remote add github "https://$GITHUB_USER:$GITHUB_TOKEN@github.com/cncf/landscapeapp"
+      git remote add github "git@github.com:cncf/landscapeapp.git"
-      git fetch github
+      echo 1
-      # git diff # Need to comment this when a diff is too large
+      GIT_SSH_COMMAND='ssh -i ~/.ssh/bot3 -o IdentitiesOnly=yes' git fetch github
-      git checkout -- .
+      echo 2
-      npm version patch || npm version patch || npm version patch
+      git --no-pager show HEAD
-      git commit -m 'Update to a new version [skip ci]' --allow-empty --amend
+      echo 3
-      git branch -D tmp || true
+      GIT_SSH_COMMAND='ssh -i ~/.ssh/bot3 -o IdentitiesOnly=yes' git push github github/master:deploy
-      git checkout -b tmp
-      git push github HEAD:master || true
-      git push github HEAD:master --tags --force
-      echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/.npmrc
-      git diff
-      npm -q publish || (sleep 5 && npm -q publish) || (sleep 30 && npm -q publish)
-      echo 'Npm package published'
     `);
     // just for debug purpose
     //now we have a different hash, because we updated a version, but for build purposes we have exactly same npm modules

netlify/server.js

@@ -0,0 +1,176 @@
+// this "server.js" script should be able to run everything itself, without
+// having to bother with any packages or similar problems.
+const fs = require('fs/promises');
+const path = require('path');
+
+const oldCreateConnection = require('https').globalAgent.createConnection;
+require('https').globalAgent.createConnection = function(options, cb) {
+    options.highWaterMark = 1024 * 1024;
+    options.readableHighWaterMark = 1024 * 1024;
+    return oldCreateConnection.apply(this, [options, cb]);
+}
+
+
+// we will get a content of all files, in a form of entries
+// "file", "content", "md5"
+async function getContent() {
+    const dirs = ["images", "hosted_logos", "cached_logos"];
+    const files = ["landscape.yml", "settings.yml", "processed_landscape.yml", "guide.md"];
+    const all = [];
+    for (let dir of dirs) {
+        const filesInDir = await fs.readdir(dir);
+        for (let file of filesInDir) {
+            if (file !== '.' && file !== '..') {
+                const content = await fs.readFile(`${dir}/${file}`, { encoding: 'base64'});
+                const md5 = require('crypto').createHash('md5').update(content).digest("hex");
+                all.push({
+                    file: `${dir}/${file}`,
+                    content: content,
+                    md5: md5
+                });
+            }
+        }
+    }
+    for (let file of files) {
+        let content;
+        try {
+            content = await fs.readFile(file, { encoding: 'base64'});
+        } catch(ex) {
+
+        }
+        if (content) {
+            const md5 = require('crypto').createHash('md5').update(content).digest("hex");
+            all.push({
+                file: file,
+                content: content,
+                md5: md5
+            });
+        }
+    }
+    return all;
+}
+
+function get(path) {
+    return new Promise(function(resolve) {
+        const base = process.env.DEBUG_SERVER ? 'http://localhost:3000' : 'https://weblandscapes.ddns.net';
+        const http = require(base.indexOf('http://') === 0 ? 'http' : 'https');
+        const originalPath = path;
+        path = `${base}/api/console/download/${path}`;
+        const req = http.request(path, function(res) {
+            const path1 = originalPath.replace('?', '.html?');
+            if (res.statusCode === 404 && path.indexOf('.html') === -1) {
+              get(path1).then( (x) => resolve(x));
+            } else {
+              resolve({
+                res: res,
+                headers: res.headers,
+                statusCode: res.statusCode
+              });
+            }
+        });
+        req.end();
+    });
+}
+
+function post({path, request}) {
+    return new Promise(function(resolve) {
+        const base = process.env.DEBUG_SERVER ? 'http://localhost:3000' : 'https://weblandscapes.ddns.net';
+        const http = require(base.indexOf('http://') === 0 ? 'http' : 'https');
+
+        let data = '';
+        const req = http.request({
+            hostname: base.split('://')[1].replace(':3000', ''),
+            port: base.indexOf('3000') !== -1 ? '3000' : 443,
+            path: path,
+            method: 'POST',
+            headers: {
+                'Accept': 'application/json',
+                'Content-Type': 'application/json; charset=UTF-8'
+            }
+        }, function(res) {
+            res.on('data', function(chunk) {
+                data += chunk;
+            });
+            res.on('close', function() {
+                resolve(JSON.parse(data));
+            });
+        });
+        req.write(JSON.stringify(request));
+        req.end();
+    });
+}
+
+// a build is done on a server side, instead of a client side, this
+// is a key moment
+// 1. get content (that is fast)
+// 2. send a list of hashes
+// 3. get back a list of existing hashes
+// 4. send a list of (file, content, hash) , but skip the content when a hash is on the server
+// 5. get a build result on a server
+let previousGlobalHash;
+let lastOutput;
+let currentPath;
+async function build() {
+    const files = await getContent();
+    if (!previousGlobalHash) {
+        console.info(`Starting a new build...`);
+    }
+    if (JSON.stringify(files.map( (x) => x.md5)) === previousGlobalHash) {
+        return;
+    }
+    if (previousGlobalHash) {
+        console.info(`Changes detected, starting a new build`);
+    }
+    previousGlobalHash = JSON.stringify(files.map( (x) => x.md5));
+    const availableIds = await post({path: '/api/console/ids', request: { ids: files.map( (x) => x.md5 ) }});
+    const availableSet = new Set(availableIds.existingIds);
+    const filesWithoutExtraContent = files.map( (file) => ({
+        file: file.file,
+        md5: file.md5,
+        content: availableSet.has(file.md5) ? '' : file.content
+    }));
+    const result = await post({path: '/api/console/preview', request: { files: filesWithoutExtraContent }});
+    if (result.success) {
+        currentPath = result.path;
+        console.info(`${new Date().toISOString()} build result: ${result.success ? 'success' : 'failure'} `);
+    } else {
+        lastOutput = result.output;
+        console.info(`${new Date().toISOString()} build result: ${result.success ? 'success' : 'failure'} `);
+        console.info(result.output);
+    }
+}
+
+function server() {
+    const http = require('http');
+    http.createServer(async function (request, response) {
+        if (!currentPath) {
+            response.writeHead(404);
+            if (lastOutput) {
+                response.end(lastOutput);
+            } else {
+                response.end('Site is not ready');
+            }
+        } else {
+            let filePath = request.url.split('?')[0];
+            const url = path.join(currentPath, 'dist', filePath + '?' + request.url.split('?')[1]);
+            console.info(`Fetching ${url}`);
+            const output = await get(url);
+            response.writeHead(output.statusCode, output.headers);
+            output.res.pipe(response);
+        }
+
+    }).listen(process.env.PORT || 8001);
+    console.log(`Development server running at http://127.0.0.1:${process.env.PORT || 8001}/`);
+}
+
+async function main() {
+    server();
+    //eslint-disable-next-line no-constant-condition
+    while(true) {
+        await build();
+    }
+}
+main().catch(function(ex) {
+    console.info(ex);
+});
+// how will a server work?

package.json

@@ -1,6 +1,6 @@
 {
   "name": "interactive-landscape",
-  "version": "1.0.655",
+  "version": "1.0.657",
   "description": "Visualization tool for building interactive landscapes",
   "engines": {
     "npm": ">=3",
@@ -16,7 +16,8 @@
     "update-github-colors": "curl https://raw.githubusercontent.com/Diastro/github-colors/master/github-colors.json > tools/githubColors.json",
     "fetch": "node tools/validateLandscape && node tools/checkWrongCharactersInFilenames && node tools/addExternalInfo.js && yarn yaml2json",
     "fetchAll": "LEVEL=complete yarn fetch",
-    "update": "(rm /tmp/landscape.json || true) && node tools/validateLandscape && yarn remove-quotes && LEVEL=medium node tools/addExternalInfo.js && yarn prune && yarn check-links && yarn yaml2json && node tools/calculateNumberOfTweets && node tools/updateTimestamps",
+    "light-update": "(rm /tmp/landscape.json || true) && node tools/validateLandscape && yarn remove-quotes && LEVEL=crunchbase node tools/addExternalInfo.js && yarn prune && yarn yaml2json && node tools/updateTimestamps",
+    "update": "(rm /tmp/landscape.json || true) && node tools/validateLandscape && yarn remove-quotes && LEVEL=medium node tools/addExternalInfo.js && yarn prune && yarn check-links && yarn yaml2json && node tools/updateTimestamps",
     "yaml2json": "node tools/generateJson.js",
     "remove-quotes": "node tools/removeQuotes",
     "prune": "node tools/pruneExtraEntries",
@@ -92,5 +93,5 @@
     "type": "git",
     "url": "https://github.com/cncf/landscapeapp"
   },
-  "packageManager": "yarn@3.2.0"
+  "packageManager": "yarn@3.2.1"
 }

specs/main.spec.js

@@ -56,29 +56,29 @@ async function waitForHeaderText(page, text) {
   await page.waitForFunction(`[...document.querySelectorAll('.sh_wrapper')].find( (x) => x.innerText.includes('${text}'))`);
 }
 
-describe("Embed test", () => {
+// describe("Embed test", () => {
-  describe("I visit an example embed page", () => {
+  // describe("I visit an example embed page", () => {
-    let frame;
+    // let frame;
-    test('page is open and has a frame', async function(){
+    // test('page is open and has a frame', async function(){
-      page = await makePage(appUrl + '/embed');
+      // page = await makePage(appUrl + '/embed');
-      frame = await page.frames()[1];
+      // frame = await page.frames()[1];
-      await frame.waitForSelector('.cards-section .mosaic');
+      // await frame.waitForSelector('.cards-section .mosaic');
-      await waitForSelector(frame, '#embedded-footer');
+      // await waitForSelector(frame, '#embedded-footer');
-    });
+    // });
 
-    test('Do not see a content from a main mode', async function() {
+    // test('Do not see a content from a main mode', async function() {
-      const title = await frame.$('h1', { text: settings.test.header })
+      // const title = await frame.$('h1', { text: settings.test.header })
-      expect(await title.boundingBox()).toBe(null)
+      // expect(await title.boundingBox()).toBe(null)
-    });
+    // });
 
-    // ensure that it is clickable
+    // // ensure that it is clickable
-    test('I can click on a tile in a frame and I get a modal after that', async function() {
+    // test('I can click on a tile in a frame and I get a modal after that', async function() {
-      await waitForSelector(frame, ".cards-section .mosaic img");
+      // await waitForSelector(frame, ".cards-section .mosaic img");
-      await frame.click(`.mosaic img`);
+      // await frame.click(`.mosaic img`);
-    });
+    // });
-    close();
+    // close();
-  }, 6 * 60 * 1000); //give it up to 1 min to execute
+  // }, 6 * 60 * 1000); //give it up to 1 min to execute
-});
+// });
 
 describe("Main test", () => {
   describe("I visit a main page and have all required elements", () => {

src/api/export.js

@@ -21,7 +21,7 @@ const processRequest = module.exports.processRequest = query => {
   // extract alias - if params != card-mode (big_picture - always show)
   // i.e. make a copy to items here - to get a list of ids
 
-  const selectedItems = flattenItems(getGroupedItems({data: items, ...params}))
+  const selectedItems = flattenItems(getGroupedItems({data: items, skipDuplicates: params.format === 'card', ...params}))
     .reduce((acc, item) => ({ ...acc, [item.id]: true }), {})
 
   const fields = allItems[0].map(([label]) => label !== 'id' && label).filter(_ => _);
@@ -44,6 +44,6 @@ module.exports.handler = async function(event) {
   return { statusCode: 200, body: body, headers }
 }
 if (__filename === process.argv[1]) {
-  console.info(JSON.stringify(processRequest(process.argv[2]), null, 4));
+  console.info(processRequest(process.argv[2]), null, 4);
 }
 

src/api/ids.js

@@ -17,7 +17,7 @@ const processRequest = module.exports.processRequest = query => {
   }
 
   const summary = getSummary({data: items, ...params});
-  const groupedItems = getGroupedItems({data: items, ...params})
+  const groupedItems = getGroupedItems({data: items, skipDuplicates: params.format === 'card', ...params })
     .map(group => {
       const items = group.items.map(({ id }) => ({ id } ))
       return { ...group, items }

src/api/items.js

@@ -16,7 +16,7 @@ const processRequest = module.exports.processRequest = query => {
     items = expandSecondPathItems(items);
   }
 
-  const groupedItems = getGroupedItems({items: items, ...params})
+  const groupedItems = getGroupedItems({data: items, ...params, skipDuplicates: true})
     .map(group => {
       const items = group.items.map(({ id, name, href }) => ({ id, name, logo: `${settings.global.website}/${href}` }))
       return { ...group, items }
@@ -26,6 +26,10 @@ const processRequest = module.exports.processRequest = query => {
 // Netlify function
 module.exports.handler = async function(event) {
   const body = processRequest(event.queryStringParameters)
-  const headers = { 'Content-Type': 'application/json' }
+  const headers = {
+    'Content-Type': 'application/json',
+    'Access-Control-Allow-Origin': '*',
+    'Access-Control-Allow-Credentials': true
+  }
   return { statusCode: 200, body: JSON.stringify(body), headers }
 }

src/components/CardRenderer.js

@@ -46,7 +46,7 @@ module.exports.renderFlatCard = function renderFlatCard({item}) {
             <div data-id="${item.id}" class="mosaic-wrap">
               <div class="mosaic">
                 <img loading="lazy" src="${assetPath(item.href)}" class="logo" alt="${h(item.name)}" />
-                <div class="separator"/>
+                <div class="separator"></div>
                 <h5>${h(item.flatName)}</h5>
               </div>
             </div>

src/components/EmbedPageRenderer.js

@@ -1,3 +1,4 @@
+const _ = require('lodash');
 // Render only for an export
 const { saneName } = require('../utils/saneName');
 const { h } = require('../utils/format');
@@ -38,12 +39,13 @@ module.exports.render = function({items, exportUrl}) {
           <div class="cards-section">
             <div class="column-content" >
               ${ groupedItems.map( (groupedItem) => {
-                const cardElements = groupedItem.items.map( (item) => cardFn({item}));
+                const uniqItems = _.uniqBy(groupedItem.items, (x) => x.name + x.logo);
+                const cardElements = uniqItems.map( (item) => cardFn({item}));
                 const header = items.length > 0 ? `
                   <div class="sh_wrapper" data-wrapper-id="${h(saneName(groupedItem.header))}">
                     <div style="font-size: 24px; padding-left: 16px; line-height: 48px; font-weight: 500;">
                       <span>${h(groupedItem.header)}</span>
-                      <span class="items-cont">&nbsp;(${groupedItem.items.length})</span>
+                      <span class="items-cont">&nbsp;(${uniqItems.length})</span>
                     </div>
                   </div>` : '';
                 return [ header, `<div data-section-id="${h(saneName(groupedItem.header))}">${cardElements.join('')}</div>`].join('');

src/components/GuideLink.js

@@ -2,7 +2,7 @@ const { h } = require('../utils/format');
 const { guideLink } = require('../utils/icons');
 module.exports.renderGuideLink = function({anchor, label, style }) {
   const ariaLabel = `Read more about ${label} on the guide`
-  const to = `h(guide#${anchor})`;
+  const to = h(`guide#${anchor}`);
 
   return `<a data-type="external" target="_blank" style="${style}" aria-label="${h(ariaLabel)}" href="${to}">
     ${guideLink}

src/components/GuideRenderer.js

@@ -1,6 +1,6 @@
 const _ = require('lodash');
 
-const { isLargeFn } = require('../utils/landscapeCalculations');
+const { sizeFn } = require('../utils/landscapeCalculations');
 const { renderItem } = require('./Item.js');
 const { h } = require('../utils/format');
 const { assetPath } = require('../utils/assetPath');
@@ -10,9 +10,14 @@ const icons = require('../utils/icons');
 
 // guide is a guide index
 module.exports.render = function({settings, items, guide}) {
-  const title = `<h1 className="title">${h(settings.global.short_name)} Landscape Guide</h1>`;
+  const currentBranch = require('child_process').execSync(`git rev-parse --abbrev-ref HEAD`, {
+    cwd: require('../../tools/settings').projectPath
+  }).toString().trim();
+
+
+  const title = `<h1 className="title" style="margin-top: -5px;">${h(settings.global.short_name)} Landscape Guide</h1>`;
   const renderSubcategoryMetadata = ({ node, entries }) => {
-    const orderedEntries = _.orderBy(entries,  (x) => !x.isLarge);
+    const orderedEntries = _.orderBy(entries,  (x) => -x.size);
     const projectEntries = entries.filter(entry => entry.project)
     return `
       ${ (node.buzzwords.length > 0 || projectEntries.length > 0) ? `<div class="metadata">
@@ -53,7 +58,7 @@ module.exports.render = function({settings, items, guide}) {
       .map(node => {
         const hasChildren = (parents[node.anchor] || 0) > 1
         return `
-          <a href="#${node.anchor}" data-level="${node.level}" class="sidebar-link expandable" style="padding-left: ${10 + node.level * 10} px;">
+          <a href="#${node.anchor}" data-level="${node.level}" class="sidebar-link expandable" style="padding-left: ${10 + node.level * 10}px;">
             ${h(node.title)} ${hasChildren ? icons.expand : ''}
           </a>
           ${hasChildren ? `
@@ -105,14 +110,27 @@ module.exports.render = function({settings, items, guide}) {
       return null;
     }
     const enhanced = { ...entry, categoryAttrs }
-    return { ...enhanced, isLarge: isLargeFn(enhanced) }
+    return { ...enhanced, size: sizeFn(enhanced) }
   }).filter( (x) => !!x);
 
   return `
+      <div class="links">
+        <div>
+          <a href="${(settings.global.self_hosted_repo || false) ? "" : "https://github.com/"}${settings.global.repo}/edit/${currentBranch}/guide.md" target="_blank">
+          ${icons.edit}
+          Edit this page</a>
+        </div>
+        <div style="height: 5px;"></div>
+          <div>
+          <a href="${(settings.global.self_hosted_repo || false) ? "" : "https://github.com/"}${settings.global.repo}/issues/new?title=Guide Issue" target="_blank">
+            ${icons.github}
+          Report issue</a>
+        </div>
+      </div>
       <div class="side-content">
         <span class="landscape-logo">
-          <a class="nav-link" href="/">
+          <a aria-label="reset filters" class="nav-link" href="/">
-            <img src="${assetPath("images/left-logo.svg")} ">
+            <img alt="landscape logo" src="${assetPath("images/left-logo.svg")} ">
           </a>
         </span>
         <div class="guide-sidebar">
@@ -128,10 +146,10 @@ module.exports.render = function({settings, items, guide}) {
       <div class="guide-header">
         <div class="container">
           <div class="content">
-            <button class="sidebar-show">${icons.sidebar}</button>
+            <button class="sidebar-show" role="none" aria-label="show sidebar">${icons.sidebar}</button>
             <span class="landscape-logo">
-              <a class="nav-link" href="/">
+              <a aria-label="reset filters" class="nav-link" href="/">
-                <img src="${assetPath("/images/left-logo.svg")}">
+                <img alt="landscape logo" src="${assetPath("/images/left-logo.svg")}">
               </a>
             </span>
             ${title}
@@ -149,7 +167,6 @@ module.exports.render = function({settings, items, guide}) {
       <div class="main-content">
         <div class="container">
           <div class="content">
-            ${title}
             ${renderContent({nodes: guide,enhancedEntries: enhancedEntries})}
           </div>
         </div>

src/components/HomePageRenderer.js

@@ -107,8 +107,8 @@ module.exports.render = function({settings, guidePayload, hasGuide, bigPictureKe
     <div id="guide-page" style="display: ${guidePayload ? "" : "none"};" data-loaded="${guidePayload ? "true" : ""}">
       ${ !guidePayload ? `<div class="side-content">
                             <span class="landscape-logo">
-                              <a class="nav-link" href="/">
+                              <a aria-label="reset filters" class="nav-link" href="/">
-                                <img src="${assetPath("images/left-logo.svg")}" />
+                                <img alt="landscape logo" src="${assetPath("images/left-logo.svg")}" />
                               </a>
                             </span>
                             <div class="guide-sidebar">
@@ -125,12 +125,12 @@ module.exports.render = function({settings, guidePayload, hasGuide, bigPictureKe
     <div id="home" style="display: ${guidePayload ? "none" : ""}" class="app">
       <div class="app-overlay"></div>
       <div class="main-parent">
-        <button class="sidebar-show">${icons.sidebar}</button>
+        <button class="sidebar-show" role="none" aria-label="show sidebar">${icons.sidebar}</button>
         <div class="header_container">
           <div  class="header">
             <span class="landscape-logo">
-              <a class="nav-link" href="/">
+              <a aria-label="reset filters" class="nav-link" href="/">
-                <img src="${assetPath("images/left-logo.svg")}" />
+                <img alt="landscape logo" src="${assetPath("images/left-logo.svg")}" />
               </a>
             </span>
             <a rel="noopener noreferrer noopener noreferrer"
@@ -163,11 +163,6 @@ module.exports.render = function({settings, guidePayload, hasGuide, bigPictureKe
             ${renderFilterCompanyType()}
             ${renderFilterIndustries()}
 
-            <a class="filters-action export">
-              ${icons.export}
-              <span>Download as CSV</span>
-            </a>
-
             <div class="sidebar-presets">
               <h4>Example filters</h4>
               ${ (settings.presets || []).map(preset => `
@@ -175,12 +170,12 @@ module.exports.render = function({settings, guidePayload, hasGuide, bigPictureKe
                   ${h(preset.label)}
                 </a> `
               ).join('')}
+            </div>
             ${ (settings.ads || []).map( (entry) => `
                 <a data-type="external" target="_blank" class="sidebar-event" href="${entry.url}" title="${h(entry.title)}">
                   <img src="${assetPath(entry.image)}" alt="${entry.title}" />
                 </a>
             `).join('') }
-            </div>
 
           </div>
         </div>
@@ -189,7 +184,7 @@ module.exports.render = function({settings, guidePayload, hasGuide, bigPictureKe
         <div class="main">
           <div class="disclaimer">
             <span> ${settings.home.header} </span>
-            Please <a data-type="external" target="_blank" class="https://github.com/${settings.global.repo}">open</a> a pull request to
+            Please <a data-type="external" target="_blank" href="${(settings.global.self_hosted_repo || false) ? "" : "https://github.com/"}${settings.global.repo}">open</a> a pull request to
             correct any issues. Greyed logos are not open source. Last Updated: ${process.env.lastUpdated}
           </div>
           <h4 class="summary"></h4>
@@ -226,7 +221,7 @@ module.exports.render = function({settings, guidePayload, hasGuide, bigPictureKe
             width: 100%;
             text-align: center;">
               ${h(settings.home.footer)} For more information, please see the&nbsp;
-            <a data-type="external" target="_blank" eventLabel="crunchbase-terms" href="https://github.com/${settings.global.repo}/blob/HEAD/README.md#license">
+            <a data-type="external" target="_blank" eventLabel="crunchbase-terms" href="${(settings.global.self_hosted_repo || false) ? "" : "https://github.com/"}${settings.global.repo}/blob/HEAD/README.md#license">
                 license
               </a> info.
           </div>

src/components/Item.js

@@ -6,14 +6,33 @@ const { readJsonFromDist } = require('../utils/readJson');
 const settings = readJsonFromDist('settings');
 
 const largeItem = function(item) {
+  const isMember = item.category === settings.global.membership;
   const relationInfo = fields.relation.valuesMap[item.relation]
+  if (!relationInfo) {
+    console.error(`no map for ${item.relation} on ${item.name}`);
+  }
   const color = relationInfo.big_picture_color;
   const label = relationInfo.big_picture_label;
   const textHeight = label ? 10 : 0
   const padding = 2
 
+  const isMultiline = h(label).length > 20;
+  const formattedLabel = isMultiline ? h(label).replace(' - ', '<br>') : h(label);
+
+  if (isMember) {
     return `
-    <div data-id="${item.id}" class="large-item item" style="background: ${color}">
+    <div data-id="${item.id}" class="large-item large-item-${item.size} item">
+      <img loading="lazy" src="${assetPath(item.href)}" alt="${item.name}" style="
+            width: calc(100% - ${2 * padding}px);
+            height: calc(100% - ${2 * padding + textHeight}px);
+            padding: 5px;
+            margin: ${padding}px ${padding}px 0 ${padding}px;
+      "/>
+    </div>`;
+  }
+
+  return `
+    <div data-id="${item.id}" class="large-item large-item-${item.size} item" style="background: ${color}">
       <img loading="lazy" src="${assetPath(item.href)}" alt="${item.name}" style="
             width: calc(100% - ${2 * padding}px);
             height: calc(100% - ${2 * padding + textHeight}px);
@@ -24,14 +43,14 @@ const largeItem = function(item) {
             position: absolute;
             bottom: 0;
             width: 100%;
-            height: ${textHeight + padding}px;
+            height: ${textHeight + padding + (isMultiline ? 6 : 0) }px;
             text-align: center;
             vertical-align: middle;
             background: ${color};
             color: white;
             font-size: 6.7px;
-            line-height: 13px;
+            line-height: ${isMultiline ? 9 : 13 }px;
-      ">${h(label)}</div>
+      ">${ formattedLabel }</div>
     </div>`;
 }
 
@@ -48,12 +67,12 @@ const smallItem = function(item) {
 }
 
 module.exports.renderItem =  function (item) {
-  const {isLarge, category, oss, categoryAttrs } = item;
+  const {size, category, oss, categoryAttrs } = item;
   const isMember = category === settings.global.membership;
-  const ossClass = isMember || oss || categoryAttrs.isLarge ? 'oss' : 'nonoss';
+  const ossClass = isMember || oss || (categoryAttrs.isLarge && !settings.global.flags?.gray_large_items) ? 'oss' : 'nonoss';
-  const isLargeClass = isLarge ? 'wrapper-large' : '';
+  const isLargeClass = size > 1 ? `wrapper-large-${size}` : '';
 
   return `<div class="${isLargeClass + ' item-wrapper ' + ossClass}">
-    ${isLarge ? largeItem({isMember, ...item}) : smallItem({...item})}
+    ${size > 1 ? largeItem({isMember, ...item}) : smallItem({...item})}
   </div>`;
 }

src/components/ItemDialogContentRenderer.js

@@ -20,6 +20,16 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     return relativeDate(new Date(x));
   };
 
+  function getLinkedIn(itemInfo) {
+    if (itemInfo.extra && itemInfo.extra.override_linked_in) {
+      return itemInfo.extra.override_linked_in;
+    }
+    if (itemInfo.crunchbaseData && itemInfo.crunchbaseData.linkedin) {
+      return itemInfo.crunchbaseData.linkedin;
+    }
+    return '';
+  }
+
   function getRelationStyle(relation) {
     const relationInfo = fields.relation.valuesMap[relation]
     if (relationInfo && relationInfo.color) {
@@ -36,6 +46,10 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
 
   const tweetButton = (function() {
     // locate zoom buttons
+
+    if (!process.env.TWITTER_KEYS) {
+      return ``
+    }
     const twitterUrl = `https://twitter.com/intent/tweet`
 
     return `<div class="tweet-button">
@@ -48,8 +62,8 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
   })();
 
 
-  const renderLinkTag = (label, { name, url = null, color = 'blue', multiline = false }) => {
+  const renderLinkTag = (label, { name, url = null, color = 'blue', multiline = false, twoLines = false }) => {
-    return `<a data-type="internal" href="${url || '/'}" class="tag tag-${color} ${multiline ? 'multiline' : ''}">
+    return `<a data-type="internal" href="${url || '/'}" class="tag tag-${color} ${multiline ? 'multiline' : ''} ${twoLines ? 'twolines' : ''}">
       ${(name ? `<span class="tag-name">${h(name)}</span>` : '')}
       <span class="tag-value">${h(label)}</span>
       </a>`
@@ -78,7 +92,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
 
     if (prefix && tag) {
       const url = closeUrl({ filters: { relation: project }})
-      return renderLinkTag(tag, {name: prefix, url })
+      return renderLinkTag(tag, {name: prefix, url, twoLines: tag.indexOf(' - ') !== -1 || tag.length > 20 || prefix.length > 20 })
     }
 
     if (isSubsidiaryProject) {
@@ -114,9 +128,13 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     }
   };
 
-  const renderLicenseTag = function({relation, license, hideLicense}) {
+  const renderLicenseTag = function({relation, license, hideLicense, extra}) {
     const { label } = _.find(fields.license.values, {id: license});
 
+    if (extra && extra.hide_license) {
+      return '';
+    }
+
     if (relation === 'company' || hideLicense) {
       return '';
     }
@@ -130,19 +148,22 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
       return '';
     }
     if (!itemInfo.bestPracticeBadgeId) {
+      if (settings.global.hide_no_best_practices) {
+        return '';
+      }
       if (itemInfo.oss) {
         const emptyUrl="https://bestpractices.coreinfrastructure.org/";
         return `<a data-type="external" target="_blank" href=${emptyUrl} class="tag tag-grass">
-          <span class="tag-value">No CII Best Practices </span>
+          <span class="tag-value">No OpenSSF Best Practices </span>
           </a>`;
       } else {
         return '';
       }
     }
     const url = `https://bestpractices.coreinfrastructure.org/en/projects/${itemInfo.bestPracticeBadgeId}`;
-    const label = itemInfo.bestPracticePercentage === 100 ? 'passing' : (itemInfo.bestPracticePercentage + '%');
+    const label = itemInfo.bestPracticePercentage === 100 ? '✓' : (itemInfo.bestPracticePercentage + '%');
     return (`<a data-type="external" target="_blank" href="${url}" class="tag tag-grass">
-      <span class="tag-name">CII Best Practices</span>
+      <span class="tag-name">OpenSSF Best Practices</span>
       <span class="tag-value">${label}</span>
       </a>`);
   }
@@ -393,9 +414,12 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
 
   const linkToOrganization = closeUrl({ grouping: 'organization', filters: {organization: itemInfo.organization}});
 
-  const renderItemCategory = function(path) {
+  const renderItemCategory = function({path, itemInfo}) {
     var separator = `<span class="product-category-separator" key="product-category-separator">•</span>`;
     var subcategory = _.find(fields.landscape.values,{id: path});
+    if (!subcategory) {
+      throw new Error(`Failed to render ${itemInfo.name}, can not find a subcategory: ${path}, available paths are below: \n${fields.landscape.values.map( (x) => x.id).join('\n')}`);
+    }
     var category = _.find(fields.landscape.values, {id: subcategory.parentId});
     var categoryMarkup = `
       <a data-type="internal" href="${closeUrl({ grouping: 'landscape', filters: {landscape: category.id}})}">${h(category.label)}</a>
@@ -410,7 +434,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     <div class="product-property row">
       <div class="product-property-name col col-40">Twitter</div>
       <div class="product-property-value col col-60">
-        <a data-type=external target=_blank href="${itemInfo.twitter}">${h(formatTwitter(itemInfo.twitter))}</a>
+        <a data-type="external" target="_blank" href="${itemInfo.twitter}">${h(formatTwitter(itemInfo.twitter))}</a>
       </div>
    </div>
   ` : '';
@@ -420,7 +444,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
       <div class="product-property-name col col-50">Latest Tweet</div>
       <div class="product-property-value col col-50">
         ${ itemInfo.latestTweetDate ? `
-          <a data-type=external target=_blank href="${h(itemInfo.twitter)}">${formatDate(itemInfo.latestTweetDate)}</a>
+          <a data-type="external" target="_blank" href="${h(itemInfo.twitter)}">${formatDate(itemInfo.latestTweetDate)}</a>
         ` : ''}
         </div>
       </div>
@@ -430,7 +454,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     <div class="product-property row">
       <div class="product-property-name col col-40">First Commit</div>
       <div class="product-property-value tight-col col-60">
-        <a data-type=external target=_blank href=${h(itemInfo.firstCommitLink)}">${formatDate(itemInfo.firstCommitDate)}</a>
+        <a data-type="external" target=_blank href="${h(itemInfo.firstCommitLink)}">${formatDate(itemInfo.firstCommitDate)}</a>
       </div>
     </div>
   ` : '';
@@ -439,7 +463,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     <div class="product-property row">
       <div class="product-property-name col col-40">Contributors</div>
       <div class="product-property-value tight-col col-60">
-        <a data-type=external target=_blank href="${itemInfo.contributorsLink}">
+        <a data-type="external" target=_blank href="${itemInfo.contributorsLink}">
           ${itemInfo.contributorsCount > 500 ? '500+' : itemInfo.contributorsCount }
         </a>
       </div>
@@ -450,7 +474,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     <div class="product-property row">
       <div class="product-property-name col col-40">Headquarters</div>
       <div class="product-property-value tight-col col-60">
-        <a data-type=external target=_blank href="${closeUrl({ grouping: 'headquarters', filters:{headquarters:itemInfo.headquarters}})}">${h(itemInfo.headquarters)}</a>
+        <a data-type="external" target=_blank href="${closeUrl({ grouping: 'headquarters', filters:{headquarters:itemInfo.headquarters}})}">${h(itemInfo.headquarters)}</a>
       </div>
     </div>
   ` : '';
@@ -460,14 +484,14 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
       <div class="product-property-name col col-40">${itemInfo.amountKind === 'funding' ? 'Funding' : 'Market Cap'}</div>
       ${  itemInfo.amountKind === 'funding' ? `
           <div class="product-property-value tight-col col-60">
-            <a data-type=external target=_blank href="${itemInfo.crunchbase + '#section-funding-rounds'}">
+            <a data-type="external" target=_blank href="${itemInfo.crunchbase + '#section-funding-rounds'}">
               ${'$' + millify(itemInfo.amount)}
             </a>
           </div>` : ''
       }
       ${ itemInfo.amountKind !== 'funding' ? `
               <div class="product-property-value tight-col col-60">
-                <a data-type=external target=_blank href="https://finance.yahoo.com/quote/${itemInfo.yahoo_finance_data.effective_ticker}">
+                <a data-type="external" target=_blank href="https://finance.yahoo.com/quote/${itemInfo.yahoo_finance_data.effective_ticker}">
                   ${'$' + millify(itemInfo.amount)}
                 </a>
               </div>` : ''
@@ -479,7 +503,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     <div class="product-property row">
       <div class="product-property-name col col-40">Ticker</div>
       <div class="product-property-value tight-col col-60">
-        <a data-type=external target=_blank href="https://finance.yahoo.com/quote/${itemInfo.yahoo_finance_data.effective_ticker}">
+        <a data-type="external" target=_blank href="https://finance.yahoo.com/quote/${itemInfo.yahoo_finance_data.effective_ticker}">
           ${h(itemInfo.yahoo_finance_data.effective_ticker)}
         </a>
       </div>
@@ -490,7 +514,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     <div class="product-property row">
       <div class="product-property-name col col-50">Latest Commit</div>
       <div class="product-property-value col col-50">
-        <a data-type=external target=_blank href="${itemInfo.latestCommitLink}">${formatDate(itemInfo.latestCommitDate)}</a>
+        <a data-type="external" target=_blank href="${itemInfo.latestCommitLink}">${formatDate(itemInfo.latestCommitDate)}</a>
       </div>
     </div>
   ` : '';
@@ -499,7 +523,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     <div class="product-property row">
       <div class="product-property-name col col-50">Latest Release</div>
       <div class="product-property-value col col-50">
-        <a data-type=external target=_blank href="${itemInfo.releaseLink}">${formatDate(itemInfo.releaseDate)}</a>
+        <a data-type="external" target=_blank href="${itemInfo.releaseLink}">${formatDate(itemInfo.releaseDate)}</a>
       </div>
     </div>
   ` : '';
@@ -511,11 +535,121 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
     </div>
   ` : '';
 
+  const specialDates = ( function() {
+    let specialKeys = ['accepted', 'incubation', 'graduated', 'archived'];
+    const names = {
+      accepted: 'Accepted',
+      incubation: 'Incubation',
+      graduated: 'Graduated',
+      archived: 'Archived'
+    }
+    let result = {};
+    for (let key of specialKeys) {
+      if (itemInfo.extra && itemInfo.extra[key]) {
+        result[key] = itemInfo.extra[key];
+        delete itemInfo.extra[key];
+      }
+    }
+
+    const keys = Object.keys(result);
+    const values = Object.values(result);
+    if (keys.length === 0) {
+      return '';
+    }
+    if (keys.length === 1) {
+      return `
+        <div class="product-property row">
+          <div class="product-property-name col col-20">${names[keys[0]]}</div>
+          <div class="product-property-name col col-80">${values[0]}</div>
+        </div>
+      `
+    }
+    if (keys.length === 2) {
+      return `
+        <div class="row">
+          <div class="col col-50">
+            <div class="product-property row">
+              <div class="product-property-name col col-40">${names[keys[0]]}</div>
+              <div class="product-property-name col col-60">${values[0]}</div>
+            </div>
+          </div>
+          <div class="col col-50">
+            <div class="product-property row">
+              <div class="product-property-name col col-50">${names[keys[1]]}</div>
+              <div class="product-property-name col col-50">${values[1]}</div>
+            </div>
+          </div>
+        </div>
+      `
+    }
+    if (keys.length === 3) {
+      return `
+        <div class="row">
+          <div class="col col-50">
+            <div class="product-property row">
+              <div class="product-property-name col col-40">${names[keys[0]]}</div>
+              <div class="product-property-name col col-60">${values[0]}</div>
+            </div>
+          </div>
+          <div class="col col-50">
+            <div class="product-property row">
+              <div class="product-property-name col col-50">${names[keys[1]]}</div>
+              <div class="product-property-name col col-50">${values[1]}</div>
+            </div>
+          </div>
+        </div>
+        <div class="product-property row">
+          <div class="product-property-name col col-20">${names[keys[2]]}</div>
+          <div class="product-property-name col col-80">${values[2]}</div>
+        </div>
+      `;
+    }
+    return '';
+  })();
+
+  const cloElement = ( function() {
+    if (!itemInfo.extra) {
+      return '';
+    }
+    if (!itemInfo.extra.clomonitor_svg) {
+      return '';
+    }
+    return `
+      <a href="https://clomonitor.io/projects/cncf/${itemInfo.extra.clomonitor_name}" target="_blank">
+        ${itemInfo.extra.clomonitor_svg}
+      </a>
+    `;
+  })();
+
   const extraElement = ( function() {
     if (!itemInfo.extra) {
       return '';
     }
     const items = Object.keys(itemInfo.extra).map( function(key) {
+      if (key.indexOf('summary_') === 0) {
+        return '';
+      }
+      if (key === 'clomonitor_name' || key === 'clomonitor_svg') {
+        return '';
+      }
+      if (key === 'hide_license') {
+        return '';
+      }
+      if (key === 'override_linked_in') {
+        return '';
+      }
+      if (key === 'audits') {
+        const value = itemInfo.extra[key];
+        const lines = (value.map ? value : [value]).map( (auditInfo) => `
+          <div>
+          <a href="${h(auditInfo.url)}" target="_blank">${h(auditInfo.type)} at ${auditInfo.date}</a>
+          </div>
+        `).join('');
+        return `<div class="product-property row">
+          <div class="product-property-name tight-col col-20">Audits</div>
+          <div class="product-proerty-value tight-col col-80">${lines}</div>
+        </div>`;
+      }
       const value = itemInfo.extra[key];
       const keyText = (function() {
         const step1 =  key.replace(/_url/g, '');
@@ -527,7 +661,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
           return h(relativeDate(new Date(value)));
         }
         if (typeof value === 'string' && (value.indexOf('http://') === 0 || value.indexOf('https://') === 0)) {
-          return `<a data-type=external target=_blank href="${h(value)}">${h(value)}</a>`;
+          return `<a data-type="external" target=_blank href="${h(value)}">${h(value)}</a>`;
         }
         return h(value);
       })();
@@ -550,7 +684,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
 
     const productLogoAndTagsAndCharts = `
       <div class="product-logo" style="${getRelationStyle(itemInfo.relation)}">
-        <img src="${assetPath(itemInfo.href)}" class="product-logo-img">
+        <img alt="product logo" src="${assetPath(itemInfo.href)}" class="product-logo-img">
       </div>
       <div class="product-tags">
         <div class="product-badges" style="width: 300px;" >
@@ -578,7 +712,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
         <div class="product-parent"><a data-type=internal href="${linkToOrganization}">
           <span>${h(itemInfo.organization)}</span>${renderMemberTag(itemInfo)}</a></div>
         ${productPaths.map( (productPath) => `
-          <div class="product-category">${renderItemCategory(productPath)}</div>
+          <div class="product-category">${renderItemCategory({path: productPath, itemInfo})}</div>
         `).join('')}
         <div class="product-description">${h(itemInfo.description)}</div>
     </div>
@@ -586,7 +720,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
       <div class="product-property row">
         <div class="product-property-name col col-20">Website</div>
         <div class="product-property-value col col-80">
-          <a href=external target=_blank href="${itemInfo.homepage_url}">${shortenUrl(itemInfo.homepage_url)}</a>
+          <a data-type=external target=_blank href="${itemInfo.homepage_url}">${shortenUrl(itemInfo.homepage_url)}</a>
         </div>
       </div>
       ${ (itemInfo.repos || []).map(({ url, stars }, idx) => {
@@ -631,12 +765,12 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
             </div>
           </div> ` : ''
       }
-      ${itemInfo.crunchbaseData && itemInfo.crunchbaseData.linkedin ? `
+      ${getLinkedIn(itemInfo) ? `
           <div class="product-property row">
             <div class="product-property-name col col-20">LinkedIn</div>
             <div class="product-property-value col col-80">
-              <a data-type=external target=_blank href="${itemInfo.crunchbaseData.linkedin}">
+              <a data-type=external target=_blank href="${getLinkedIn(itemInfo)}">
-                ${shortenUrl(itemInfo.crunchbaseData.linkedin)}
+                ${shortenUrl(getLinkedIn(itemInfo))}
               </a>
             </div>
           </div> ` : ''
@@ -646,18 +780,26 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
           ${ twitterElement }
           ${ firstCommitDateElement }
           ${ contributorsCountElement }
-          ${ headquartersElement }
-          ${ amountElement }
-          ${ tickerElement }
         </div>
         <div class="col col-50">
           ${ latestTweetDateElement }
           ${ latestCommitDateElement }
           ${ releaseDateElement }
+        </div>
+      </div>
+      ${specialDates}
+      <div class="row">
+        <div class="col col-50">
+          ${ headquartersElement }
+          ${ amountElement }
+          ${ tickerElement }
+        </div>
+        <div class="col col-50">
           ${ crunchbaseEmployeesElement }
         </div>
       </div>
       ${extraElement}
+      ${cloElement}
       </div>
   `;
 
@@ -670,7 +812,7 @@ module.exports.render = function({settings, tweetsCount, itemInfo}) {
         ${renderParticipation(itemInfo)}
       </div>
       ${ itemInfo.twitter ? `<div class="twitter-timeline">
-          <a class="twitter-timeline" data-tweet-limit="5" href="${itemInfo.twitter}"></a>
+          <a class="twitter-timeline" aria-hidden="true" data-tweet-limit="5" href="${itemInfo.twitter}"></a>
         </div>` : '' }
     </div>
   </div>`;

src/components/LandscapeContentRenderer.js

@@ -22,7 +22,11 @@ module.exports.render = function({landscapeSettings, landscapeItems}) {
     if (element.type === 'LandscapeInfo') {
       return renderLandscapeInfo(element)
     }
-    const category = landscapeItems.find(c => c.key === element.category) || {}
+    const category = landscapeItems.find(c => c.key === element.category);
+    if (!category) {
+      console.info(`Can not find the ${element.category}`);
+      console.info(`Valid values: ${landscapeItems.map( (x) => x.key).join('; ')}`);
+    }
     const attributes = extractKeys(element, ['width', 'height', 'top', 'left', 'color', 'fit_width', 'is_large'])
     const subcategories = category.subcategories.map(subcategory => {
       const allItems = subcategory.allItems.map(item => ({ ...item, categoryAttrs: attributes }))

src/components/ObsoleteListRenderer.js

@@ -0,0 +1,78 @@
+const _ = require('lodash');
+const { h } = require('../utils/format');
+const l = function(x) {
+  return h((x || "").replace("https://", ""));
+}
+const { formatNumber } = require('../utils/formatNumber');
+function highlightLinks(s) {
+  if (!s) {
+    return '';
+  }
+  // markdown styles
+  s = s.replace(/\[(.*?)\]\((https?:.*?)\)/g, '<a target="_blank" href="$2">$1</a>')
+  s = s.replace(/(\s|^)(https?:.*?)(\s|$)/g, ' <a target="_blank" href="$2">$2</a> ')
+  return s;
+}
+const getDate = function(date) {
+  if (!date) {
+    return '';
+  }
+  return new Date(date).toISOString().substring(0, 10);
+}
+const today = getDate(new Date());
+
+module.exports.render = function({items}) {
+  console.info(items[0], items[0].latestCommitDate);
+
+  const old = _.orderBy( items.filter( (x) => x.latestCommitDate && new Date(x.latestCommitDate).getTime() + 3 * 30 * 86400 * 1000 < new Date().getTime()), 'latestCommitDate');
+
+  console.info(old.map( (x) => x.name));
+
+  return `
+    <head>
+    <link rel="icon" type="image/png" href="/favicon.png" />
+    <style>
+    </style>
+    </head>
+    <body>
+    <h1>List of obsolete items</h1>
+    ${old.map(function(item) {
+      return `
+        <div style="display: flex; flex-direction: row;">
+          <div style="width: 200px; overflow: hidden; padding: 5px;">
+            <img src="logos/${item.image_data.fileName}"></img>
+          </div>
+          <div style="width: 300px; overflow: hidden; padding: 5px;">
+          <h3>${item.name}</h1>
+          <h3>${item.path}</h2>
+          <h3><span><b>Latest Commit: </b></span> ${getDate(item.latestCommitDate)}</h3>
+          <h3><span><b>Repo: </b></span> ${highlightLinks(item.repo_url)}</h3>
+          </div>
+          <div style="width: 600px; padding: 5px;">
+            <h4>Manual Actions</h4>
+            <pre>
+1. Create an issue in their repo, inform that that a repo is scheduled for deletion.
+2. Mark this item in a <b>landscape.yml</b> with an <b>extra</b> property <b>obsolete_since</b> equal to ${today}
+3. After a month remove the entry from the repo
+===
+Issue Template:
+Title: ${item.name} is going to be unreferenced from the interactive landscape because of no activity since ${getDate(item.latestCommitDate)}
+Body:
+<div style="font-size: 8px;">
+Dear project maintainers of ${item.name},
+
+I hope this message finds you well.
+I noticed that your project has had no activity for the last 3 months and is about to be removed from the interactive landscape https://landscape.cncf.io/?selected=${item.id}
+As a maintainer of an interactive landscape, we have included your project as a reference for our users and would like to ensure that the information we provide is up-to-date.
+We understand that maintaining a project can be challenging and time-consuming, and we would like to offer any assistance that we can. Please let us know if there are any plans to continue the development of the project or if there is anything we can do to help.
+Thank you for your time and efforts in creating and maintaining this project. We appreciate the value it has provided to the community and hope to continue to reference it in our interactive landscape.
+Best regards, CNCF Landscape Team
+</div>
+            </pre>
+          </div>
+        </div>
+      `;
+    }).join('<hr>')}
+    </body>
+  `
+}

src/components/OtherLandscapeLink.js

@@ -6,14 +6,15 @@ const { categoryBorder, categoryTitleHeight, subcategoryTitleHeight } = require(
 
 const renderCardLink = ({ url, children }) => {
   if (url.indexOf('http') === 0) {
-    return `<a data-type=external target=_blank href="${url} style="display: flex; flex-direction: column">${children}</a>`;
+    return `<a data-type=external target=_blank href="${url}" style="display: flex; flex-direction: column;">${children}</a>`;
   } else {
     url = stringifyParams({ mainContentMode: url });
-    return `<a data-type=tab href="${url} style="display: flex; flex-direction: column">${children}</a>`;
+    return `<a data-type=tab href="${url}" style="display: flex; flex-direction: column;">${children}</a>`;
   }
 };
 
 module.exports.renderOtherLandscapeLink = function({top, left, height, width, color, title, image, url, layout}) {
+  title = title || ''; //avoid undefined!
   const imageSrc = image || assetPath(`images/${url}_preview.png`);
   if (layout === 'category') {
     return `<div style="
@@ -30,13 +31,14 @@ module.exports.renderOtherLandscapeLink = function({top, left, height, width, co
       display: flex;
     ">
       ${renderCardLink({url: url, children: `
-        <div style="width: ${width}px;height: 30px; line-height: 28px; text-align: center; color: 'white'; font-size: 12px;">${h(title)}</div>
+        <div style="width: ${width}px;height: 30px; line-height: 28px; text-align: center; color: white; font-size: 12px;">${h(title)}</div>
-        <div style="flex: 1; background: white; position: relative; display: flex; justify-content: center; align-items: center">
+        <div style="flex: 1; background: white; position: relative; display: flex; justify-content: center; align-items: center;">
             <img loading="lazy" src="${imageSrc}" style="
-              width: ${width - 12}px; height: ${height - 42}px;
+              width: ${width - 12}px;
+              height: ${height - 42}px;
               object-fit: contain;
               background-position: center;
-              background-repeat: no-repeat;" alt=${title} />
+              background-repeat: no-repeat;" alt="${h(title)}" />
         </div>`})}
   </div>`;
   }
@@ -59,18 +61,18 @@ module.exports.renderOtherLandscapeLink = function({top, left, height, width, co
             right: 0;
             boxShadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2), 0 6px 20px 0 rgba(0, 0, 0, 0.2);
             padding: ${categoryBorder}px;
-            display: flex
+            display: flex;
           "
         >
           <div style="
             width: ${categoryTitleHeight}px;
-            writing-mode: 'vertical-rl';
+            writing-mode: vertical-rl;
             transform: rotate(180deg);
             text-align: center;
             display: flex;
             align-items: center;
             justify-content: center;
-            font-size: 12px,
+            font-size: 12px;
             line-height: 13px;
             color: white;
           ">
@@ -82,7 +84,7 @@ module.exports.renderOtherLandscapeLink = function({top, left, height, width, co
             background: white;
             justify-content: center;
             align-items: center; ">
-              <img loading="lazy" src="${imageSrc}" alt="${title}"
+              <img loading="lazy" src="${imageSrc}" alt="${h(title)}"
                   style="width: ${width - 42}px;
                          height: ${height - 32}px;
                          object-fit: contain;

src/components/SummaryRenderer.js

@@ -0,0 +1,624 @@
+const _ = require('lodash');
+const { h } = require('../utils/format');
+const l = function(x) {
+  return h((x || "").replace("https://", ""));
+}
+const { formatNumber } = require('../utils/formatNumber');
+
+const getLanguages = function(item) {
+  if (item.extra && item.extra.summary_languages) {
+    return item.extra.summary_languages;
+  }
+  if (item.github_data && item.github_data.languages) {
+    const total = _.sum(item.github_data.languages.map( (x) => x.value));
+    const matching = item.github_data.languages.filter( (x) => x.value > total * 0.3).map( (x) => x.name);
+    return matching.join(', ');
+  } else {
+    return '';
+  }
+}
+
+function highlightLinks(s) {
+  if (!s) {
+    return '';
+  }
+  // markdown styles
+  s = s.replace(/\[(.*?)\]\((https?:.*?)\)/g, '<a target="_blank" href="$2">$1</a>')
+  s = s.replace(/(\s|^)(https?:.*?)(\s|$)/g, ' <a target="_blank" href="$2">$2</a> ')
+  return s;
+}
+
+const getDate = function(date) {
+  if (!date) {
+    return '';
+  }
+  return new Date(date).toISOString().substring(0, 10);
+}
+
+module.exports.render = function({items}) {
+
+  const projects = items.filter( (x) => !!x.relation && x.relation !== 'member');
+  const categories = _.uniq(projects.map( (x) => x.path.split(' / ')[0]));
+  const categoriesCount = {};
+  const categoryItems = {};
+  const subcategories = {};
+  for (let k of categories) {
+    categoriesCount[k] = projects.filter( (x) => x.path.split(' / ')[0] === k).length;
+    categoryItems[k] = projects.filter( (x) => x.path.split(' / ')[0] === k).map( (x) => projects.indexOf(x));
+    const arr = _.uniq(projects.filter( (x) => x.path.split(' / ')[0] === k).map( (x) => x.path.split(' / ')[1]));
+    for (let subcategory of arr) {
+      categoryItems[k + ':' + subcategory] = projects.filter( (x) => x.path === k + ' / ' + subcategory).map( (x) => projects.indexOf(x));
+    }
+    subcategories[k] = arr;
+  }
+
+
+
+
+  const columnWidth = 250;
+
+  return `
+    <head>
+    <link rel="icon" type="image/png" href="/favicon.png" />
+    <style>
+      ${require('fs').readFileSync('src/fonts.css', 'utf-8')}
+      ::root {
+        --navy: #38404a;
+        --navy-light: #696D70;
+        --blue: #2E67BF;
+        --blue-hover: #1D456B;
+        --spacing: 1em;
+      }
+      body {
+        color: rgba(0, 0, 0, 0.87);
+        margin: 0;
+        font-size: 0.875rem;
+        font-family: "Roboto", "Helvetica", "Arial", sans-serif;
+        font-weight: 400;
+        line-height: 1.43;
+        letter-spacing: 0.01071em;
+        background-color: #fafafa;
+      }
+
+      td a {
+        text-decoration: none;
+        color: #2E67BF;
+      }
+
+      .category {
+        font-size: 24px;
+        font-weight: bold;
+      }
+
+      .categories {
+        display: inline-block;
+        margin: 5px;
+        font-size: 14px;
+      }
+
+      .subcategories {
+        display: inline-block;
+        margin: 5px;
+        font-size: 14px;
+      }
+
+      table {
+        width: ${columnWidth * projects.length}px;
+        table-layout: fixed;
+        border-collapse: separate;
+        border-spacing: 0;
+        border-top: 1px solid white;
+      }
+
+      #headers {
+        width: 152px;
+        position: sticky;
+        left: 0;
+      }
+
+      #data {
+        position: absolute;
+        left: 155px;
+        top: 0px;
+        z-index: -1;
+      }
+
+      td,
+      th {
+        white-space: pre-wrap;
+        width: ${columnWidth}px;
+        margin: 0;
+        border: 1px solid white;
+        border-top-width: 0px;
+        height: 53px;
+        padding: 5px;
+        overflow: hidden;
+        font-size: 0.8em;
+        color: var(--navy);
+      }
+
+      .project-name {
+        text-align: center;
+        font-size: 15px;
+        font-weight: bold;
+      }
+
+      html, body {
+        height: 100%;
+        overflow: hidden;
+        padding: 0;
+        margin: 0;
+        outline: 0;
+      }
+
+
+      .table-wrapper {
+        position: absolute;
+        top: 165px;
+        width: calc(100% - 16px);
+        bottom: 0;
+        overflow-x: scroll;
+        overflow-y: scroll;
+        padding: 0;
+        left: 16px;
+      }
+
+      .sticky {
+        position: relative;
+        background-color: #1b446c;
+        color: white;
+        width: 152px;
+        top: auto;
+        font-size: 0.8em;
+        font-weight: bold;
+        padding: 0px;
+      }
+      .first-line {
+        background-color: #1b446c;
+        color: white;
+      }
+      .alternate-line {
+        background-color: #e5e5e5;
+      }
+      .sticky span {
+        white-space: nowrap;
+        position: absolute;
+        left: 50%;
+        top: 50%;
+        transform: translate(-50%, -50%);
+      }
+
+      h1 {
+        font-size: 2.1rem;
+        line-height: 30px;
+        display: block;
+        margin: 0 0 14px;
+        color: var(--navy);
+      }
+      .landscape-logo {
+        width: 160px;
+        height: 48px;
+        display: inline-block;
+      }
+      .landscapeapp-logo {
+        position: absolute;
+        right: 5px;
+        top: 14px;
+      }
+      .landscapeapp-logo img {
+        width: 200px;
+      }
+
+
+      .main-header {
+        padding: 16px;
+      }
+
+      /* select starting stylings ------------------------------*/
+.select {
+  font-family: 'Roboto';
+	position: relative;
+	width: 240px;
+  margin-bottom: 10px;
+}
+.select-disabled {
+  opacity: 0.35;
+  pointer-events: none;
+}
+
+.select-text {
+	position: relative;
+	font-family: inherit;
+	background-color: transparent;
+	width: 100%;
+  font-size: 11px;
+	padding: 10px 22px 10px 0;
+	border-radius: 0;
+	border: none;
+	border-bottom: 1px solid rgba(0,0,0, 0.12);
+}
+
+/* Remove focus */
+.select-text:focus {
+	outline: none;
+	border-bottom: 1px solid rgba(0,0,0, 0);
+}
+
+	/* Use custom arrow */
+.select .select-text {
+	appearance: none;
+	-webkit-appearance:none
+}
+
+.select:after {
+	position: absolute;
+	top: 18px;
+	right: 10px;
+	/* Styling the down arrow */
+	width: 0;
+	height: 0;
+	padding: 0;
+	content: '';
+	border-left: 6px solid transparent;
+	border-right: 6px solid transparent;
+	border-top: 6px solid rgba(0, 0, 0, 0.12);
+	pointer-events: none;
+}
+
+
+/* LABEL ======================================= */
+.select-label {
+	color: rgb(105, 109, 112);
+	font-size: 10px;
+	font-weight: normal;
+	position: absolute;
+	pointer-events: none;
+	left: 0;
+	top: 10px;
+	transition: 0.2s ease all;
+}
+
+/* active state */
+.select-text:focus ~ .select-label, .select-text ~ .select-label {
+	top: -10px;
+	transition: 0.2s ease all;
+	font-size: 11px;
+}
+
+/* BOTTOM BARS ================================= */
+.select-bar {
+	position: relative;
+	display: block;
+	width: 100%;
+}
+
+.select-bar:before, .select-bar:after {
+	content: '';
+	height: 2px;
+	width: 0;
+	bottom: 1px;
+	position: absolute;
+	background: #2F80ED;
+	transition: 0.2s ease all;
+}
+
+.select-bar:before {
+	left: 50%;
+}
+
+.select-bar:after {
+	right: 50%;
+}
+
+/* active state */
+.select-text:focus ~ .select-bar:before, .select-text:focus ~ .select-bar:after {
+	width: 50%;
+}
+
+.select-highlight {
+	position: absolute;
+	height: 60%;
+	width: 100px;
+	top: 25%;
+	left: 0;
+	pointer-events: none;
+	opacity: 0.5;
+}
+
+.info {
+    position: absolute;
+    top: 0px;
+    left: 540px;
+    font-size: 12px;
+    line-height: 1.2;
+    max-width: 500px;
+}
+
+    </style>
+    </head>
+    <body>
+    <div class="main-header">
+    <span class="landscape-logo">
+      <a aria-label="reset filters" class="nav-link" href="/">
+        <img alt="landscape logo" src="/images/left-logo.svg">
+      </a>
+    </span>
+
+    <span style="display: inline-block; position: relative; top: -8px; left: 20px;">
+      <h1>CNCF Project Summary Table</h1>
+    </span>
+
+    <a rel="noopener noreferrer noopener noreferrer" class="landscapeapp-logo" title="CNCF" target="_blank" href="https://www.cncf.io">
+      <img src="/images/right-logo.svg" title="CNCF">
+    </a>
+    </div>
+
+    <div style="padding: 16px; position: relative; top: -19px;">
+    <div class="categories">
+      <div class="select">
+        <select class="select-text" required="">
+          <option value="" selected="">All: ${projects.length}</option>
+          ${categories.map( (name) => `<option value="${name}">${name}: ${categoriesCount[name]}</option>`).join('')}
+        </select>
+        <span class="select-highlight"></span>
+        <span class="select-bar"></span>
+        <label class="select-label">Category</label>
+      </div>
+    </div>
+
+    <div class="subcategories" style="display: none">
+      <div class="select">
+        <select class="select-text" required="">
+
+        </select>
+        <span class="select-highlight"></span>
+        <span class="select-bar"></span>
+        <label class="select-label">Subcategory</label>
+      </div>
+    </div>
+    <div class="info">
+    The <i>CNCF Project Summary Table</i> provides a standardized, summary of CNCF projects.<br/><div style="height: 5px;"></div>
+<b style="color: rgb(58,132,247);">The filters on the left side help refine your view.</b> Start by filtering by category (e.g., <i>orchestration and management</i>) and then subcategory (e.g., <i>service mesh</i> for an overview of all available CNCF service meshes).
+    </div>
+    </div>
+
+
+    <div class="table-wrapper">
+    <table id="headers">
+      <tr class="landscape first-line">
+        <td class="sticky">
+          <span> Project </span>
+        </td>
+      </tr>
+      <tr class="landscape">
+        <td class="sticky">
+           <span>Description</span>
+        </td>
+      </tr>
+      <tr class="landscape alternate-line">
+        <td class="sticky">
+           <span>Maturity</span>
+        </td>
+      </tr>
+      <tr class="landscape">
+        <td class="sticky">
+           <span>Target Users</span>
+        </td>
+      </tr>
+      <tr class="landscape alternate-line">
+        <td class="sticky">
+           <span>Tags</span>
+        </td>
+      </tr>
+      <tr class="landscape">
+        <td class="sticky">
+           <span>Use Case</span>
+        </td>
+      </tr>
+      <tr class="landscape alternate-line">
+        <td class="sticky">
+           <span>Business Use</span>
+        </td>
+      </tr>
+      <tr class="landscape">
+        <td class="sticky">
+           <span>Languages</span>
+        </td>
+      </tr>
+      <tr class="landscape alternate-line">
+        <td class="sticky">
+           <span>First Commit</span>
+        </td>
+      </tr>
+      <tr class="landscape">
+        <td class="sticky">
+           <span>Last Commit</span>
+        </td>
+      </tr>
+      <tr class="landscape alternate-line">
+        <td class="sticky">
+           <span>Release Cadence</span>
+        </td>
+      </tr>
+      <tr class="landscape">
+        <td class="sticky">
+           <span>Github Stars</span>
+        </td>
+      </tr>
+      <tr class="landscape alternate-line">
+        <td class="sticky">
+           <span>Integrations</span>
+        </td>
+      </tr>
+      <tr class="landscape">
+        <td class="sticky">
+           <span>Website</span>
+        </td>
+      </tr>
+      <tr class="landscape alternate-line">
+        <td class="sticky">
+           <span>Github</span>
+        </td>
+      </tr>
+      <tr class="landscape">
+        <td class="sticky">
+           <span>Overview Video</span>
+        </td>
+      </tr>
+    </table>
+    <table id="data">
+      <tr class="landscape first-line">
+        ${projects.map( (project, index) => `
+          <td class="project-name" data-project-index="${index}">${h(project.name)}</td>
+        `).join('')}
+      </tr>
+      <tr class="landscape">
+          ${projects.map( (project) => `
+            <td>${h((project.github_data || project)['description'])}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape alternate-line">
+          ${projects.map( (project) => `
+            <td>${h(project.relation)}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape">
+          ${projects.map( (project) => `
+            <td>${h((project.extra || {})['summary_personas']) || '&nbsp;'}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape alternate-line">
+          ${projects.map( (project) => `
+            <td>${h((project.extra || {})['summary_tags'] || '').split(',').map( (tag) => `<div>- ${tag.trim()}</div>`).join('') }</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape">
+          ${projects.map( (project) => `
+            <td>${h((project.extra || {})['summary_use_case']) || '&nbsp;'}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape alternate-line">
+          ${projects.map( (project) => `
+            <td>${h((project.extra || {})['summary_business_use_case']) || '&nbsp;'}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape">
+          ${projects.map( (project) => `
+            <td>${h(getLanguages(project))}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape alternate-line">
+          ${projects.map( (project) => `
+            <td>${h(getDate((project.github_start_commit_data || {}).start_date))}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape">
+          ${projects.map( (project) => `
+            <td>${h(getDate((project.github_data || {}).latest_commit_date))}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape alternate-line">
+          ${projects.map( (project) => `
+            <td>${h((project.extra || {})['summary_release_rate']) || '&nbsp;'}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape">
+          ${projects.map( (project) => `
+            <td>${h(formatNumber((project.github_data || {}).stars))}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape alternate-line">
+          ${projects.map( (project) => `
+            <td>${highlightLinks((project.extra || {})['summary_integrations']) || '&nbsp;'}</td>
+          `).join('')}
+      </tr>
+      <tr class="landscape">
+          ${projects.map( (project) => `
+            <td><a href="${h((project.homepage_url))}" target="_blank">${l(project.homepage_url)}</a></td>
+          `).join('')}
+      </tr>
+      <tr class="landscape alternate-line">
+          ${projects.map( (project) => project.repo_url ? `
+            <td><a href="${h(project.repo_url)}" target="_blank">${l(project.repo_url)}</a></td>
+          `: '<td>&nbsp;</td>').join('')}
+      </tr>
+      <tr class="landscape">
+          ${projects.map( (project) => project.extra && project.extra.summary_intro_url ? `
+            <td><a href="${h(project.extra.summary_intro_url)}" target="_blank">${l(project.extra.summary_intro_url)}</a></td>
+          `: '<td>&nbsp;</td>').join('')}
+      </tr>
+    </table>
+    <div style="height: 20px;"></div>
+    </div>
+    <script>
+      function setHeight() {
+        const rows = [...document.querySelectorAll('#data tr')];
+        const headersRows = [...document.querySelectorAll('#headers tr')];
+        for (let row of rows) {
+          const index = rows.indexOf(row);
+          const headerEl = headersRows[index].querySelector('td');
+          const firstEl = [...row.querySelectorAll('td')].filter((x) => x.style.display !== 'none')[0];
+          headerEl.style.height = (firstEl.getBoundingClientRect().height) + 'px';
+        }
+      }
+
+      window.App = {
+        totalCount: ${projects.length},
+        categories: ${JSON.stringify(categories)},
+        categoryItems: ${JSON.stringify(categoryItems)},
+        subcategories: ${JSON.stringify(subcategories)}
+      };
+      document.querySelector('.categories select').addEventListener('change', function(e) {
+        const selectedOption = Array.from(document.querySelectorAll('.categories option')).find( (x) => x.selected);
+        const categoryId = selectedOption.value;
+        if (!categoryId) {
+          document.querySelector('#data').style.width = '';
+          document.querySelector('.subcategories').style.display = 'none';
+        } else {
+          document.querySelector('.subcategories').style.display = '';
+          const newWidth = ${columnWidth} * App.categoryItems[categoryId].length;
+          document.querySelector('#data').style.width = newWidth + 'px';
+
+          const subcategories = window.App.subcategories[categoryId];
+          const baseMarkup = '<option value="">All</option>';
+          const markup = subcategories.map( (s) => '<option value="' + s + '">' + s + ':&nbsp;' + window.App.categoryItems[categoryId + ':' + s].length  + '</option>').join('');
+          document.querySelector('.subcategories select').innerHTML = baseMarkup + markup;
+
+        }
+
+        for (let tr of [...document.querySelectorAll('tr')]) {
+          let index = 0;
+          for (let td of [...tr.querySelectorAll('td')].slice(1)) {
+            const isVisible = categoryId ? App.categoryItems[categoryId].includes(index) : true;
+            td.style.display = isVisible ? '' : 'none';
+            index += 1;
+          }
+        }
+        setHeight();
+      });
+
+      document.querySelector('.subcategories select').addEventListener('change', function(e) {
+        const categoryId = Array.from(document.querySelectorAll('.categories option')).find( (x) => x.selected).value;
+        const subcategoryId = Array.from(document.querySelectorAll('.subcategories option')).find( (x) => x.selected).value;
+
+        let key = subcategoryId ? (categoryId + ':' + subcategoryId) : categoryId;
+
+        const newWidth = ${columnWidth} * App.categoryItems[key].length;
+        document.querySelector('#data').style.width = newWidth + 'px';
+
+        for (let tr of [...document.querySelectorAll('tr')]) {
+          let index = 0;
+          for (let td of [...tr.querySelectorAll('td')].slice(1)) {
+            const isVisible = App.categoryItems[key].includes(index);
+            td.style.display = isVisible ? '' : 'none';
+            index += 1;
+          }
+        }
+        setHeight();
+      });
+      setHeight();
+    </script>
+    </body>
+
+  `
+}

src/components/VerticalCategory.js

@@ -26,7 +26,7 @@ module.exports.renderVerticalCategory = function({header, guideInfo, subcategori
       flex-direction: column;
     " class="big-picture-section">
       <div style="height: ${categoryTitleHeight}px; width: 100%; display: flex;">
-        ${renderCategoryHeader({href: href, label: header, guideAnchor: guideInfo && guideInfo[header], background: color})}
+        ${renderCategoryHeader({href: href, label: header, guideAnchor: guideInfo, background: color})}
       </div>
       <div style="
         width: 100%;

src/embedded-script.js

@@ -1,13 +1,18 @@
 // An embedded version of a script
 const CncfLandscapeApp = {
   init: function() {
-    // get initial state from the url
     setInterval(function() {
-      document.body.style.height = document.querySelector('.column-content').scrollHeight;
+      window.parent.postMessage({
+        type: 'landscapeapp-resize',
+        height: document.body.scrollHeight
+      }, '*');
     }, 1000);
+
     this.state = {
       selected: null
     }
+
+    if (window.parentIFrame) {
       document.addEventListener('keydown', (e) => {
         if (e.keyCode === 27) {
           if (CncfLandscapeApp.state.selected) {
@@ -16,6 +21,7 @@ const CncfLandscapeApp = {
           }
         }
       });
+    }
 
     document.body.addEventListener('click', (e) => {
       const cardEl = e.target.closest('[data-id]');
@@ -111,6 +117,18 @@ const CncfLandscapeApp = {
   },
 
   showSelectedItem: async function(selectedItemId) {
+    if (!window.parentIFrame) {
+      window.parent.postMessage({
+        type: 'landscapeapp-show',
+        selected: selectedItemId,
+        location: {
+          search: window.location.search,
+          pathname: window.location.pathname
+        }
+      }, '*');
+      return;
+    }
+
     this.selectedItems = this.selectedItems || {};
     if (!this.selectedItems[selectedItemId]) {
       const result = await fetch(`${this.basePath}/data/items/info-${selectedItemId}.html`);

src/iframeResizer.js

@@ -1,13 +1,31 @@
-document.addEventListener('DOMContentLoaded', function() {
+addEventListener('message', function(e) {
-  window.iFrameResize({
+  if (e.data && e.data.type === 'landscapeapp-resize') {
-    log: false,
+    document.querySelector('#landscape').style.height = e.data.height + 'px';
-    onMessage : function(messageData){ // Callback fn when message is received
+  }
-      if (messageData.message.type === 'showModal') {
+  if (e.data && e.data.type === 'landscapeapp-show') {
-        document.querySelector('body').style.overflow = 'hidden';
+    const iframe = document.createElement('iframe');
+    document.body.appendChild(iframe);
+    window.landscapeappModalIframe = iframe;
+    const url = new URL(document.querySelector('#landscape').src);
+    const search = url.search || '?a=a';
+    let src = url.origin + url.pathname.replace('/pages/', '/pages-modal/') + search + '&selected=' + e.data.selected;
+    if (src.indexOf('/pages-modal') === -1) {
+      //support a version with ?embed=yes
+      src = src + '&showmodal=yes'
+    }
+    iframe.src = src;
+    iframe.style.position = 'fixed';
+    iframe.style.left = 0;
+    iframe.style.top = 0;
+    iframe.style.width = '100%';
+    iframe.style.height = '100%';
+    iframe.style.zIndex = 10000;
+    iframe.focus();
+  }
+  if (e.data && e.data.type === 'landscapeapp-hide') {
+    const iframe = window.landscapeappModalIframe;
+    if (iframe) {
+      document.body.removeChild(iframe);
     }
-      if (messageData.message.type === 'hideModal') {
-        document.querySelector('body').style.overflow = 'auto';
   }
-    },
-  }, '#landscape');
 });

src/modal-script.js

@@ -0,0 +1,150 @@
+// An embedded version of a script
+const CncfLandscapeApp = {
+  init: function() {
+    // get initial state from the url
+    this.state = {
+      selected: new URLSearchParams(location.search).get('selected')
+    };
+
+    document.addEventListener('keydown', (e) => {
+      if (e.keyCode === 27) {
+        if (CncfLandscapeApp.state.selected) {
+          this.state.selected = null;
+          this.hideSelectedItem();
+        }
+      }
+    });
+
+    document.body.addEventListener('click', (e) => {
+      const modalBodyEl = e.target.closest('.modal-body');
+      const shadowEl = e.target.closest('.modal-container');
+      if (shadowEl && !modalBodyEl) {
+        this.state.selected = null;
+        this.hideSelectedItem();
+        e.preventDefault();
+        e.stopPropagation();
+      }
+
+      const modalClose = e.target.closest('.modal-close');
+      if (modalClose) {
+        this.state.selected = null;
+        this.hideSelectedItem();
+        e.preventDefault();
+        e.stopPropagation();
+      }
+
+      const nextItem = e.target.closest('.modal-next');
+      if (nextItem && CncfLandscapeApp.nextItemId) {
+        CncfLandscapeApp.state.selected = CncfLandscapeApp.nextItemId;
+        this.showSelectedItem(CncfLandscapeApp.state.selected);
+        e.preventDefault();
+        e.stopPropagation();
+      }
+
+      const prevItem = e.target.closest('.modal-prev');
+      if (prevItem && CncfLandscapeApp.prevItemId) {
+        CncfLandscapeApp.state.selected = CncfLandscapeApp.prevItemId;
+        this.showSelectedItem(CncfLandscapeApp.state.selected);
+        e.preventDefault();
+        e.stopPropagation();
+      }
+
+      const selectedItemInternalLinkEl = e.target.closest('.modal-body a[data-type=internal]')
+      if (selectedItemInternalLinkEl) {
+        e.preventDefault();
+        e.stopPropagation();
+        return
+      }
+
+    }, false);
+
+    // support custom css styles and custom js eval code through iframe
+    window.addEventListener('message', (event) => {
+      var data = event.data;
+      if (data.type === "css") {
+        var styles = data.css;
+        var el = document.createElement('style');
+        el.type = 'text/css';
+        if (el.styleSheet) {
+          el.styleSheet.cssText = styles;
+        } else {
+          el.appendChild(document.createTextNode(styles));
+        }
+        document.getElementsByTagName("head")[0].appendChild(el);
+      }
+      if (data.type === "js") {
+        eval(data.js);
+      }
+    });
+
+    // support css styles via param
+    const params = new URLSearchParams(window.location.search.substring(1));
+    if (params.get('css')) {
+      const element = document.createElement("link");
+      element.setAttribute("rel", "stylesheet");
+      element.setAttribute("type", "text/css");
+      element.setAttribute("href", params.get('css'));
+      document.getElementsByTagName("head")[0].appendChild(element);
+    }
+    if (params.get('style')) {
+      const element = document.createElement("style");
+      let style = params.get('style');
+      try {
+        style = JSON.parse(style)
+      } catch(ex) {
+
+      }
+      element.innerHTML = style;
+      document.getElementsByTagName("head")[0].appendChild(element);
+    }
+    this.showSelectedItem(this.state.selected);
+  },
+
+  showSelectedItem: async function(selectedItemId) {
+    const allIdElements = document.querySelectorAll(`[data-id]`);
+    let allIds = []
+    for (let element of allIdElements) {
+      allIds.push(element.getAttribute('data-id'));
+    }
+    this.selectedItems = this.selectedItems || {};
+    if (!this.selectedItems[selectedItemId]) {
+      const result = await fetch(`${this.basePath}/data/items/info-${selectedItemId}.html`);
+      const text = await result.text();
+      this.selectedItems[selectedItemId] = text;
+    }
+    document.querySelector('.modal').style.display="";
+    document.querySelector('.modal .modal-content').outerHTML = this.selectedItems[selectedItemId];
+    document.querySelector('body').style.overflow = 'hidden';
+
+    if (window.twttr) {
+      window.twttr.widgets.load();
+    } else {
+      setTimeout( () => window.twttr && window.twttr.widgets.load(), 1000);
+    }
+
+    //calculate previous and next items;
+    const index = allIds.indexOf(selectedItemId);
+    const prevItem = allIds[index - 1] || null;
+    const nextItem = allIds[index + 1] || null;
+
+    this.nextItemId = nextItem;
+    this.prevItemId = prevItem;
+
+    if (nextItem) {
+      document.querySelector('.modal-next').removeAttribute('disabled');
+    } else {
+      document.querySelector('.modal-next').setAttribute('disabled', '');
+    }
+
+    if (prevItem) {
+      document.querySelector('.modal-prev').removeAttribute('disabled');
+    } else {
+      document.querySelector('.modal-prev').setAttribute('disabled', '');
+    }
+  },
+
+  hideSelectedItem: function() {
+    window.parent.postMessage({type: 'landscapeapp-hide'}, '*');
+  }
+}
+document.addEventListener('DOMContentLoaded', () => CncfLandscapeApp.init());

src/script.js

@@ -14,11 +14,19 @@ const CncfLandscapeApp = {
 
     if (CncfLandscapeApp.state.embed) {
       document.querySelector('html').classList.add('embed');
+      if (CncfLandscapeApp.state.showModal) {
+        document.querySelector('html').classList.add('modal-embed')
+      } else {
         setInterval(function() {
-        document.body.style.height = document.querySelector('.column-content').scrollHeight + 80;
+          const realHeight = document.querySelector('.column-content').scrollHeight + 80;
+          window.parent.postMessage({
+            type: 'landscapeapp-resize',
+            height: realHeight
+          }, '*');
         }, 1000);
         document.querySelector('#embedded-footer a').href = this.stringifyBrowserUrl({...this.state, embed: false});
       }
+    }
     if (CncfLandscapeApp.state.cardStyle === 'borderless') {
       document.querySelector('html').classList.add('borderless-mode');
     }
@@ -42,6 +50,16 @@ const CncfLandscapeApp = {
         } else if (document.querySelector('.select-popup').style.display === '') {
           document.querySelector('.select-popup').style.display = "none";
         }
+      } else {
+        if (document.querySelector('.select-popup').style.display === '') {
+          if (e.key >= 'a' && e.key <= 'z') {
+            const items = [...document.querySelectorAll('.select-popup .pure-material-checkbox span')];
+            const first = items.find( (i) => i.innerText.toLowerCase().startsWith(e.key.toLowerCase()));
+            if (first) {
+              first.scrollIntoView();
+            }
+          }
+        }
       }
     });
 
@@ -159,7 +177,7 @@ const CncfLandscapeApp = {
         const linkState = this.parseUrl({search: groupingInternalLinkEl.getAttribute('href'), pathname: '', hash: ''});
 
         const f = (name, x) => this.calculateFullSelection(name, x);
-        const allowedProps = ['grouping', 'sort', 'bestpractices', 'enduser', 'parent', 'language'];
+        const allowedProps = ['grouping', 'sort', 'bestpractices', 'enduser', 'parent', 'language', 'specification'];
         const otherProps = ['category', 'project', 'license', 'organization', 'headquarters', 'company-type', 'industries']
         for (let key of allowedProps) {
           newState[key] = linkState[key] || CncfLandscapeApp.initialState[key];
@@ -187,7 +205,7 @@ const CncfLandscapeApp = {
         // Only set certain properties: filterable + invisible filters
         // for visible filter we need to always expand a current selection
         const f = (name, x) => this.calculateFullSelection(name, x);
-        const allowedProps = ['grouping', 'sort', 'bestpractices', 'enduser', 'parent', 'language'];
+        const allowedProps = ['grouping', 'sort', 'bestpractices', 'enduser', 'parent', 'language', 'specification'];
         const otherProps = ['category', 'project', 'license', 'organization', 'headquarters', 'company-type', 'industries']
         for (let key of allowedProps) {
           newState[key] = linkState[key] || CncfLandscapeApp.initialState[key];
@@ -264,7 +282,7 @@ const CncfLandscapeApp = {
         const newState = {...CncfLandscapeApp.state };
 
         const f = (name, x) => this.calculateFullSelection(name, x);
-        const allowedProps = ['bestpractices', 'enduser', 'parent', 'language'];
+        const allowedProps = ['bestpractices', 'enduser', 'parent', 'language', 'specification'];
         const otherProps = ['category', 'project', 'license', 'organization', 'headquarters', 'company-type', 'industries']
         for (let key of allowedProps) {
           newState[key] = CncfLandscapeApp.initialState[key];
@@ -645,6 +663,7 @@ const CncfLandscapeApp = {
 
     const parseMode = (x) => (x || '').indexOf('-mode') !== -1 ? 'card' : (x || CncfLandscapeApp.initialMode);
     const parseCardStyle = (x) => (x || '').indexOf('-mode') !== -1 ? x.replace('-mode', '') : 'card';
+    const parseParamStyle = (x) => ['logo', 'borderless', 'flat'].indexOf(x) !== -1 ? x : '';
 
     return {
       zoom: +params.get('zoom') / 100 || 1,
@@ -653,7 +672,7 @@ const CncfLandscapeApp = {
       activeSection: hash,
 
       mode: parseMode(params.get('format') || pathname) || CncfLandscapeApp.initialMode,
-      cardStyle: params.get('style') || parseCardStyle(pathname),
+      cardStyle: parseParamStyle(params.get('style')) || parseCardStyle(pathname),
 
       grouping: params.get('grouping') || 'project',
       sort: params.get('sort') || 'name',
@@ -670,9 +689,10 @@ const CncfLandscapeApp = {
       enduser: params.get('enduser') || '',
       parent: params.get('parent') || '',
       language: params.get('language') || '',
-
+      specification: params.get('specification') || '',
       selected: params.get('selected') || null,
       embed: params.has('embed'),
+      showModal: params.has('showmodal'),
     };
   },
   propagateStateToUiAndUrl: function() {
@@ -700,6 +720,18 @@ const CncfLandscapeApp = {
     assignSingleSelect('sort');
     assignSingleSelect('grouping');
     assignMultiSelect('category');
+
+    const isCardMode = CncfLandscapeApp.state.mode === 'card';
+    if (isCardMode) {
+      document.querySelector('.select[data-name=sort]').classList.remove('select-disabled');
+      document.querySelector('.select[data-name=grouping]').classList.remove('select-disabled');
+      document.querySelector('.select[data-name=category]').classList.remove('select-disabled');
+    } else {
+      document.querySelector('.select[data-name=sort]').classList.add('select-disabled');
+      document.querySelector('.select[data-name=grouping]').classList.add('select-disabled');
+      document.querySelector('.select[data-name=category]').classList.add('select-disabled');
+    }
+
     assignMultiSelect('project');
     assignMultiSelect('license');
     assignMultiSelect('organization');
@@ -812,7 +844,7 @@ const CncfLandscapeApp = {
       params[field] = CncfLandscapeApp.calculateShortSelection(field).url
     }
     // no fields for certain filters yet
-    for (let field of ['sort', 'grouping', 'bestpractices', 'enduser', 'parent', 'language']) {
+    for (let field of ['sort', 'grouping', 'bestpractices', 'enduser', 'parent', 'language', 'specification']) {
       params[field] = state[field]
     }
 
@@ -845,8 +877,7 @@ const CncfLandscapeApp = {
       }
     }
     // no fields for certain filters yet
-    for (let field of ['grouping', 'sort', 'selected', 'bestpractices', 'enduser', 'parent', 'language',
+    for (let field of ['grouping', 'sort', 'selected', 'bestpractices', 'enduser', 'parent', 'language', 'specification', 'fullscreen', 'embed']) {
-      'fullscreen', 'embed']) {
       if (state[field] !== initialState[field]) {
         params[field] = state[field]
       }
@@ -869,6 +900,18 @@ const CncfLandscapeApp = {
     return url;
   },
   showSelectedItem: async function(selectedItemId) {
+    if (this.state.embed && !this.state.showModal) {
+      window.parent.postMessage({
+        type: 'landscapeapp-show',
+        selected: selectedItemId,
+        location: {
+          search: window.location.search,
+          pathname: window.location.pathname
+        }
+      }, '*');
+      return;
+    }
+
     this.selectedItems = this.selectedItems || {};
     if (!this.selectedItems[selectedItemId]) {
       const result = await fetch(`${this.basePath}/data/items/info-${selectedItemId}.html`);
@@ -889,7 +932,7 @@ const CncfLandscapeApp = {
     let prevItem = null;
     let nextItem = null;
     if (this.state.mode === 'card') {
-      const items = this.groupedItems.flatMap( (x) => x.items);
+      const items = (this.groupedItems || []).flatMap( (x) => x.items);
       for (let i = 0; i < items.length; i++) {
         if (items[i].id === selectedItemId) {
           prevItem = (items[i - 1] || {}).id;
@@ -920,35 +963,11 @@ const CncfLandscapeApp = {
     } else {
       document.querySelector('.modal-prev').setAttribute('disabled', '');
     }
-
-    if (window.parentIFrame) {
-      window.parentIFrame.sendMessage({type: 'showModal'});
-      window.parentIFrame.getPageInfo(function(info) {
-        var offset = info.scrollTop - info.offsetTop;
-        var maxHeight = info.clientHeight * 0.9;
-        if (maxHeight > 640) {
-          maxHeight = 640;
-        }
-        var defaultTop = (info.windowHeight - maxHeight) / 2;
-        var top = defaultTop + offset;
-        if (top < 0 && info.iframeHeight <= 600) {
-          top = 10;
-        }
-        setTimeout(function() {
-          const modal = document.querySelector('.modal-body');
-          if (modal) {
-            modal.style.top = top + 'px';
-            modal.style.marginTop = 0;
-            modal.style.marginBottom = 0;
-            modal.style.bottom = '';
-            modal.style.maxHeight = maxHeight + 'px';
-          }
-        }, 10);
-      });
-    }
-
   },
   updateUrl: function() {
+    if (CncfLandscapeApp.state.embed) {
+      return;
+    }
     const newUrl = CncfLandscapeApp.stringifyBrowserUrl(CncfLandscapeApp.state);
     if (newUrl !== this.previousUrl) {
       history.pushState(CncfLandscapeApp.state, '', newUrl);
@@ -957,12 +976,12 @@ const CncfLandscapeApp = {
     }
   },
   hideSelectedItem: function() {
+    if (this.state.showModal) {
+      window.parent.postMessage({type: 'landscapeapp-hide'}, '*');
+      return;
+    }
     document.querySelector('.modal').style.display="none";
     document.querySelector('body').style.overflow = '';
-    if (window.parentIFrame && this.state.embed) {
-      window.parentIFrame.sendMessage({type: 'hideModal'})
-    }
-
     this.updateUrl();
   },
   fetchApiData: async function() {
@@ -1044,7 +1063,7 @@ const CncfLandscapeApp = {
 
     // edge case: we just opened a tab without filters - then just display everything!
     if (this.state.mode === this.initialMode) {
-      const allowedProps = ['bestpractices', 'enduser', 'parent', 'language'];
+      const allowedProps = ['bestpractices', 'enduser', 'parent', 'language', 'specification'];
       const otherProps = ['project', 'license', 'organization', 'headquarters', 'company-type', 'industries']
       let same = true;
       for (let key of [...allowedProps, ...otherProps]) {
@@ -1114,6 +1133,7 @@ const CncfLandscapeApp = {
         result[card.getAttribute('data-id')] = card;
       }
       this.cards = result;
+
     }
 
     const apiData = await this.fetchApiData();
@@ -1153,6 +1173,10 @@ const CncfLandscapeApp = {
     }
     document.querySelector('.column-content').replaceChildren(fragment);
     this.lastCards = JSON.stringify(this.groupedItems);
+
+    if (this.state.selected) {
+      this.propagateStateToUi();
+    }
   }
 }
 document.addEventListener('DOMContentLoaded', () => CncfLandscapeApp.init());

src/style.css

@@ -249,6 +249,42 @@ a:hover svg {
   position: relative;
 }
 
+#guide-page .links {
+  position: fixed;
+  right: 10px;
+  top: 100px;
+  font-size: 16px;
+  width: 130px;
+  font-weight: bold;
+}
+
+@media (max-width: 1250px) {
+  #guide-page .links {
+    position: inherit;
+    width: 100%;
+    left: 0;
+    top: 0;
+  }
+  #guide-page .links > div {
+    display: inline-block;
+    padding-right: 100px;
+  }
+}
+
+#guide-page .links > div {
+  position: relative;
+}
+
+#guide-page .links svg {
+  fill: currentColor;
+  position: relative;
+  top: 2px;
+  width: 16px;
+  height: 16px;
+}
+
+
+
 #guide-page h1 {
   font-size: 2.5em;
 }
@@ -1293,6 +1329,7 @@ html.ios.has-selected-item, html.ios.has-selected-item body {
   cursor: pointer;
   min-width: 40px;
   text-align: center;
+  margin-top: -6px;
 }
 .right-buttons .disabled {
   cursor: default;
@@ -1306,6 +1343,16 @@ html.ios.has-selected-item, html.ios.has-selected-item body {
   display: none;
 }
 
+.embed.modal-embed .column-content {
+  display: none;
+}
+.embed.modal-embed {
+  background: inherit;
+}
+.embed.modal-embed #embedded-footer {
+  display: none;
+}
+
 .tweet-button {
   width: 105px;
   display: flex;
@@ -1416,6 +1463,13 @@ html.ios.has-selected-item, html.ios.has-selected-item body {
   height: 63px;
 }
 
+.inner-landscape .large-item.large-item-16, .items .large-item.large-item-16 {
+  cursor: pointer;
+  position: relative;
+  width: 145px;
+  height: 128px;
+}
+
 .inner-landscape .small-item, .items .small-item {
   cursor: pointer;
   position: relative;
@@ -1434,11 +1488,16 @@ html.ios.has-selected-item, html.ios.has-selected-item body {
   grid-row-end: span 1;
 }
 
-.inner-landscape .item-wrapper.wrapper-large, .items .item-wrapper.wrapper-large {
+.inner-landscape .item-wrapper.wrapper-large-4, .items .item-wrapper.wrapper-large-4 {
   grid-column-end: span 2;
   grid-row-end: span 2;
 }
 
+.inner-landscape .item-wrapper.wrapper-large-16, .items .item-wrapper.wrapper-large-16 {
+  grid-column-end: span 4;
+  grid-row-end: span 4;
+}
+
 
 
 @media (max-width: var(--sm-screen)) {
@@ -1636,6 +1695,7 @@ html.ios.has-selected-item, html.ios.has-selected-item body {
     height: 180px;
     top: 0;
     left: 0;
+    user-select: none;
   }
 
   .modal-content .product-tags {
@@ -1831,6 +1891,31 @@ html.ios.has-selected-item, html.ios.has-selected-item body {
   height: 22px;
 }
 
+.twolines {
+  height: 22px !important;
+  width: 145px !important;
+  position: relative !important;
+}
+
+.twolines .tag-name {
+  position: absolute;
+  left: 0px;
+  top: -1px;
+  width: 145px;
+  padding: 0 !important;
+  margin: 0 !important;
+  text-align: center;
+
+}
+.twolines .tag-value {
+  position: absolute;
+  left: 0px;
+  width: 145px;
+  white-space: normal !important;
+  top: 5px;
+  text-align: center;
+}
+
 .modal-content .single-column .product-property .col:nth-child(1) {
   display: inline-block;
   width: 40%;
@@ -1848,6 +1933,10 @@ html.ios.has-selected-item, html.ios.has-selected-item body {
 	width: 180px;
   margin-bottom: 10px;
 }
+.select-disabled {
+  opacity: 0.35;
+  pointer-events: none;
+}
 
 .select-text {
 	position: relative;

src/svg/edit.svg

@@ -0,0 +1,3 @@
+<svg viewBox="0 0 16 16">
+    <path fill-rule="evenodd" d="M11.013 1.427a1.75 1.75 0 012.474 0l1.086 1.086a1.75 1.75 0 010 2.474l-8.61 8.61c-.21.21-.47.364-.756.445l-3.251.93a.75.75 0 01-.927-.928l.929-3.25a1.75 1.75 0 01.445-.758l8.61-8.61zm1.414 1.06a.25.25 0 00-.354 0L10.811 3.75l1.439 1.44 1.263-1.263a.25.25 0 000-.354l-1.086-1.086zM11.189 6.25L9.75 4.81l-6.286 6.287a.25.25 0 00-.064.108l-.558 1.953 1.953-.558a.249.249 0 00.108-.064l6.286-6.286z"></path>
+</svg>

src/types/fields.js

@@ -233,6 +233,19 @@ const fields = {
     },
     values: [{id: true, label: 'Yes', url: 'yes'}, {id: false, label: 'No', url: 'no'}]
   },
+  specification: {
+    id: 'extra',
+    label: 'Specification',
+    url: 'specification',
+    filterFn: function(filter, value, item) {
+      if (filter === null) {
+        return true;
+      }
+      const isSpecification = item.extra && (item.extra.specification === true || item.extra.specification === 'yes' || item.extra.specification === 'true');
+      return filter === true ? isSpecification : !isSpecification;
+    },
+    values: [{id: true, label: 'Yes', url: 'yes'}, {id: false, label: 'No', url: 'no'}]
+  },
   parents: {
     id: 'parent',
     url: 'parent',

src/utils/format.js

@@ -21,6 +21,9 @@ module.exports.millify = function(value) {
 }
 
 module.exports.h = function(html) {
+  if (!html) {
+    return '';
+  }
   var entityMap = {
     '&': '&',
     '<': '&lt;',

src/utils/itemsCalculator.js

@@ -35,8 +35,9 @@ const getFilteredItems = module.exports.getFilteredItems = function({data, filte
     var filterByLanguage = filterFn({field: 'language', filters});
     var filterByCompanyType = filterFn({field: 'companyType', filters});
     var filterByIndustries = filterFn({field: 'industries', filters});
+    var filterBySpecification = filterFn({field: 'specification', filters});
     return data.filter(function(x) {
-      return filterHostedProject(x) && filterByLicense(x) && filterByOrganization(x) && filterByHeadquarters(x) && filterByLandscape(x) && filterByBestPractices(x) && filterByEnduser(x) && filterByParent(x) && filterByLanguage(x) && filterByCompanyType(x) && filterByIndustries(x);
+      return filterHostedProject(x) && filterByLicense(x) && filterByOrganization(x) && filterByHeadquarters(x) && filterByLandscape(x) && filterByBestPractices(x) && filterByEnduser(x) && filterByParent(x) && filterByLanguage(x) && filterByCompanyType(x) && filterByIndustries(x) && filterBySpecification(x);
     });
 }
 
@@ -83,14 +84,16 @@ const getSortedItems = function({data, filters, sortField, sortDirection}) {
   return sortedViaMainSort.concat(sortedViaName1).concat(sortedViaName2).concat(sortedViaName3).concat(sortedViaName4);
 }
 
-const getGroupedItems = module.exports.getGroupedItems = function({ data, filters, sortField, sortDirection, grouping }) {
+const getGroupedItems = module.exports.getGroupedItems = function({ data, filters, sortField, sortDirection, grouping, skipDuplicates }) {
   const items = getSortedItems({ data, filters, sortField, sortDirection });
 
+  const uniq = skipDuplicates ? (x) => _.uniqBy(x, 'id') : (x) => x;
+
   if (grouping === 'no') {
     return [{
       key: 'key',
       header: 'No Grouping',
-      items: items
+      items: uniq(items)
     }]
   }
 
@@ -105,7 +108,7 @@ const getGroupedItems = module.exports.getGroupedItems = function({ data, filter
     return {
       key: properKey,
       header: groupingLabel(grouping, properKey),
-      items: value,
+      items: uniq(value),
       href: stringifyParams({filters: newFilters, grouping, sortField})
     }
   }), (group) => groupingOrder(grouping)(group.key));

src/utils/landscapeCalculations.js

@@ -7,8 +7,6 @@ const settings = readJsonFromDist('settings');
 const itemMargin = module.exports.itemMargin = 3;
 const smallItemWidth = module.exports.smallItemWidth = 34;
 const smallItemHeight = module.exports.smallItemHeight = 30;
-const largeItemWidth = module.exports.largeItemWidth = 2 * smallItemWidth + itemMargin;
-const largeItemHeight = module.exports.largeItemHeight = 2 * smallItemHeight + itemMargin;
 const subcategoryMargin = module.exports.subcategoryMargin = 6;
 const subcategoryTitleHeight = module.exports.subcategoryTitleHeight = 20;
 const dividerWidth = module.exports.dividerWidth = 2;
@@ -19,13 +17,19 @@ const headerHeight = module.exports.headerHeight = 40;
 /* eslint-enable */
 
 // Check if item is large
-const isLargeFn = module.exports.isLargeFn = ({ relation, category, member, categoryAttrs }) => {
+const sizeFn = module.exports.sizeFn = ({ relation, category, member, categoryAttrs }) => {
   const relationInfo = fields.relation.valuesMap[relation]
+  if (!relationInfo) {
+    console.error(`No relation with name ${relation}`);
+  }
+  if (relationInfo.x4) {
+    return 16;
+  }
   if (category === settings.global.membership) {
     const membershipInfo = settings.membership[member];
-    return membershipInfo && !!membershipInfo.is_large;
+    return (membershipInfo && !!membershipInfo.is_large) ? 4 : 1;
   }
-  return !!categoryAttrs.isLarge || !!relationInfo.big_picture_order;
+  return (!!categoryAttrs.isLarge || !!relationInfo.big_picture_order) ? 4 : 1;
 }
 
 // Compute if items are large and/or visible.
@@ -34,10 +38,9 @@ const isLargeFn = module.exports.isLargeFn = ({ relation, category, member, cate
 const computeItems = (subcategories, addInfoIcon = false) => {
   return subcategories.map(subcategory => {
     const filteredItems = subcategory.items.reduce((acc, { id }) => ({ ...acc, [id]: true }), {})
-    const allItems = subcategory.allItems.map(item => ({ ...item, isLarge: isLargeFn(item), isVisible: filteredItems[item.id]  }))
+    const allItems = subcategory.allItems.map(item => ({ ...item, size: sizeFn(item), isVisible: filteredItems[item.id]  }))
-    const itemsCount = allItems.reduce((count, item) => count + (item.isLarge ? 4 : 1), 0) + (addInfoIcon ? 1 : 0)
+    const itemsCount = allItems.reduce((count, item) => count + item.size, 0) + (addInfoIcon ? 1 : 0)
-    const largeItemsCount = allItems.reduce((count, item) => count + (item.isLarge ? 1 : 0), 0)
+    const largeItemsCount = allItems.reduce((count, item) => count + (item.size === 16 ? 4 : item.size === 4 ? 1 : 0), 0)
-
     return { ...subcategory, allItems, itemsCount, largeItemsCount }
   })
 }

src/utils/landscapeSettings.js

@@ -6,7 +6,7 @@ function calcLandscapeSettingsList(settingsObj) {
     .sort((a, b) => a.tab_index - b.tab_index)
     .map(({ url, ...rest }) => {
       const basePath = url === 'landscape' ? null : url
-      const isMain = settingsObj.big_picture.main.url === url
+      const isMain = !rest.category;
       return { url, basePath, isMain, ...rest }
     })
 

src/utils/saneName.js

@@ -1,5 +1,12 @@
 const  _ = require('lodash');
 const { paramCase } = require('change-case');
+let hash = {};
 module.exports.saneName = function(x) {
-  return _.deburr(paramCase(x));
+  let result = _.deburr(paramCase(x));
+  if (hash[result] && hash[result] !== x) {
+    result = result + '-2';
+    console.info(`Hash for ${x} is ${result}`);
+  }
+  hash[result] = x;
+  return result;
 };

tools/addExternalInfo.js

@@ -12,6 +12,7 @@ const { extractSavedCrunchbaseEntries, fetchCrunchbaseEntries } = require('./cru
 const { fetchGithubEntries } = require('./fetchGithubStats');
 const { getProcessedRepos, getProcessedReposStartDates } = require('./repos');
 const { fetchStartDateEntries } = require('./fetchGithubStartDate');
+const { fetchCloEntries } = require('./fetchCloData');
 const { extractSavedTwitterEntries, fetchTwitterEntries } = require('./twitter');
 const {
   extractSavedBestPracticeEntries,
@@ -43,11 +44,15 @@ function reportOptions() {
 }
 if (key.toLowerCase() === 'easy') {
   reportOptions();
+} else if (key.toLowerCase() === 'crunchbase') {
+
+  useCrunchbaseCache = false;
+  reportOptions();
 }
 else if (key.toLowerCase() === 'medium') {
   useTwitterCache=false;
   useGithubCache=false;
-  useCrunchbaseCache=false;
+  useCrunchbaseCache=true;
   useBestPracticesCache=false;
   reportOptions();
 }
@@ -164,11 +169,12 @@ async function main() {
 
   console.info('Fetching last tweet dates');
   const savedTwitterEntries = await extractSavedTwitterEntries();
-  const twitterEntries = await fetchTwitterEntries({
+
-    cache: savedTwitterEntries,
+  // const twitterEntries = await fetchTwitterEntries({
-    preferCache: useTwitterCache,
+    // cache: savedTwitterEntries,
-    crunchbaseEntries: crunchbaseEntries
+    // preferCache: useTwitterCache,
-  });
+    // crunchbaseEntries: crunchbaseEntries
+  // });
 
   if (hasFatalErrors()) {
     console.info('Reporting fatal errors');
@@ -184,6 +190,10 @@ async function main() {
     cache: savedBestPracticeEntries,
     preferCache: useBestPracticesCache
   });
+  require('fs').writeFileSync('/tmp/bp.json', JSON.stringify(bestPracticeEntries, null, 4));
+
+  console.info('Fetching CLOMonitor data');
+  const cloEntries = await fetchCloEntries();
 
   const tree = traverse(landscape);
   console.info('Processing the tree');
@@ -309,10 +319,9 @@ async function main() {
       }
       node.best_practice_data = bestPracticeEntry;
       delete node.best_practice_data.repo_url;
-      // twitter
+      //twitter
       const twitter = actualTwitter(node, node.crunchbase_data);
-
+      const twitterEntry = _.clone(_.find(savedTwitterEntries, {
-      const twitterEntry = _.clone(_.find(twitterEntries, {
         url: twitter
       }));
       if (twitterEntry) {
@@ -320,6 +329,11 @@ async function main() {
         delete twitterEntry.url;
       }
 
+      const cloEntry = _.clone(_.find(cloEntries, { clomonitor_name: node.extra?.clomonitor_name }));
+      // svg clomonitor
+      if (cloEntry) {
+        node.extra.clomonitor_svg = cloEntry.svg
+      }
     }
   });
 

tools/apiClients.js

@@ -62,9 +62,12 @@ const requestWithRetry = async ({ attempts = maxAttempts, resolveWithFullRespons
       const response = await axios(rest);
       return resolveWithFullResponse ? response : response.data
     } catch (ex) {
+
       const { response = {}, ...error } = ex
       const { status } = response
 
+      const isGithubIssue = (response?.data?.message || '').indexOf('is too large to list') !== -1;
+
       const message = [
         `Attempt #${maxAttempts - attempts + 1}`,
         `(Status Code: ${status || error.code})`,
@@ -72,10 +75,12 @@ const requestWithRetry = async ({ attempts = maxAttempts, resolveWithFullRespons
       ].join(' ')
       if (key === keys[keys.length - 1]) {
         console.info(message);
+      } else {
+        console.info(`Failed to use key #${keys.indexOf(key)} of ${keys.length}`);
       }
       const rateLimited = retryStatuses.includes(status)
       const dnsError = error.code === 'ENOTFOUND' && error.syscall === 'getaddrinfo'
-      if (attempts <= 0 || (!rateLimited && !dnsError)) {
+      if (attempts <= 0 || (!rateLimited && !dnsError) || isGithubIssue) {
         throw ex;
       }
       lastEx = ex;
@@ -125,7 +130,7 @@ module.exports.CrunchbaseClient = ApiClient({
 
 module.exports.GithubClient = ApiClient({
   baseURL: 'https://api.github.com',
-  retryStatuses: [403], // Github returns 403 when rate limiting.
+  retryStatuses: [401, 403], // Github returns 403 when rate limiting.
   delayFn: error => {
     const rateLimitRemaining = parseInt(_.get(error, ['response', 'headers', 'x-ratelimit-remaining'], 1))
     const rateLimitReset = parseInt(_.get(error, ['response', 'headers', 'x-ratelimit-reset'], 1)) * 1000

tools/crunchbase.js

@@ -18,6 +18,7 @@ const { addError, addFatal } = errorsReporter('crunchbase');
 const EXCHANGE_SUFFIXES = {
   'ams': 'AS', // Amsterdam
   'bit': 'MI', // Milan
+  'bme': 'MC', // Madrid
   'epa': 'PA', // Paris
   'etr': 'DE', // XETRA
   'fra': 'F',  // Frankfurt
@@ -138,16 +139,21 @@ const fetchCrunchbaseOrganization = async id => {
 const fetchData = module.exports.fetchData = async function(name) {
   const result = await fetchCrunchbaseOrganization(name)
   const mapAcquisitions = function(a) {
-    const result = {
+    let result1;
+    try {
+      result1 = {
         date: a.announced_on.value,
         acquiree: a.acquiree_identifier.value,
       }
       if (a.price) {
         result.price = a.price.value_usd
       }
-    return result;
+    } catch(ex) {
+      return null;
     }
-  let acquisitions = result.cards.acquiree_acquisitions.map(mapAcquisitions);
+    return result1;
+  }
+  let acquisitions = result.cards.acquiree_acquisitions.map(mapAcquisitions).filter( (x) => !!x);
   const limit = 100;
   let lastPage = result;
   while (lastPage.cards.acquiree_acquisitions.length === limit) {
@@ -165,7 +171,8 @@ const fetchData = module.exports.fetchData = async function(name) {
     const parentOrganization = lastOrganization.cards.parent_organization[0].identifier.permalink
     if (parents.map(p => p.identifier.permalink).includes(parentOrganization)) {
       const { permalink } = lastOrganization.properties.identifier
-      throw new Error(`Circular dependency detected: ${permalink} and ${parentOrganization} are parents of each other`)
+      console.info(`Circular dependency detected: ${permalink} and ${parentOrganization} are parents of each other`)
+      break;
     }
     lastOrganization = await fetchCrunchbaseOrganization(parentOrganization)
     parents.push({ ...lastOrganization.properties, delisted: isDelisted(lastOrganization) })
@@ -178,6 +185,9 @@ const fetchData = module.exports.fetchData = async function(name) {
   const totalFunding = firstWithTotalFunding ? + firstWithTotalFunding.funding_total.value_usd.toFixed() : undefined;
 
   const getAddressPart = function(part) {
+    if (!result.cards.headquarters_address[0]) {
+      return " N/A";
+    }
     return (result.cards.headquarters_address[0].location_identifiers.filter( (x) => x.location_type === part)[0] || {}).value
   }
 
@@ -192,10 +202,6 @@ const fetchData = module.exports.fetchData = async function(name) {
     }
   })();
 
-  if (!result.cards.headquarters_address[0]) {
-    return 'no address';
-  }
-
   return {
     name: result.properties.name,
     description: result.properties.short_description,
@@ -237,11 +243,6 @@ module.exports.fetchCrunchbaseEntries = async function({cache, preferCache}) {
     }
     try {
       const result = await fetchData(c.name);
-      if (result === 'no address') {
-        fatalErrors.push(`no headquarter addresses for ${c.name} at ${c.crunchbase}`);
-        reporter.write(fatal("F"));
-        return null;
-      }
 
       const entry = {
         url: c.crunchbase,
@@ -256,8 +257,12 @@ module.exports.fetchCrunchbaseEntries = async function({cache, preferCache}) {
       if (!(c.ticker === null) && (entry.ticker || c.ticker)) {
         // console.info('need to get a ticker?');
         entry.effective_ticker = c.ticker || entry.ticker;
+        try {
           entry.market_cap = await getMarketCap(entry.effective_ticker, entry.stockExchange);
           entry.kind = 'market_cap';
+        } catch(ex) {
+          console.info(`Skipping market cap calculation`);
+        }
       } else if (entry.funding) {
         entry.kind = 'funding';
       } else {
@@ -267,6 +272,7 @@ module.exports.fetchCrunchbaseEntries = async function({cache, preferCache}) {
       return entry;
       // console.info(entry);
     } catch (ex) {
+      console.info(ex);
       if (cachedEntry) {
         errors.push(`Using cached entry, because can not fetch: ${c.name} ` +  ex.message.substring(0, 200));
         reporter.write(error("E"));

tools/fetchBestPractices.js

@@ -37,7 +37,7 @@ async function getLandscapeItems() {
 }
 
 async function fetchEntriesNoRetry() {
-  const maxNumber = 200;
+  const maxNumber = 300;
   const items = await Promise.map(_.range(1, maxNumber), async function(number) {
     const result = await requestWithRetry({
       url: `https://bestpractices.coreinfrastructure.org/en/projects.json?page=${number}`

tools/fetchCloData.js

@@ -0,0 +1,52 @@
+const path = require('path');
+const Promise = require('bluebird');
+const traverse = require('traverse');
+const _ = require('lodash');
+
+const { requestWithRetry } = require('./requestWithRetry');
+const { projectPath } = require('./settings');
+
+async function getLandscapeItems() {
+  const source = require('js-yaml').load(require('fs').readFileSync(path.resolve(projectPath, 'landscape.yml')));
+  const traverse = require('traverse');
+  const tree = traverse(source);
+  const items = [];
+  tree.map(function(node) {
+    if (!node) {
+      return;
+    }
+    if (node.item !== null) {
+      return;
+    }
+    if (!node.extra) {
+      return;
+    }
+    if (!node.extra.clomonitor_name) {
+      return;
+    }
+    items.push({clomonitor_name: node.extra.clomonitor_name });
+  });
+  return items;
+}
+
+module.exports.fetchCloEntries = async function() {
+  const items = await getLandscapeItems();
+  const result = await Promise.mapSeries(items, async function(item) {
+    try {
+      const svg = await requestWithRetry({
+        url: `https://clomonitor.io/api/projects/cncf/${item.clomonitor_name}/report-summary?theme=light`
+      });
+      return {
+        clomonitor_name: item.clomonitor_name,
+        svg: svg
+      }
+    } catch(ex) {
+      console.info(`Warning: failed to fetch ${item.clomonitor_name}`);
+      return {
+        clomonitor_name: item.clomonitor_name,
+        svg: ''
+      }
+    }
+  });
+  return result;
+}

tools/fetchImages.js

@@ -31,6 +31,7 @@ async function getLandscapeItems() {
     if (node.item !== null) {
       return;
     }
+    saneName(node.name);
     items.push({logo: node.logo, name: node.name, organization: node.organization});
   });
   _.each(items, function(item) {

tools/fetchTweetDate.js

@@ -0,0 +1,20 @@
+const twitter = "https://twitter.com/vitessio";
+async function main() {
+  const puppeteer = require('puppeteer');
+  const browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox', '--ignore-certificate-errors'], headless: false});
+  const page = await browser.newPage();
+  page.setDefaultNavigationTimeout(120 * 1000);
+
+  try {
+    await page.goto(twitter);
+    await page.waitForTimeout(5000);
+    const data = await page.evaluate(`Array.from(document.querySelectorAll('time[datetime]')).map( (x) => x.dateTime)`)
+    console.info(data);
+
+
+  } catch(ex) {
+    console.info(ex);
+  }
+
+}
+main();

tools/fundingHistoryData.js

@@ -14,24 +14,17 @@ require('child_process').execSync(`cd '${projectPath}'; git remote add github ht
 console.info(require('child_process').execSync(`cd '${projectPath}'; git fetch github`).toString('utf-8'));
 
 function getFileFromHistory(days) {
-  const commit = getCommitFromHistory(days);
+  try {
-  const content = require('child_process').execSync(`cd '${projectPath}'; git show ${commit}:processed_landscape.yml`, {
+    const content = require('child_process').execSync(`cd '${projectPath}'; git show HEAD~${days}:processed_landscape.yml 2>/dev/null`, {
       maxBuffer: 100 * 1024 * 1024
     }).toString('utf-8');
     const source = require('js-yaml').load(content);
     return source;
+  } catch(ex) {
+    return { landscape: []};
+  }
 }
 
-function getCommitFromHistory(days) {
-  const defaultBranch = execSync(`cd '${projectPath}'; git remote show github | grep HEAD`).toString().trim().split(' ').pop()
-  const commit = execSync(`cd '${projectPath}'; git log --format='%H' -n 1 --before='{${days} days ago}' --author='CNCF-bot' github/${defaultBranch}`, {
-    maxBuffer: 100 * 1024 * 1024
-  }).toString('utf-8').trim();
-  return commit;
-}
-
-
-
 function getFileFromFs() {
   const content = require('fs').readFileSync(path.resolve(projectPath, 'processed_landscape.yml'), 'utf-8');
   const source = require('js-yaml').load(content);

tools/generateJson.js

@@ -141,7 +141,7 @@ async function main () {
           result = formatCity(node.crunchbase_data);
         }
         if (!result) {
-          result = 'N/A';
+          result = ' N/A';
         }
         return result;
       };
@@ -162,14 +162,14 @@ async function main () {
       };
       const getLicense = function() {
         if (node.license) {
-          return node.license;
+          return node.license || 'Unknown License';
         }
         if ((node.hasOwnProperty('open_source') && !node.open_source) || (!node.github_data && !node.other_repo_url)) {
           return 'NotOpenSource';
         }
 
         if (node.github_data) {
-          return node.github_data.license;
+          return node.github_data.license || 'Unknown License';
         }
 
         return 'Unknown License';
@@ -289,6 +289,7 @@ async function main () {
     }
   });
 
+  settings.global.flags = settings.global.flags || {};
   if (settings.global.flags.companies) {
     // Handle companies in a special way
     const hasCompanyCategory = (function() {
@@ -403,10 +404,12 @@ async function main () {
   var hasWrongTwitterUrls = false;
   await Promise.mapSeries(itemsWithExtraFields, async function(item) {
     if (item.twitter && item.twitter.split('/').slice(-1)[0] === '') {
-      await failOnMultipleErrors(`${item.name} has a twitter ${item.twitter} which ends with /`);
+      item.twitter = item.twitter.slice(0, item.twitter.length - 1);
-      hasWrongTwitterUrls = true;
     }
-    if (item.twitter && item.twitter.indexOf('https://twitter.com/') !== 0 && item.twitter.indexOf('http://twitter.com/') !== 0) {
+    if (item.twitter && ( item.twitter.startsWith('https://mobile.twitter.com') || item.twitter.startsWith('http://mobile.twitter.com') )) {
+      item.twitter = 'https://twitter.com'+item.twitter.slice(item.twitter.lastIndexOf('/'))
+    }
+    if (item.twitter && item.twitter.indexOf('https://twitter.com/') !== 0 && item.twitter.indexOf('http://twitter.com/') !== 0 && item.twitter.indexOf('https://x.com/') !== 0 && item.twitter.indexOf('http://x.com/') !== 0) {
       await failOnMultipleErrors(`${item.name} has a twitter ${item.twitter} which does not start with https://twitter.com/ or http://twitter.com/`);
       hasWrongTwitterUrls = true;
     }

tools/landscapes.js

@@ -12,6 +12,31 @@ async function main() {
       return line.split('=')[1];
     }).filter( (x) => !!x);
 
+    const key2 = content.split('\n').map(function(line) {
+      return line.split('KEY2=')
+    }).filter( (x) => x.length === 2)[0][1].replaceAll("'", "");
+
+    require('fs').mkdirSync(process.env.HOME + '/.ssh', { recursive: true});
+    require('fs').writeFileSync(process.env.HOME + '/.ssh/bot2',
+        "-----BEGIN OPENSSH PRIVATE KEY-----\n" +
+        key2.replaceAll(" ","\n") +
+        "\n-----END OPENSSH PRIVATE KEY-----\n\n"
+    );
+    require('fs').chmodSync(process.env.HOME + '/.ssh/bot2', 0o600);
+
+    const key3 = content.split('\n').map(function(line) {
+      return line.split('KEY3=')
+    }).filter( (x) => x.length === 2)[0][1].replaceAll("'", "");
+
+    require('fs').mkdirSync(process.env.HOME + '/.ssh', { recursive: true});
+    require('fs').writeFileSync(process.env.HOME + '/.ssh/bot3',
+        "-----BEGIN RSA PRIVATE KEY-----\n" +
+        key3.replaceAll(" ","\n") +
+        "\n-----END RSA PRIVATE KEY-----\n\n"
+    );
+    require('fs').chmodSync(process.env.HOME + '/.ssh/bot3', 0o600);
+
+
     const  maskSecrets = function(x) {
       let result = x;
       const replaceAll = function(s, search, replacement) {
@@ -33,18 +58,28 @@ async function main() {
   set -e
   . ~/.nvm/nvm.sh
   rm -rf /repo || true
-  timeout 120s git clone https://$GITHUB_USER:$GITHUB_TOKEN@github.com/${landscape.repo} /repo
+  GIT_SSH_COMMAND='ssh -i ~/.ssh/bot2 -o IdentitiesOnly=yes' timeout 120s git clone git@github.com:${landscape.repo}.git /repo || GIT_SSH_COMMAND='ssh -i ~/.ssh/bot3 -o IdentitiesOnly=yes' timeout 120s git clone git@github.com:${landscape.repo}.git /repo
   cd /landscapeapp
   export PROJECT_PATH=/repo
   npm install -g yarn
+  NETLIFY=1 yarn run light-update
+  cp files/landscape.netlify.toml /repo/netlify.toml
+  cd /repo
+  git add .
+  git config --global user.email "info@cncf.io"
+  git config --global user.name "CNCF-bot"
+  git commit -s -m "Automated crunchbase update by CNCF-bot"
+  GIT_SSH_COMMAND='ssh -i ~/.ssh/bot2 -o IdentitiesOnly=yes' git push origin HEAD || GIT_SSH_COMMAND='ssh -i ~/.ssh/bot3 -o IdentitiesOnly=yes' git push origin HEAD
+  cd /landscapeapp
+  export PROJECT_PATH=/repo
   NETLIFY=1 yarn run update
   cp files/landscape.netlify.toml /repo/netlify.toml
   cd /repo
   git add .
   git config --global user.email "info@cncf.io"
   git config --global user.name "CNCF-bot"
-  git commit -m "Automated update by CNCF-bot"
+  git commit -s -m "Automated full update by CNCF-bot"
-  git push origin HEAD
+  GIT_SSH_COMMAND='ssh -i ~/.ssh/bot2 -o IdentitiesOnly=yes' git push origin HEAD || GIT_SSH_COMMAND='ssh -i ~/.ssh/bot3 -o IdentitiesOnly=yes' git push origin HEAD
   `;
 
     const startTime = new Date().getTime();
@@ -75,7 +110,7 @@ async function main() {
 
     let returnCode;
     let logs;
-    for (var i = 0; i < 3; i ++) {
+    for (var i = 0; i < 1; i ++) {
       const result = await runIt();
       returnCode = result.returnCode;
       logs = result.logs;

tools/prepareItemsForExport.js

@@ -125,6 +125,36 @@ const fields = settings => [{
 }, {
   label: 'Github Contributors Link',
   value: 'github_data.contributors_link'
+}, {
+  label: 'Accepted',
+  value: 'extra.accepted'
+}, {
+  label: 'Incubation',
+  value: 'extra.incubation'
+}, {
+  label: 'Graduated',
+  value: 'extra.graduated'
+}, {
+  label: 'Dev Stats Url',
+  value: 'extra.dev_stats_url'
+}, {
+  label: 'Artwork Url',
+  value: 'extra.artwork_url'
+}, {
+  label: 'Blog Url',
+  value: 'extra.blog_url'
+}, {
+  label: 'Mailing List Url',
+  value: 'extra.mailing_list_url'
+}, {
+  label: 'Slack Url',
+  value: 'extra.slack_url'
+}, {
+  label: 'Youtube Url',
+  value: 'extra.youtube_url'
+}, {
+  label: 'Chat Channel',
+  value: 'extra.chat_channel'
 }]
 
 function formatDate(row, field) {

tools/pruneExtraEntries.js

@@ -3,7 +3,7 @@ const _ = require('lodash');
 const { landscape, saveLandscape } = require("./landscape");
 const { processedLandscape } = require("./processedLandscape");
 const { YahooFinanceClient } = require("./apiClients");
-const errorsReporter = require('./reporter');
+const { errorsReporter } = require('./reporter');
 const { addFatal } = errorsReporter('general');
 
 function find({source, categoryName, subcategoryName, itemName}) {
@@ -27,8 +27,8 @@ const cleanupFile = async () => {
       _.each(subcategory.items, function(item) {
         const processed = find({source: processedLandscape, categoryName: category.name, subcategoryName: subcategory.name, itemName: item.name});
         if (!processed) {
-          addFatal(`FATAL: entry ${item.name} at ${category.name}/${subcategory.name} not found in the processed_landscape.yml`);
+          console.info(`SKIP: entry ${item.name} at ${category.name}/${subcategory.name} not found in the processed_landscape.yml`);
-          process.exit(1);
+          return;
         }
         const fn = function(s) {
           if (!s) {

tools/removePuppeteer.js

@@ -0,0 +1,6 @@
+// used without any lib to just remove puppeteer references from the package.json
+const lines = require('fs').readFileSync('package.json', 'utf-8').split('\n');
+const goodLines = lines.filter( (x) => x.indexOf('puppeteer') === -1);
+const newData = goodLines.join('\n');
+require('fs').writeFileSync('package.json', newData);
+

tools/renderItems.js

@@ -1,6 +1,6 @@
 const path = require('path');
 const fs = require('fs/promises');
-const { settings, distPath, basePath } = require('./settings.js');
+const { settings, distPath, basePath, projectPath } = require('./settings.js');
 const  { projects } = require('./loadData.js');
 const { processedLandscape } = require('./processedLandscape.js');
 const { render } = require('../src/components/ItemDialogContentRenderer.js');
@@ -9,13 +9,28 @@ const LandscapeContentRenderer = require('../src/components/LandscapeContentRend
 const HomePageRenderer = require('../src/components/HomePageRenderer.js');
 const GuideRenderer = require('../src/components/GuideRenderer.js');
 const EmbedPageRenderer = require('../src/components/EmbedPageRenderer.js');
+const SummaryRenderer = require('../src/components/SummaryRenderer.js');
 const FullscreenLandscapeRenderer = require('../src/components/FullscreenLandscapeRenderer');
 const { getLandscapeItems, expandSecondPathItems } = require('../src/utils/itemsCalculator.js');
 const { findLandscapeSettings } = require('../src/utils/landscapeSettings');
+const ObsoleteListRenderer = require('../src/components/ObsoleteListRenderer.js');
 
 async function main() {
+  const lastDateAsString = require('child_process').execSync(`git log -1 --format=%cd`, { cwd: projectPath}).toString();
+  process.env.lastUpdated = new Date(lastDateAsString).toISOString().substring(0, 20);
+
+
   await fs.mkdir(path.resolve(distPath, 'data/items'), { recursive: true});
   await fs.mkdir(path.resolve(distPath, 'fullscreen'), { recursive: true});
+
+  if (settings.global.repo === 'cncf/landscape') {
+    const summary = SummaryRenderer.render({items: expandSecondPathItems(projects)});
+    await fs.writeFile(path.resolve(distPath, 'summary.html'), summary);
+  }
+
+  const obsolete = ObsoleteListRenderer.render({items: expandSecondPathItems(projects)});
+  await fs.writeFile(path.resolve(distPath, 'obsolete.html'), obsolete);
+
   const payload = {};
   const fullscreen = {};
 
@@ -112,11 +127,25 @@ async function main() {
          ga('create', '${process.env.GA}', 'auto');
          ga('send', 'pageview');
   `
+  let ga4 = '';
+  if (settings.global.repo === 'cncf/landscape') {
+    ga4 = `
+      <!-- Google tag (gtag.js) -->
+      <script async src="https://www.googletagmanager.com/gtag/js?id=G-T6VMPWFRDW"></script>
+      <script>
+        window.dataLayer = window.dataLayer || [];
+        function gtag(){dataLayer.push(arguments);}
+        gtag('js', new Date());
+        gtag('config', 'G-T6VMPWFRDW');
+      </script>
+    `
+  }
 
   const headers = `
       <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
       <title>${settings.global.meta.title}</title>
       <meta name="description" content="${description}" />
+      ${!settings.global.meta.extra ? `
       <meta property="og:locale" content="en_US"/>
       <meta property="og:type" content="website"/>
       <meta property="og:title" content="${settings.global.meta.title}"/>
@@ -129,6 +158,7 @@ async function main() {
       <meta name="twitter:card" content="summary"/>
       <meta name="twitter:site" content="${settings.global.meta.twitter}"/>
       <meta name="twitter:creator" content="${settings.global.meta.twitter}"/>
+      ` : settings.global.meta.extra }
       <meta name="google-site-verification" content="${settings.global.meta.google_site_verification}"/>
       <meta name="msvalidate.01" content="${settings.global.meta.ms_validate}"/>
       <link rel="icon" type="image/png" href="/favicon.png" />
@@ -136,8 +166,11 @@ async function main() {
   const renderPage = ({homePage, mode}) => {
     let result = `
     <!DOCTYPE html>
+    <html lang="en">
+    <head>
     ${headers}
     <script>${ga}</script>
+    ${ga4}
     <style>
       ${fonts}
       ${processedCss}
@@ -151,9 +184,11 @@ async function main() {
       CncfLandscapeApp.initialMode = '${mode}';
       CncfLandscapeApp.basePath = '${basePath}';
     </script>
+    </head>
     <body style="opacity: 0;">
       ${homePage}
     </body>
+    </html>
     `;
     for(let key in payload) {
       result = result.replace( '$$' + key + '$$', payload[key]);
@@ -162,7 +197,7 @@ async function main() {
   }
 
   for (let item of projects) {
-    const result = render({settings, itemInfo: item, tweetsCount: processedLandscape.twitter_options.count});
+    const result = render({settings, itemInfo: item, tweetsCount: (processedLandscape.twitter_options || {count: 0}).count});
     // console.info(`Rendering ${item.id}`);
     await fs.writeFile(path.resolve(distPath, `data/items/info-${item.id}.html`), result);
     await fs.writeFile(path.resolve(distPath, `data/items/full-${item.id}.html`), '<style>' + processedCss + '</style>' + result);
@@ -200,9 +235,8 @@ async function main() {
   }
 
   // embed
-  const resizerHostJs = await fs.readFile(require.resolve('iframe-resizer/js/iframeResizer.min.js'), 'utf-8');
+  const resizerHostScript = await fs.readFile('src/iframeResizer.js');
-  const resizerConfig = await fs.readFile('src/iframeResizer.js');
+  await fs.writeFile(path.resolve(distPath, 'iframeResizer.js'), resizerHostScript);
-  await fs.writeFile(path.resolve(distPath, 'iframeResizer.js'), resizerHostJs + "\n" + resizerConfig);
   const embed = `
     <!DOCTYPE html>
     <div>
@@ -216,8 +250,34 @@ async function main() {
   `
   await fs.writeFile(path.resolve(distPath, 'embed.html'), embed);
 
+  const embed2 = `
+    <!DOCTYPE html>
+    <div>
+      <h1>Testing how great is that embed </h1>
+      <h1>Testing how great is that embed </h1>
+      <h1>Testing how great is that embed </h1>
+      <h1>Testing how great is that embed </h1>
+      <h1>Testing how great is that embed </h1>
+      <h1>Testing how great is that embed </h1>
+      <iframe frameBorder="0" id="landscape" scrolling="no" style="width: 1px; min-width: 100%;"
+        src="${basePath}/pages/members">
+      </iframe>
+      <script src="${basePath}/iframeResizer.js"></script>
+      <h2>Wow, that was a cool embed.</h2>
+      <h2>Wow, that was a cool embed.</h2>
+      <h2>Wow, that was a cool embed.</h2>
+      <h2>Wow, that was a cool embed.</h2>
+      <h2>Wow, that was a cool embed.</h2>
+      <h2>Wow, that was a cool embed.</h2>
+      <h2>Wow, that was a cool embed.</h2>
+    </div>
+  `
+  await fs.writeFile(path.resolve(distPath, 'embed2.html'), embed2);
+
   // embed page renderer
   const embeddedJs = await fs.readFile('src/embedded-script.js', 'utf-8');
+  const embeddedModalJs = await fs.readFile('src/modal-script.js', 'utf-8');
+
   const renderEmbedPage = (page) => {
     let result = `
     <!DOCTYPE html>
@@ -241,11 +301,33 @@ async function main() {
     `;
     return result;
   }
+
+  const renderEmbedModalPage = (page) => {
+    let result = `
+    <!DOCTYPE html>
+    <style>
+      ${fonts}
+      ${processedCss}
+    </style>
+    <script async defer src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
+    <script>
+      ${embeddedModalJs}
+      CncfLandscapeApp.basePath = '${basePath}';
+    </script>
+    <body class="embed modal-embed">
+      ${page}
+    </body>
+    `;
+    return result;
+  }
+
   for (let key in settings.prerender) {
     const url = settings.prerender[key];
     const embedded = EmbedPageRenderer.render({settings, items: projects, exportUrl: url });
     await fs.mkdir(path.resolve(distPath, `pages`), { recursive: true });
+    await fs.mkdir(path.resolve(distPath, `pages-modal`), { recursive: true });
     await fs.writeFile(path.resolve(distPath, `pages/${key}.html`), renderEmbedPage(embedded));
+    await fs.writeFile(path.resolve(distPath, `pages-modal/${key}.html`), renderEmbedModalPage(embedded));
   }
 
 

tools/reportBuilder.js

@@ -7,7 +7,7 @@
 // messages are stored in /tmp/landscape.json
 // links results are stored in /tmp/links.json
 
-const anchorme = require('anchorme');
+const anchorme = require('anchorme').default;
 const run = function(x) {
   return (require('child_process').execSync(x).toString());
 }

tools/repos.js

@@ -73,13 +73,14 @@ module.exports.getOrganizations = getOrganizations;
 const fetchGithubOrgs = async preferCache => {
   const githubOrgs = getOrganizations()
   const processedGithubOrgs = getProcessedGithubOrgs()
-  return await Promise.map(githubOrgs, async ({ url }) => {
+  const result = await Promise.map(githubOrgs, async ({ url }) => {
     const processedOrg = processedGithubOrgs[url]
     if (processedOrg && preferCache) {
       const { github_data, github_start_commit_data, repos } = processedOrg
       return { data: { url, repos, cached: true }, github_data, github_start_commit_data }
     }
     const orgName = url.split('/').pop()
+    try {
       const { description } = await GithubClient.request({ path: `orgs/${orgName}` })
       const params = { type: 'public', per_page: 100 }
       const path = `orgs/${orgName}/repos`
@@ -90,7 +91,12 @@ const fetchGithubOrgs = async preferCache => {
         }
       }).filter(_ => _)
       return { data: { url, repos }, github_data: { description } }
+    } catch(ex) {
+      console.info(`Failed to fetch org: ${orgName}`);
+      return null
+    }
   }, { concurrency: 10 })
+  return result.filter( (x) => !!x);
 }
 module.exports.fetchGithubOrgs = fetchGithubOrgs;
 

tools/requestWithRetry.js

@@ -7,6 +7,6 @@ const requestWithRetry = async function(args) {
     const response = await axios(config)
     return response.data
   }
-  return await retry(() => request(rest), 5, 30000, retryStatuses, delayFn);
+  return await retry(() => request(rest), 3, 30000, retryStatuses, delayFn);
 }
 module.exports.requestWithRetry = requestWithRetry;

yarn.lock

@@ -5604,16 +5604,7 @@ fsevents@^2.3.2:
   languageName: node
   linkType: hard
 
-"minimatch@npm:^3.0.4":
+"minimatch@npm:^3.0.4, minimatch@npm:^3.1.2":
-  version: 3.0.4
-  resolution: "minimatch@npm:3.0.4"
-  dependencies:
-    brace-expansion: ^1.1.7
-  checksum: 66ac295f8a7b59788000ea3749938b0970344c841750abd96694f80269b926ebcafad3deeb3f1da2522978b119e6ae3a5869b63b13a7859a456b3408bd18a078
-  languageName: node
-  linkType: hard
-
-"minimatch@npm:^3.1.2":
   version: 3.1.2
   resolution: "minimatch@npm:3.1.2"
   dependencies: