From a93448da0a23dc5ad2bfb24e043c477a1288bfb7 Mon Sep 17 00:00:00 2001 From: jaywcjlove <398188662@qq.com> Date: Tue, 6 Aug 2019 21:58:48 +0800 Subject: [PATCH] Add create sitemap.xml script. --- build/sitemap.js | 23 ++++ package-lock.json | 266 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 1 + 3 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 build/sitemap.js diff --git a/build/sitemap.js b/build/sitemap.js new file mode 100644 index 0000000000..f114f5e918 --- /dev/null +++ b/build/sitemap.js @@ -0,0 +1,23 @@ +const path = require('path'); +const fs = require('fs-extra'); +const SitemapGenerator = require('sitemap-generator'); + +const xmlFile = path.join(process.cwd(), '.deploy/sitemap.xml') +const generator = SitemapGenerator('http://192.168.31.69:1987', { + maxDepth: 0, + filepath: xmlFile, + maxEntriesPerFile: 50000, + stripQuerystring: true, + decodeResponses: true, +}); + +// register event listeners +generator.on('done', async () => { + const str = await fs.readFileSync(xmlFile); + const newStr = str.toString().replace(/http\:\/\/192\.168\.31\.69\:1987/g, 'https://wangchujiang.com/linux-command'); + await fs.outputFile(xmlFile, newStr); + console.log('sitemaps created!') +}); + +// start the crawler +generator.start(); \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 8efa6186ef..af41cd17eb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,9 +1,15 @@ { "name": "linux-command", - "version": "1.2.5", + "version": "1.2.12", "lockfileVersion": 1, "requires": true, "dependencies": { + "@types/node": { + "version": "12.6.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.6.9.tgz", + "integrity": "sha512-+YB9FtyxXGyD54p8rXwWaN1EWEyar5L58GlGWgtH2I9rGmLGBQcw63+0jw+ujqVavNuO47S1ByAjm9zdHMnskw==", + "dev": true + }, "abbrev": { "version": "1.1.1", "resolved": "http://registry.npm.taobao.org/abbrev/download/abbrev-1.1.1.tgz", @@ -122,6 +128,12 @@ "tweetnacl": "^0.14.3" } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=", + "dev": true + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -138,6 +150,20 @@ "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.2", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", + "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=", + "dev": true, + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, "chownr": { "version": "1.1.1", "resolved": "http://registry.npm.taobao.org/chownr/download/chownr-1.1.1.tgz", @@ -201,12 +227,57 @@ "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "dev": true }, + "cp-file": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cp-file/-/cp-file-6.0.0.tgz", + "integrity": "sha512-OtHMgPugkgwHlbph25wlMKd358lZNhX1Y2viUpPoFmlBPlEiPIRhztYWha11grbGPnlM+urp5saVmwsChCIOEg==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.2", + "make-dir": "^1.0.0", + "nested-error-stacks": "^2.0.0", + "pify": "^3.0.0", + "safe-buffer": "^5.0.1" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } + } + }, + "crypto-random-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-1.0.0.tgz", + "integrity": "sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4=", + "dev": true + }, "css-parse": { "version": "1.7.0", "resolved": "https://registry.npmjs.org/css-parse/-/css-parse-1.7.0.tgz", "integrity": "sha1-Mh9s9zeCpv91ERE5D8BeLGV9jJs=", "dev": true }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "dev": true, + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", + "dev": true + }, "dashdash": { "version": "1.14.1", "resolved": "http://registry.npm.taobao.org/dashdash/download/dashdash-1.14.1.tgz", @@ -216,6 +287,12 @@ "assert-plus": "^1.0.0" } }, + "date-fns": { + "version": "1.29.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-1.29.0.tgz", + "integrity": "sha512-lbTXWZ6M20cWH8N9S6afb0SBm6tMk+uUg6z3MqHPKE9atmsY3kJkTm8vKe93izJ2B2+q5MV990sM2CHgtAZaOw==", + "dev": true + }, "debug": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.0.1.tgz", @@ -256,6 +333,41 @@ "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", "dev": true }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "dev": true, + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dev": true, + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "dev": true, + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "http://registry.npm.taobao.org/ecc-jsbn/download/ecc-jsbn-0.1.2.tgz", @@ -278,6 +390,12 @@ "integrity": "sha512-IMn9dnwLMsgZjdUHswB/UZ0S8LQ/u+2/qjnHJ9tCtp3QHZsIYwJCiJOo2FT0i3CwwK/dtSODYtxuvzV4D9MY5g==", "dev": true }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "dev": true + }, "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", @@ -484,6 +602,33 @@ "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", "dev": true }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "dev": true, + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.4.0.tgz", + "integrity": "sha512-jItXPLmrSR8jmTRmRWJXCnGJsfy85mB3Wd/uINMXA65yrnFo0cPClFIUWzo2najVNSl+mx7/4W8ttlLWJe99pQ==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "http-signature": { "version": "1.2.0", "resolved": "http://registry.npm.taobao.org/http-signature/download/http-signature-1.2.0.tgz", @@ -638,6 +783,23 @@ "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==", "dev": true }, + "make-dir": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-1.3.0.tgz", + "integrity": "sha512-2w31R7SJtieJJnQtGc7RVL2StM2vGYVfqUOvUDxH6bC6aJTxPxTF0GnIgCyu7tjockiUWAYQRbxa7vKn34s5sQ==", + "dev": true, + "requires": { + "pify": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + } + } + }, "marked": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/marked/-/marked-0.6.2.tgz", @@ -693,6 +855,12 @@ "minipass": "^2.2.1" } }, + "mitt": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-1.1.3.tgz", + "integrity": "sha512-mUDCnVNsAi+eD6qA0HkRkwYczbLHJ49z17BGe2PYRhZL4wpZUFZGJHU7/5tmvohoma+Hdn0Vh/oJTiPEmgSruA==", + "dev": true + }, "mkdirp": { "version": "0.5.1", "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", @@ -742,6 +910,12 @@ } } }, + "nested-error-stacks": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz", + "integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug==", + "dev": true + }, "node-pre-gyp": { "version": "0.11.0", "resolved": "https://registry.npm.taobao.org/node-pre-gyp/download/node-pre-gyp-0.11.0.tgz", @@ -810,6 +984,15 @@ "set-blocking": "~2.0.0" } }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "dev": true, + "requires": { + "boolbase": "~1.0.0" + } + }, "number-is-nan": { "version": "1.0.1", "resolved": "http://registry.npm.taobao.org/number-is-nan/download/number-is-nan-1.0.1.tgz", @@ -859,6 +1042,15 @@ "os-tmpdir": "^1.0.0" } }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "path": { "version": "0.12.7", "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz", @@ -957,6 +1149,12 @@ "strict-uri-encode": "^1.0.0" } }, + "querystringify": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.1.1.tgz", + "integrity": "sha512-w7fLxIRCRT7U8Qu53jQnJyPkYZIaR4n5151KMfcJlO/A9397Wxb1amJvROTK6TOnp7PfoAmg/qXiNHI+08jRfA==", + "dev": true + }, "rc": { "version": "1.2.8", "resolved": "http://registry.npm.taobao.org/rc/download/rc-1.2.8.tgz", @@ -1020,6 +1218,12 @@ "uuid": "^3.3.2" } }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha1-kl0mAdOaxIXgkc8NpcbmlNw9yv8=", + "dev": true + }, "rimraf": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", @@ -1029,6 +1233,12 @@ "glob": "^7.0.5" } }, + "robots-parser": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/robots-parser/-/robots-parser-1.0.2.tgz", + "integrity": "sha512-c979P2TTe5Ezjg+VV5XWC59bab1w1Xyb8KLnJ5SKuwN7YH+ys8Y5KCZ2OP2J7zXVE7Yq0eYWlrqUM/urIbx9sg==", + "dev": true + }, "safe-buffer": { "version": "5.1.2", "resolved": "http://registry.npm.taobao.org/safe-buffer/download/safe-buffer-5.1.2.tgz", @@ -1072,6 +1282,44 @@ "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "dev": true }, + "simplecrawler": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/simplecrawler/-/simplecrawler-1.1.6.tgz", + "integrity": "sha512-TsXJVzrKZuExJPq7juipV76KIE+D3a9IWWve6v+BBCuwFWhFYl9BOFZtARWBwb08KtI4eTZ/GGu9QuboJzo62w==", + "dev": true, + "requires": { + "async": "^2.1.4", + "iconv-lite": "^0.4.13", + "robots-parser": "^1.0.0", + "urijs": "^1.18.11" + } + }, + "sitemap-generator": { + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/sitemap-generator/-/sitemap-generator-8.4.0.tgz", + "integrity": "sha512-afpw4M879Z+1SFHqbz3QGarMfxoUHWE6NQouHRZ0NonDmd8k7Dzkpo4GplHpgt5xEEcZ3ep8H2tNW7SDy7TYMQ==", + "dev": true, + "requires": { + "async": "2.6.1", + "cheerio": "1.0.0-rc.2", + "cp-file": "6.0.0", + "crypto-random-string": "1.0.0", + "date-fns": "1.29.0", + "lodash": "4.17.11", + "mitt": "1.1.3", + "normalize-url": "3.3.0", + "simplecrawler": "1.1.6", + "url-parse": "1.4.3" + }, + "dependencies": { + "normalize-url": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-3.3.0.tgz", + "integrity": "sha512-U+JJi7duF1o+u2pynbp2zXDW2/PADgC30f0GsHZtRh+HOcXHnw137TrNlyxxRvWW5fjKd3bcLHPxofWuCjaeZg==", + "dev": true + } + } + }, "sort-keys": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", @@ -1307,6 +1555,22 @@ "punycode": "^2.1.0" } }, + "urijs": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/urijs/-/urijs-1.19.1.tgz", + "integrity": "sha512-xVrGVi94ueCJNrBSTjWqjvtgvl3cyOTThp2zaMaFNGp3F542TR6sM3f2o8RqZl+AwteClSVmoCyt0ka4RjQOQg==", + "dev": true + }, + "url-parse": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.3.tgz", + "integrity": "sha512-rh+KuAW36YKo0vClhQzLLveoj8FwPJNu65xLb7Mrt+eZht0IPT0IXgSv8gcMegZ6NvjJUALf6Mf25POlMwD1Fw==", + "dev": true, + "requires": { + "querystringify": "^2.0.0", + "requires-port": "^1.0.0" + } + }, "util": { "version": "0.10.4", "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz", diff --git a/package.json b/package.json index b2de2b946e..9974ca3ca8 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "marked": "^0.6.2", "path": "^0.12.7", "prismjs": "^1.16.0", + "sitemap-generator": "^8.4.0", "sqlite3": "^4.0.6", "stylus": "^0.54.5", "uglify-js": "^3.4.9"