diff --git a/build/crawler.js b/build/crawler.js new file mode 100644 index 0000000000..be86da80e6 --- /dev/null +++ b/build/crawler.js @@ -0,0 +1,45 @@ +var request = require('superagent'); +var cheerio = require('cheerio'); +var toMarkdown = require('to-markdown'); +var path = require('path'); +var fs = require('fs'); + + +var param = process.argv.slice(2); + +var from_path = param[0]; +var to_path = param[1]; + +// CreatMarkdown(from_path,to_path) + +function CreatMarkdown(from_path,to_path){ + + var new_to_path = path.join(path.dirname(__dirname),to_path) + new_to_path = path.dirname(new_to_path) + mkdirsSync(new_to_path,0777,function(){ + + request.get(from_path).end(function(err, res){ + console.log("to_path::",to_path) + fs.writeFileSync(to_path, toMarkdown(res.text).toString() ,'utf-8'); + console.log(" → ",to_path) + }); + + }); +} + + +// 同步循环创建所有目录 resolvePath +function mkdirsSync(dirpath, mode, callback) { + if(fs.existsSync(dirpath)){ + callback&&callback(dirpath); + return true; + }else{ + if(mkdirsSync(path.dirname(dirpath), mode)){ + fs.mkdirSync(dirpath, mode, callback); + callback&&callback(dirpath); + return true; + }else{ + callback&&callback(dirpath); + } + } +}; \ No newline at end of file diff --git a/package.json b/package.json index e8b12f521b..649a1d1901 100644 --- a/package.json +++ b/package.json @@ -32,8 +32,11 @@ "watch": "^1.0.1" }, "dependencies": { + "cheerio": "^0.22.0", "gh-pages": "^0.12.0", "loading-cli": "^1.0.2", + "superagent": "^3.0.0", + "to-markdown": "^3.0.3", "uglify-js": "^2.7.4" } }