CSDN博客搬家到WordPress
1,113 views
0
一直想把CSDN上写的博客,搬家移植到自己的WordPress博客,网上介绍了一些方法,例如先把CSDN博客导入博客园,然后从博客园导出xml文件,最后用WordPress导入工具导入进WordPress博客,操作步骤太繁杂。
于是周末在家自己动手用Python写了爬虫脚本,利用WordPress自定义的投稿功能(wp_insert_post),导入进WordPress博客,步骤只有简单的两步:
步骤1:Python脚本爬取CSDN博客,包括博客标题,正文,发布日期,浏览数,标签
步骤2:利用WordPress投稿功能插入到WordPress,把所有CSDN博客导入WordPress
Python完整代码:
#!/usr/bin/env python # -*- coding:utf8 -*- ''' author : yanggang@mimvp.com date : 2015-12-01 blog : https://blog.mimvp.com demo : https://blog.mimvp.com/category/csdn_blog/ 安装库文件: 1. MySQLdb,连接MySQL数据库,修改浏览次数 2. bs4, 格式化HTML,抓取过滤CSDN网页内容 WordPress博客POST文章接口API https://blog.mimvp.com/tougao/ 查询自动POST文章ID: select id from wp_posts where post_title like '【米扑代理】%' 修改POST文章浏览数: update wp_postmeta set meta_value=meta_value+100 where meta_key='views' and post_id in (select id from wp_posts where post_title like '【米扑代理】%'); 修改浏览次数小于100 select post_id, meta_value from wp_postmeta where meta_key='views' and meta_value < 100 and post_id in (select id from wp_posts where post_status = 'publish'); ''' import time, datetime import os, random import urllib, urllib2 import base64 import bs4 import MySQLdb import sys reload(sys) sys.setdefaultencoding('utf-8') # 排除要抓取的CSDN博客链接 EXCLUDE_BLOG_URL = [ 'http://blog.csdn.net/sunboy_2050/article/details/45868613', 'http://blog.csdn.net/sunboy_2050/article/details/49449319', 'http://blog.csdn.net/sunboy_2050/article/details/48622023', 'http://blog.csdn.net/sunboy_2050/article/details/47858847', ] # 根据CSDN博客标题打标签tags CATEGORY_DICT = { 'Python' : 1, 'C/C++' : 2, '网络常识' : 3, 'Algrithm' : 11, 'Clojure' : 4, 'CSDN' : 5, 'Git/SVN' : 6, 'Go' : 7, 'HTML/CSS/JS' : 8, 'iOS/Android' : 9, 'Java/JSP' : 113, 'Linux/Unix' : 95, 'MacBook' : 111, 'Nginx/Apache' : 112, 'PHP' : 777, 'SQL/NoSQL' : 114, 'Storm/Hadoop' : 115, 'WP技巧' : 12, '产品经理' : 116, '创业邦' : 3, '理财' : 117, '生活小札' : 4, '科技资讯' : 10, '米扑代理' : 118, '系统架构' : 97, '软件测试' : 119, 'leetcode' : 11, '链表' : 11, '算法' : 11, 'leetcode' : 11, 'django' : 1, 'tornado' : 1, 'c语言' : 2, 'c++' : 2, 'c#' : 2, 'vc' : 2, 'qt' : 2, '网络' : 3, '算法' : 11, 'Git' : 6, 'SVN' : 6, '版本控制' : 6, 'HTML' : 8, 'CSS' : 8, 'JS' : 8, 'javascript' : 8, 'iOS' : 9, 'Android' : 9, 'Java' : 113, 'JSP' : 113, 'JVM' : 113, 'Spring' : 113, 'Eclipse' : 113, 'Linux' : 95, 'Unix' : 95, 'Ubuntu' : 95, 'CentOS' : 95, 'Shell' : 95, 'AWK' : 95, 'vim' : 95, 'Nginx' : 112, 'Apache' : 112, 'Tomcat' : 112, '数据库' : 114, 'SQL' : 114, 'NoSQL' : 114, 'MySQL' : 114, 'Redis' : 114, 'Memcache': 114, 'mongo' : 114, 'sqlite' : 114, 'WP' : 12, 'WordPress' : 12, '软件' : 119, '测试' : 119, 'Storm' : 115, 'Hadoop' : 115, } # CSDN博客用urllib2 + header抓取不了,改用了curl爬取网页 headers = { 'Use-Agent' : 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36', 'Cookie' : 'bdshare_firstime=1430381399166; uuid_tt_dd=-4349350129693538496_20150430; __gads=ID=b549fb1461110ca8:T=1430381399:S=ALNI_MbhiN2nMgWWiXPP25667Pq-BPZf5g; CloudGuest=dbieDkbnW7cz5P9qimYEjpecak8Udv0BPR8Iflg0PlBd3HR1Wj+RyQQksR2cDE9ab/hPXNGjFpuKsRe1dFjVJjY+mf3bfeWiP6kN0TKk1rY6g5SOuowPs/8F5FJJBdddW71JZ7rp4Q9b8DsLk2TASIPHLj3iL599bPUGKga0mRsaTJi0td73QBZaNlY7+VAl; __qca=P0-127195741-1432118980220; lzstat_uv=29735680241046032471|2955225@3582543@2675686@3411160; __utma=1722629.287164464.1430711804.1438215324.143920134.51; __utmz=1722629.14380001130.49.44.utmcsr=baidu|utmccn=(organic)|utmcmd=organic; UN=Sunboy_2050; UE="yanggang_2050@19.com"; __message_district_code=110000; uuid=92c68418-b2bd-41bc-846d-a1e4e9cf094; _ga=GA1.2.287164464.1430711804; scvh=2011-09-09+17%3a11%3a33+003; FullCookie=1; ViewMode=list; avh=8471508%2c298041%2c45868613%2c2659431%2c17398807; __message_sys_msg_id=0; __message_gu_msg_id=0; __message_cnel_msg_id=0; __message_in_school=0; lzstat_ss=3875140043_20_1448553416_2955225; dc_tos=nyewqw; dc_session_id=1448524616939', 'Host' : 'blog.csdn.net', 'DNT' : 1, 'Cache-Control' : 'max-age=0', 'Connection' : 'keep-alive', 'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Encoding' : 'gzip, deflate, sdch', 'Accept-Language' : 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4', 'DNT' : 1, } # hangzhou of 115.29.237.28 PROXY_MYSQL_SERVER = { "host" : "localhost", "port" : 3306, "user" : "root", "passwd" : "123456", "dbname" : "wp_blog", } # 获取当前的实时日期 def get_now_datetime(): return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') def split_category(blog_title=''): cat_keys = CATEGORY_DICT.keys() blog_title = blog_title.lower() print blog_title cat_list = ['5'] for cat in cat_keys: cat_lower = cat.lower() if blog_title.find(cat_lower) >= 0: cat_value = str(CATEGORY_DICT.get(cat)) cat_list.append(cat_value) cat_join = ",".join(cat_list) return cat_join # 爬取博客链接,先爬取翻页数 ——》 拼接翻页链接 ——》 爬取翻页正文 ——》 获取博客链接 def spider_blog_url_list(blog_url='http://blog.csdn.net/sunboy_2050/'): blog_url_list = [] blog_url = blog_url print("blog_url: " + blog_url) ######################### 爬取博客翻页链接 ######################## try: # req = urllib2.Request(blog_url, headers=headers) # content = urllib2.urlopen(req).read() content = os.popen('curl ' + blog_url).read() content = bs4.BeautifulSoup(content) content = content.prettify() content = bs4.BeautifulSoup(content, from_encoding='GB18030') table_soup = content.find('div', {"id":"papelist"}).find_all('a') last_a = table_soup[len(table_soup)-1] last_a_href = last_a['href'] list_a_prefix, list_num = os.path.split(last_a_href) list_num = int(list_num) blog_url_page_list = [] for i in range(list_num): i += 1 page_url = 'http://blog.csdn.net' + list_a_prefix + '/' + str(i) print page_url blog_url_page_list.append(page_url) except Exception as ex: print("spider_url() - error_msg: " + str(ex)) ######################### 爬取每页的博客链接 ######################## blog_url_set = set() list_len = len(blog_url_page_list) index = 0 for page_url in blog_url_page_list: index += 1 print("++++ ", index, "/", list_len, "page_url: " + page_url) try: content = os.popen('curl ' + page_url).read() content = bs4.BeautifulSoup(content) content = content.prettify() content = bs4.BeautifulSoup(content) span_list = content.find_all('span', {"class":"link_title"}) for span in span_list: a_list = span.find_all('a') for a in a_list: href = a['href'] blog_url = 'http://blog.csdn.net' + href print("blog_url: " + blog_url) blog_url_set.add(blog_url) except Exception as ex: print("spider_url() - error_msg: " + str(ex)) blog_url_list = blog_url_set return blog_url_list # 爬取博客正文 def spider_blog_url(blog_url='http://blog.csdn.net/sunboy_2050/article/details/45868613'): print blog_url if blog_url in EXCLUDE_BLOG_URL: print("blog_url is IN EXCLUDE_BLOG_URL, blog_url: " + blog_url) return blog_title = '' blog_content = '' blog_tags = '' blog_cat = '5' blog_postdate = '' blog_viewsCount = 0 try: content = os.popen('curl ' + blog_url).read() content = bs4.BeautifulSoup(content) content = content.prettify() content = bs4.BeautifulSoup(content) blog_title = content.find('span', {"class":"link_title"}).text.strip() blog_title = blog_title.replace("\n", "").replace("[置顶]", "").strip() blog_cat = split_category(blog_title) tags_set = set() tags_link = content.find('span', {"class":"link_categories"}) if tags_link: a_list = tags_link.find_all('a') for a in a_list: tags_set.add(a.text.strip()) blog_tags = ",".join(tags_set) blog_postdate = content.find('span', {"class":"link_postdate"}).text.strip() blog_postdate = blog_postdate + ":" + str(random.randint(10,60)) blog_viewsCount = content.find('span', {"class":"link_view"}).text.strip() blog_viewsCount = blog_viewsCount.replace("人阅读", "") blog_content = content.find('div', {"class":"article_content"}) # 语法高亮,例如: <pre class="python" xxx ===> <pre class="brush:python" xxxx blog_content = str(blog_content).replace('<pre class="', '<pre class="brush:') except Exception as ex: print("spider_blog_url() - error_msg: " + str(ex)) print blog_title print blog_tags print blog_postdate print blog_viewsCount print blog_cat # print blog_content post_blog(blog_title, blog_tags, blog_content, blog_postdate, blog_viewsCount, blog_cat, blog_url) # 爬取博客正文 def spider_blog(blog_root='http://blog.csdn.net/sunboy_2050/'): blog_url_list = spider_blog_url_list(blog_root) list_len = len(blog_url_list) index = 0 for blog_url in blog_url_list: index += 1 print "++++++++++++++ ", index, "/", list_len, "++++++++++++++ ", blog_url spider_blog_url(blog_url) # 写入博客, cat=5 : CSDN分类 def post_blog(title='test_title', tags='test_tag', content='test_content', postdate='', viewsCount=0, cat='5', blog_url=''): if postdate == '': postdate = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') post_data = { 'tougao_form' : 'blog_mimvp', 'tougao_authorname' : 'admin', 'tougao_authoremail' : 'yanggang@mimvp.com', 'tougao_authorblog' : 'blog.mimvp.com', 'tougao_title' : title, 'tougao_tags' : tags, 'tougao_cat' : cat, 'tougao_content' : content, 'tougao_date' : postdate, } content_head = "<div style='font-size: 16px;'>" content_foot = "</div><div style='margin: 50px auto 50px;'><h3><font color='red'>原文:</font> <a target='_blank' href='{blog_url}'>{blog_title}</a></h3></div>".format(blog_url=blog_url, blog_title=title) tougao_content = content_head + str(content) + str(content_foot) try: POST_URL = 'https://blog.mimvp.com/tougao/' post_data['tougao_content'] = tougao_content post_data = urllib.urlencode(post_data) req = urllib2.Request(POST_URL, data=post_data) res = urllib2.urlopen(req).read() except Exception as ex: print("error_msg: " + str(ex)) # 延时一分钟后,等文章发布后,修改浏览次数 print("sleep 3, then modify post_views...") time.sleep(3) modify_post_views(title, viewsCount) # 文章发布后,修改浏览次数 def modify_post_views(tougao_title='', viewsCount=100): if tougao_title == '': print("error: no post title") return sql = "update wp_postmeta set meta_value=meta_value+{viewsCount} where meta_key='views' and post_id in (select id from wp_posts where post_title = '{tougao_title}');".format(viewsCount=viewsCount, tougao_title=tougao_title) print sql try: sql_conn = MySQLdb.connect(host=PROXY_MYSQL_SERVER['host'], port=int(PROXY_MYSQL_SERVER['port']), user=PROXY_MYSQL_SERVER['user'], passwd=PROXY_MYSQL_SERVER['passwd'], db=PROXY_MYSQL_SERVER['dbname'], charset='utf8') sql_cursor = sql_conn.cursor() sql_cursor.execute(sql) sql_cursor.close() sql_conn.close() except Exception, ex: print("check_except_province_little_proxy() -- error_msg: %r" % ex) if __name__ == '__main__': spider_blog()
博客搬家示例
导出 CSDN原博客 : http://blog.csdn.net/sunboy_2050
导入 WordPress博客: https://blog.mimvp.com/category/csdn_blog/
米扑博客: https://blog.mimvp.com
参考推荐:
版权所有: 本文系米扑博客原创、转载、摘录,或修订后发表,最后更新于 2017-01-02 19:07:35
侵权处理: 本个人博客,不盈利,若侵犯了您的作品权,请联系博主删除,莫恶意,索钱财,感谢!