# -*- coding:utf-8 -*-

import urllib2
import re
import csv
from socket import error as SocketError
import codecs
import time
import get_mobile_id


class JD:
    def __init__(self):
        # 手机分类页面
        self.baseUrl = 'http://list.jd.com/list.html?cat=9987%2C653%2C655&go=0'
        # 手机品牌名称
        self.fileNameList = []
        # 产品参数
        self.paramList = []


    # 开启网络请求
    def requestOpener(self, url):
        flag = 0
        while True:
            try:
                opener = urllib2.build_opener(urllib2.HTTPHandler())
                if flag != 0:
                    opener.addheaders = [('User_Agent',
                                          'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.112 Safari/537.36')]
                else:
                    opener.addheaders = [('User_Agent',
                                          'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0')]
                response = opener.open(url, timeout=20)
                return response.read()
            except urllib2.HTTPError, e:
                print e.code
                #return ''
            except urllib2.URLError, e:
                print e.reason
                #return ''
            except SocketError, e:
                print 'SocketError: ' + str(e.errno)
                #return ''
            flag += 1
            time.sleep(3)
            if flag == 3:
                return ''

    # 创建文件
    def createCSV(self, name):
        #print name
        try:
            with open(name, 'ab') as f:
                f.write(codecs.BOM_UTF8)
                w = csv.writer(f, dialect='excel')
                w.writerow([])

            self.fileNameList.append(name)
        except Exception:
            print '创建文件失败'

    # 写入文件
    def writeCSV(self, file, boo):
        try:
            with open(file, 'ab') as f:
                f.write(codecs.BOM_UTF8)
                w = csv.writer(f, dialect='excel')
                if boo:
                    w.writerow(['品牌','型号','价格','上市年份','上市月份','操作系统',
                               '操作系统版本','CPU品牌','CPU型号','CPU核数',
                               'GPU','机身内存','运行内存','最大存储扩展',
                               '屏幕尺寸','分辨率','后置摄像头','前置摄像头',
                                '链接','备注','店铺','评价'])

                for x in self.paramList:
                    #print x
                    w.writerow(x)
        except Exception:
            print '写入数据失败'


    # 打开产品详情页获取参数
    def getDetailParame(self, content, url, name, comment, shop, price):
        p1 = re.compile('<td class=.*?tdTitle.*?>(.*?)<.*?<td>(.*?)<', re.S)
        r1 = re.findall(p1, content)

        parameDict = {}
        parames = []
        for i in r1:
            parameDict[i[0]] = i[1]

        #print len(parameDict)
        parames.append(parameDict.get('品牌',''))
        parames.append(parameDict.get('型号',''))
        parames.append(price)
        parames.append(parameDict.get('上市年份',''))
        parames.append(parameDict.get('上市月份',''))
        parames.append(parameDict.get('操作系统',''))
        parames.append(parameDict.get('操作系统版本',''))
        parames.append(parameDict.get('CPU品牌',''))
        parames.append(parameDict.get('CPU型号',''))
        parames.append(parameDict.get('CPU核数',''))
        parames.append(parameDict.get('GPU',''))
        parames.append(parameDict.get('机身内存',''))
        parames.append(parameDict.get('运行内存',''))
        parames.append(parameDict.get('最大存储扩展',''))
        parames.append(parameDict.get('屏幕尺寸',''))
        parames.append(parameDict.get('分辨率',''))
        parames.append(parameDict.get('后置摄像头',''))
        parames.append(parameDict.get('前置摄像头',''))
        parames.append('http:'+url)
        parames.append(name)
        parames.append(shop)
        parames.append(comment)

        self.paramList.append(parames)
        #print len(self.paramList)
        #print '获取完成'


    # 获取产品价格
    def getPrice(self, id):
        co = self.requestOpener(
            'http://p.3.cn/prices/get?type=1&area=1_72_2799&pdtk=&pduid=360917087&pdpin=&pdbp=0&skuid=J_' + id + '&callback=cnp')
        if co != '':
            p = re.compile('"p":"(.*?)"', re.S)
            r = re.search(p, co)
            return r.group(1)
        else:
            return ''


    # 获取产品ID
    def getProductId(self, url):
        try:
            p = re.compile('/.*?/(\d.*?).htm', re.S)
            r = re.search(p, url)
            return r.group(1)
        except Exception:
            print '获取产品ID失败'
            return ''



    # 获取每个产品的详情页链接
    def getDetailUrl(self, content):
        try:
            pattern = re.compile('<div class="p-icons J-pro-icons".*?href="(.*?)".*?<em>(.*?)</em>.*?#comment">(.*?)<.*?data-shop_name="(.*?)"', re.S)
            items = re.findall(pattern, content)
            for i in items:
                print 'url----: ' + i[0] + ' name-----: ' + i[1]
                id = self.getProductId(i[0])
                if id != '':
                    price = self.getPrice(id)
                    co = self.requestOpener('http://item.m.jd.com/ware/detail.json?wareId=' + id)
                    if co != '':
                        self.getDetailParame(co, i[0], i[1], i[2], i[3], price)

        except Exception:
            print '获取产品链接失败'


    # 开始执行任务
    def start(self, fromPge, toPge):

        for i in range(fromPge, toPge+1):
            print '第' + str(i) + '页扫描开始～～～～～～～～～～～～～～～～'
            content = self.requestOpener(self.baseUrl + str(i))
            self.getDetailUrl(content)
            if i==fromPge:
                self.writeCSV(True)
            else:
                self.writeCSV(False)
            self.paramList = []


    # 获取产品分页码和url
    def getPerProduct(self, url):
        try:
            content = self.requestOpener(url)
            #print content
            pattern = re.compile('<span class="fp-text">.*?<i>(.*?)</i>.*?class="hide ">0</a>.*?href="(.*?)"',re.S)
            res = re.search(pattern, content)
            return res
        except Exception:
            print '获取手机的分页信息异常'
            return None


    # 根据分页获取每页数据
    def getPgeProducts(self, url, file):
        res = self.getPerProduct(url)
        # 处理有分页情况
        if res != None:
            print res.group(1) + res.group(2)
            num = res.group(1)
            url = res.group(2)
            url = url.replace('page=1', 'page=')

            for i in range(1, int(num) + 1):
                u = url.replace('page=', 'page=' + str(i))
                print u
                content = self.requestOpener('http://list.jd.com' + u)
                self.getDetailUrl(content)
                if i == 1:
                    self.writeCSV(file, True)
                else:
                    self.writeCSV(file, False)
                self.paramList = []
        else:
            content = self.requestOpener(url)
            self.getDetailUrl(content)
            self.writeCSV(file, True)
            self.paramList = []


    # 获取品牌的英文名
    def getEngName(self, name):
        try:
            p = re.compile('\\w+', re.S)
            r = re.search(p, name)
            return r.group(0).upper()
        except Exception:
            print '获取品牌英文名异常'
            return name

    # 推荐品牌
    def index(self):
        startTime = time.time()

        content = self.requestOpener(self.baseUrl)
        p = re.compile('data-initial=\'.*?href="(.*?)".*?title="(.*?)"', re.S)
        s = re.findall(p, content)
        for i in s:
            print i[0] + ' tt ' + i[1]
            file_name = './file/'+self.getEngName(i[1])+'.csv'

            self.getPgeProducts('http://list.jd.com' + i[0], file_name)

        print '总共用时' + str(time.time() - startTime)


    # 选择品牌
    def select(self):
        startTime = time.time()
        mo_id = get_mobile_id.GETID().start()
        for i in mo_id:
            print i
            url = 'http://list.jd.com/list.html?cat=9987%2C653%2C655&go=0&ev=exbrand_' + i[0]
            file_name = './file/' + self.getEngName(i[1]) + '.csv'
            # print url
            # print file_name
            self.getPgeProducts(url, file_name)

        print '总共用时' + str(time.time() - startTime)


jd = JD()
# jd.index()
jd.select()