import urllib.request
import urllib.parse

def loadPage(url):
    '''
        Function: Fetching url and accessing the webpage content
        url: the wanted webpage url
    '''
    headers = {
            'Accept': 'text/html',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
    }
    print('To send HTTP request to %s ' % url)
    request = urllib.request.Request(url, headers=headers)
    
    return urllib.request.urlopen(request).read().decode('utf-8')

def writePage(html, filename):
    '''
        Function : To write the content of html into a local file
        html : The response content
        filename : the local filename to be used stored the response
    '''
    print('To write html into a local file %s ...' % filename)
    with open(filename, 'w', encoding='utf-8') as f:
        f.write(str(html))
    print('Work done!')
    print('---'*10)

def tiebaCrawler(url, beginPage, endPage, keyword):
    '''
        Function: The scheduler of crawler, is used to access every wanted url in turns
        url: the url of tieba's webpage
        beginPage: initial page
        endPage: end page
        keyword: the wanted keyword
    '''
    for page in range(beginPage,endPage+1):
        pn = (page - 1) * 50
        queryurl = url + '&pn=' + str(pn)
        filename = keyword + str(page) + '_tieba.html'
        writePage(loadPage(queryurl), filename)
        

if __name__ == '__main__':
    kw = input('Please input the wanted tieba\'s name:')
    beginPage = int(input('The beginning page number:'))
    endPage = int(input('The ending page number:'))
    
    # 百度贴吧查询 url 例子：http://tieba.baidu.com/f?ie=utf-8&kw=%E8%B5%B5%E4%B8%BD%E9%A2%96&red_tag=m2239217474
    url = 'http://tieba.baidu.com/f?ie=utf-8&'
    key = urllib.parse.urlencode({'kw':kw})
    queryurl = url + key
    
    tiebaCrawler(queryurl, beginPage, endPage, kw)